diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index 11e746f1b1b0033f8b4ff22c57d3e477f1f6ef6c..a13792dab9b7d216427e296eced01dcc515cd924 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -12,6 +12,11 @@ prompt_id: null extra_backup_patterns: - "/root" - "/home" +pip_check_requirements: + - click==6.7 + - first==2.0.1 + - pip-tools==1.9.0 + - six==1.10.0 # Internal use only. prompt_colour_mapping: diff --git a/roles/common/files/pip_check_requirements_upgrades.sh b/roles/common/files/pip_check_requirements_upgrades.sh new file mode 100755 index 0000000000000000000000000000000000000000..9d721287bebd8400ec0b811d2f31ee871a399b81 --- /dev/null +++ b/roles/common/files/pip_check_requirements_upgrades.sh @@ -0,0 +1,269 @@ +#!/bin/bash +# +# pip_check_requirements_upgrades.sh +# +# Copyright (C) 2017, Branko Majic +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +program="pip_check_requirements_upgrades.sh" + +function usage() { + cat <. +EOF +} + +function version() { + cat < | +| | +| This program is free software: you can redistribute it and/or modify | +| it under the terms of the GNU General Public License as published by | +| the Free Software Foundation, either version 3 of the License, or | +| (at your option) any later version. | +| | +| This program is distributed in the hope that it will be useful, | +| but WITHOUT ANY WARRANTY; without even the implied warranty of | +| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | +| GNU General Public License for more details. | +| | +| You should have received a copy of the GNU General Public License | +| along with this program. If not, see . | ++-----------------------------------------------------------------------+ + +EOF +} + +# Set-up colours for message printing if we're not piping and terminal is +# capable of outputting the colors. +_color_terminal=$(tput colors 2>&1) +if [[ -t 1 ]] && (( ${_color_terminal} > 0 )); then + _text_bold=$(tput bold) + _text_white=$(tput setaf 7) + _text_blue=$(tput setaf 6) + _text_green=$(tput setaf 2) + _text_yellow=$(tput setaf 3) + _text_red=$(tput setaf 1) + _text_reset=$(tput sgr0) +else + _text_bold="" + _text_white="" + _text_blue="" + _text_green="" + _text_yellow="" + _text_red="" + _text_reset="" +fi + +# Set-up functions for printing coloured messages. +function debug() { + if [[ $debug != 0 ]]; then + echo "${_text_bold}${_text_blue}[DEBUG]${_text_reset}" "$@" + fi +} + +function info() { + echo "${_text_bold}${_text_white}[INFO] ${_text_reset}" "$@" +} + +function success() { + echo "${_text_bold}${_text_green}[OK] ${_text_reset}" "$@" +} + +function warning() { + echo "${_text_bold}${_text_yellow}[WARN] ${_text_reset}" "$@" +} + +function error() { + echo "${_text_bold}${_text_red}[ERROR]${_text_reset}" "$@" >&2 +} + +# Clean-up command for temporary files. +function on_exit() { + debug "Cleaning-up temporary file: $tmp_current" + [[ -f $tmp_current ]] && rm "$tmp_current" + + debug "Cleaning-up temporayr file: $tmp_new" + [[ -f $tmp_new ]] && rm "$tmp_new" +} +trap on_exit EXIT + +# Define error codes. +SUCCESS=0 +ERROR_ARGUMENTS=1 +ERROR_CONFIG_DIR=2 +ERROR_PIP_TOOLS_MISSING=3 + +# Disable debug and quiet modes by default. +debug=0 +quiet=0 + +# Set-up defaults. +virtualenv="" + +# If no arguments were given, just show usage help. +if [[ -z $1 ]]; then + usage + exit "$SUCCESS" +fi + +# Parse the arguments +while getopts "V:qdvh" opt; do + case "$opt" in + V) virtualenv="$OPTARG";; + q) quiet=1;; + d) debug=1;; + v) version + exit "$SUCCESS";; + h) usage + exit "$SUCCESS";; + *) usage + exit "$ERROR_ARGUMENTS";; + esac +done +i=$OPTIND +shift $(($i-1)) + + +if [[ ${#@} == 0 ]]; then + error "Must pass configuration directory." + usage + exit "$ERROR_ARGUMENTS" +fi + +# Verfiy positional arguments. +config_dir="$1" +if [[ ! -d $config_dir ]]; then + error "No such directory: $config_dir" + exit "$ERROR_ARGUMENTS" +fi + +# Make sure virtual environment exists. +if [[ -n $virtualenv && ! -f $virtualenv/bin/activate ]]; then + error "Invalid virtual environment specified: $virtualenv" + exit "$ERROR_ARGUMENTS" +fi + +# Activate the virtual environment if it was specified. +[[ -n $virtualenv ]] && source "$virtualenv/bin/activate" + +# Verify pip-compile is available. +if ! which pip-compile >/dev/null 2>&1; then + error "Could not find command pip-compile from packagep pip-tools. Package might not be available in the virtual environment." + exit "$ERROR_PIP_TOOLS_MISSING" +fi + +# Create temporary files where files where "normalised" outputs will be +# stored. +tmp_current=$(mktemp) +tmp_new=$(mktemp) + +# Process each environment. +for environment in "$config_dir"/*; do + # Empty directory. + if [[ ! -e $environment ]]; then + error "Configuration directory is empty: $config_dir" + exit "$ERROR_CONFIG_DIR" + fi + + # Process each .in file. + for req_in in "$environment"/*.in; do + # Directory without .in files. + if [[ ! -e $req_in ]]; then + error "No .in files in directory, skipping: $environment" + continue + fi + + # Figure out if .txt file exists. + req_txt="${req_in%.in}.txt" + if [[ ! -f $req_txt ]]; then + warning "Missing .txt file for: $req_in" + continue + fi + + # Deploy the existing requirements file and the new one. + sed -e 's/[[:blank:]]*#.*//' "$req_txt" | grep -v "^$" | sort -u > "$tmp_current" + if ! pip-compile --no-header --no-annotate --no-index --output-file "$tmp_new" "$req_in" > /dev/null; then + error "Failed while running pip-compile command against (see error stack trace above): $req_in" + continue + fi + + # Run diff, storing the output and result. + diff=$(diff -u "$tmp_current" "$tmp_new") + result="$?" + + # Show warning about available updates. + if [[ $result == 0 ]]; then + [[ $quiet == 0 ]] && info "No upgrades available for: $req_txt" + else + warning "Upgrades available for: $req_txt" + echo + echo "$diff" + echo + fi + done +done diff --git a/roles/common/files/pipreqcheck_requirements.in b/roles/common/files/pipreqcheck_requirements.in new file mode 100644 index 0000000000000000000000000000000000000000..2c362e602b6b13e495e9b08fc2afbc41b9e9fc84 --- /dev/null +++ b/roles/common/files/pipreqcheck_requirements.in @@ -0,0 +1 @@ +pip-tools \ No newline at end of file diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 17ca82f4befb2cdf6cda16dab31403394e035e26..84802926801cb72f1d7811151ec952855d0722b7 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -149,6 +149,65 @@ - name: Install apticron (for checking available upgrades) apt: name=apticron state=installed +# Implementation for checking pip requirements files via via pip-tools. +- name: Install virtualenv for pip requirements checks + apt: name=virtualenv state=installed + +- name: Create dedicated group for user running pip requirements checks + group: name="pipreqcheck" gid="{{ pipreqcheck_gid | default(omit) }}" state=present + +- name: Create user for running pip requirements checks + user: name="pipreqcheck" uid="{{ pipreqcheck_uid | default(omit) }}" group="pipreqcheck" + home="/var/lib/pipreqcheck" state=present + +- name: Create directory for Python virtual environment used for installing/running pip-tools + file: path="/var/lib/pipreqcheck/virtualenv" state=directory + owner="pipreqcheck" group="pipreqcheck" mode="0750" + +- name: Create Python virtual environment used for installing/running pip-tools + become_user: "pipreqcheck" + command: /usr/bin/virtualenv --prompt "(pipreqcheck)" "/var/lib/pipreqcheck/virtualenv" creates="/var/lib/pipreqcheck/virtualenv/bin/activate" + +- name: Create directory for storing pip requirements files + file: path="/etc/pip_check_requirements_upgrades" state="directory" + owner="root" group="pipreqcheck" mode=750 + +- name: Set-up directory for storing pip requirements file for pip-tools virtual environment itself + file: path="/etc/pip_check_requirements_upgrades/pipreqcheck" state="directory" + owner="root" group="pipreqcheck" mode=750 + +- name: Deploy .in file for pip requirements in pip-tools virtual environment + copy: src="pipreqcheck_requirements.in" dest="/etc/pip_check_requirements_upgrades/pipreqcheck/requirements.in" + owner="root" group="pipreqcheck" mode=640 + +- name: Deploy requirements file for pipreqcheck virtual environment + template: src="pipreqcheck_requirements.txt.j2" dest="/etc/pip_check_requirements_upgrades/pipreqcheck/requirements.txt" + owner="root" group="pipreqcheck" mode=640 + +- name: Install latest pip in pip-tools virtual environment + become_user: "pipreqcheck" + pip: name=pip state=latest virtualenv="~pipreqcheck/virtualenv" + +- name: Install pip-tools if not present + become_user: "pipreqcheck" + pip: name=pip-tools state=present virtualenv="~pipreqcheck/virtualenv" + +- name: Synchronise pip-tools virtual environment via deployed requirements file + become_user: "pipreqcheck" + shell: "source ~pipreqcheck/virtualenv/bin/activate && pip-sync /etc/pip_check_requirements_upgrades/pipreqcheck/requirements.txt" + args: + executable: /bin/bash + register: pipreqcheck_pip_sync + changed_when: "pipreqcheck_pip_sync.stdout != 'Everything up-to-date'" + +- name: Deploy script for checking available upgrades + copy: src="pip_check_requirements_upgrades.sh" dest="/usr/local/bin/pip_check_requirements_upgrades.sh" + owner=root group=root mode=755 + +- name: Deploy crontab entry for checking pip requirements + cron: name="check_pip_requirements" cron_file="check_pip_requirements" hour=0 minute=0 job="/usr/local/bin/pip_check_requirements_upgrades.sh /etc/pip_check_requirements_upgrades" + state=present user=pipreqcheck + - name: Explicitly run all handlers include: ../handlers/main.yml when: "handlers | default(False) | bool() == True" diff --git a/roles/common/templates/pipreqcheck_requirements.txt.j2 b/roles/common/templates/pipreqcheck_requirements.txt.j2 new file mode 100644 index 0000000000000000000000000000000000000000..3ab42c6c01ed9b5480eaa48384498c5121d4e921 --- /dev/null +++ b/roles/common/templates/pipreqcheck_requirements.txt.j2 @@ -0,0 +1,3 @@ +{% for requirement in pip_check_requirements %} +{{ requirement }} +{% endfor %}