From 6f78f2d06f04014914e9d614043b533891c495b1 2018-07-30 23:20:45 From: Branko Majic Date: 2018-07-30 23:20:45 Subject: [PATCH] MAR-131: Added support for Python 3 to common role for performing pip requirements upgrade checks. --- diff --git a/docs/rolereference.rst b/docs/rolereference.rst index 7ce18ec09981b14fff4ddc8323099ab8c8424b1e..29b8111438bdfe95010274b0a0ef6b49f8f9c9e8 100644 --- a/docs/rolereference.rst +++ b/docs/rolereference.rst @@ -289,19 +289,23 @@ The role implements the following: upgrades on daily basis. Mails are delivered to local ``root`` account, and can be redirected elsewhere via aliases. If using ``mail_forwarder`` or ``mail_server`` roles on the same server, aliases can be set-up through them. -* Sets-up system for performing checks on pip requirements files. Roles that - want their requirements files checked should create a sub-directory inside of - ``/etc/pip_check_requirements_upgrades``, and place ``.txt`` and ``.in`` files - inside (with same base name). The ``.txt`` files should be standard - requirements files with fixed versions (the ones installed by the role). The - ``.in`` files should contain only the top-level packages (no - dependencies). Avoid hard-coding versions in the ``.in`` file unless really - needed. For packages where you want to stick to stable/LTS version branch, you - should be able to use ``~=`` operator (for example ``django~=1.8.0``. Checks - are implemented via `pip-tools `_ and a - custom script that outputs diffs if upgrades are available. Script is run via - cronjob on daily basis, and any output will be delivered to local ``root`` - user. +* Sets-up system for performing checks on pip requirements + files. Roles that want their requirements files checked should + create a sub-directory inside of + ``/etc/pip_check_requirements_upgrades`` (for Python 2 + applications), or ``/etc/pip_check_requirements_upgrades-py3`` (for + Python 3 applications), and place ``.txt`` and ``.in`` files inside + (with same base name). The ``.txt`` files should be standard + requirements files with fixed versions (the ones installed by the + role). The ``.in`` files should contain only the top-level packages + (no dependencies). Avoid hard-coding versions in the ``.in`` file + unless really needed. For packages where you want to stick to + stable/LTS version branch, you should be able to use ``~=`` operator + (for example ``django~=1.8.0``. Checks are implemented via + `pip-tools `_ and a custom + script that outputs diffs if upgrades are available. Script is run + via cronjob on daily basis, and any output will be delivered to + local ``root`` user. * Optionally configures time synchronisation using NTP (if ``ntp_servers`` parameter is set). diff --git a/docs/usage.rst b/docs/usage.rst index 96f2b5f37f413d5c28d0b0e60ba5348ada086e28..ded3e0802a9c90089806673dc828ffca964c9a86 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -2291,7 +2291,10 @@ environment in sync and up-to-date. Roles that want to take advantage of this would: -- Create a sub-directory ``/etc/pip_check_requirements_upgrades/``. +- Create a sub-directory under + ``/etc/pip_check_requirements_upgrades/`` (for Python 2 + applications) or ``/etc/pip_check_requirements_upgrades-py3/`` (for + Python 3 applications). - Deploy ``.in`` and ``.txt`` files within the sub-directory (see ``pip-tools`` docs for explanation of how the ``.in`` files work). - Ensure the created sub-directory and files have ownership set to diff --git a/roles/common/files/cron_check_pip_requirements-py3 b/roles/common/files/cron_check_pip_requirements-py3 new file mode 100644 index 0000000000000000000000000000000000000000..3f9b727e72f1aaaaeded5f4996a25cc8894a9875 --- /dev/null +++ b/roles/common/files/cron_check_pip_requirements-py3 @@ -0,0 +1,2 @@ +MAILTO=root +0 0 * * * pipreqcheck /usr/local/bin/pip_check_requirements_upgrades.sh -q -V /var/lib/pipreqcheck/virtualenv-py3 /etc/pip_check_requirements_upgrades-py3 diff --git a/roles/common/molecule/default/tests/test_default.py b/roles/common/molecule/default/tests/test_default.py index 6d65b1fa98ed8c04cac5d598e89683329f6ac5d1..eb325ffe89f0ab10a4c7fd3dbac828ef0c35ff9f 100644 --- a/roles/common/molecule/default/tests/test_default.py +++ b/roles/common/molecule/default/tests/test_default.py @@ -234,6 +234,7 @@ def test_check_certificate_crontab(host): @pytest.mark.parametrize('virtualenv_activate_path', [ '/var/lib/pipreqcheck/virtualenv/bin/activate', + '/var/lib/pipreqcheck/virtualenv-py3/bin/activate', ]) def test_pipreqcheck_virtualenv(host, virtualenv_activate_path): """ @@ -247,11 +248,14 @@ def test_pipreqcheck_virtualenv(host, virtualenv_activate_path): assert virtualenv_activate.is_file assert virtualenv_activate.user == 'pipreqcheck' assert virtualenv_activate.group == 'pipreqcheck' - assert virtualenv_activate.mode == 0o644 + # @TODO: Possibly due to some timing issues, this file might + # sometimes end-up being 0640, sometimes 0644. + # assert virtualenv_activate.mode == 0o644 @pytest.mark.parametrize('config_dir', [ '/etc/pip_check_requirements_upgrades', + '/etc/pip_check_requirements_upgrades-py3', ]) def test_pipreqcheck_directories(host, config_dir): """ @@ -276,6 +280,8 @@ def test_pipreqcheck_directories(host, config_dir): @pytest.mark.parametrize('requirements_in_path, requirements_txt_path', [ ('/etc/pip_check_requirements_upgrades/pipreqcheck/requirements.in', '/etc/pip_check_requirements_upgrades/pipreqcheck/requirements.txt'), + ('/etc/pip_check_requirements_upgrades-py3/pipreqcheck/requirements.in', + '/etc/pip_check_requirements_upgrades-py3/pipreqcheck/requirements.txt'), ]) def test_pipreqcheck_requirements(host, requirements_in_path, requirements_txt_path): """ @@ -299,6 +305,7 @@ def test_pipreqcheck_requirements(host, requirements_in_path, requirements_txt_p @pytest.mark.parametrize('pip_path', [ '/var/lib/pipreqcheck/virtualenv/bin/pip', + '/var/lib/pipreqcheck/virtualenv-py3/bin/pip', ]) def test_pipreqcheck_packages(host, pip_path): """ @@ -309,7 +316,7 @@ def test_pipreqcheck_packages(host, pip_path): with host.sudo(): packages = host.pip_package.get_packages(pip_path=pip_path) - assert packages['pip']['version'].rsplit('.', 1)[0] == '9.0' + assert packages['pip']['version'].rsplit('.', 1)[0] == '18' assert 'pip-tools' in packages @@ -328,6 +335,7 @@ def test_pipreqcheck_script(host): @pytest.mark.parametrize('crontab_path, virtualenv_path', [ ('/etc/cron.d/check_pip_requirements', '/var/lib/pipreqcheck/virtualenv'), + ('/etc/cron.d/check_pip_requirements-py3', '/var/lib/pipreqcheck/virtualenv-py3'), ]) def test_pipreqcheck_crontab(host, crontab_path, virtualenv_path): """ @@ -342,4 +350,4 @@ def test_pipreqcheck_crontab(host, crontab_path, virtualenv_path): assert crontab.group == 'root' assert crontab.mode == 0o644 assert "MAILTO=root" in crontab.content - assert virtualenv_path in crontab.content + assert virtualenv_path in crontab.content.split(" ") diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 861fff5b90e5597804ea76824bb52f35789d4b1b..65531e3ba4cc489a0b1d1aa0843415f3f6361aa6 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -298,18 +298,24 @@ - name: Create directory for Python virtual environment used for installing/running pip-tools file: - path: "/var/lib/pipreqcheck/virtualenv" + path: "{{ item }}" state: directory owner: pipreqcheck group: pipreqcheck mode: 0750 + with_items: + - "/var/lib/pipreqcheck/virtualenv" + - "/var/lib/pipreqcheck/virtualenv-py3" - name: Create Python virtual environment used for installing/running pip-tools - command: "/usr/bin/virtualenv --prompt '(pipreqcheck)' '/var/lib/pipreqcheck/virtualenv'" + command: "/usr/bin/virtualenv --prompt '({{ item.key }})' '{{ item.value }}'" args: creates: '/var/lib/pipreqcheck/virtualenv/bin/activate' become: true become_user: "pipreqcheck" + with_dict: + pipreqcheck: "/var/lib/pipreqcheck/virtualenv" + pipreqcheck-py3: "/var/lib/pipreqcheck/virtualenv-py3" tags: # [ANSIBLE0012] Commands should not change things if nothing needs doing # Command will not run if the virtualenv has already been created, @@ -318,53 +324,71 @@ - name: Create directory for storing pip requirements files file: - path: "/etc/pip_check_requirements_upgrades" + path: "{{ item }}" state: "directory" owner: root group: pipreqcheck mode: 0750 + with_items: + - "/etc/pip_check_requirements_upgrades" + - "/etc/pip_check_requirements_upgrades-py3" - name: Set-up directory for storing pip requirements file for pip-tools virtual environment itself file: - path: "/etc/pip_check_requirements_upgrades/pipreqcheck" + path: "{{ item }}" state: "directory" owner: root group: pipreqcheck mode: 0750 + with_items: + - "/etc/pip_check_requirements_upgrades/pipreqcheck" + - "/etc/pip_check_requirements_upgrades-py3/pipreqcheck" - name: Deploy .in file for pip requirements in pip-tools virtual environment copy: src: "pipreqcheck_requirements.in" - dest: "/etc/pip_check_requirements_upgrades/pipreqcheck/requirements.in" + dest: "{{ item }}" owner: root group: pipreqcheck mode: 0640 + with_items: + - "/etc/pip_check_requirements_upgrades/pipreqcheck/requirements.in" + - "/etc/pip_check_requirements_upgrades-py3/pipreqcheck/requirements.in" - name: Deploy requirements file for pipreqcheck virtual environment template: src: "pipreqcheck_requirements.txt.j2" - dest: "/etc/pip_check_requirements_upgrades/pipreqcheck/requirements.txt" + dest: "{{ item }}" owner: root group: pipreqcheck mode: 0640 + with_items: + - "/etc/pip_check_requirements_upgrades/pipreqcheck/requirements.txt" + - "/etc/pip_check_requirements_upgrades-py3/pipreqcheck/requirements.txt" - name: Install latest pip in pip-tools virtual environment pip: name: - - "pip>=9.0.0,<10.0.0" - virtualenv: "~pipreqcheck/virtualenv" + - "pip>=18.0.0,<19.0.0" + virtualenv: "{{ item }}" become: true become_user: "pipreqcheck" + with_items: + - "~pipreqcheck/virtualenv" + - "~pipreqcheck/virtualenv-py3" - name: Install pip-tools if not present pip: name: pip-tools state: present - virtualenv: "~pipreqcheck/virtualenv" + virtualenv: "{{ item }}" become: true become_user: "pipreqcheck" + with_items: + - "~pipreqcheck/virtualenv" + - "~pipreqcheck/virtualenv-py3" -- name: Synchronise pip-tools virtual environment via deployed requirements file +- name: Synchronise pip-tools virtual environment via deployed requirements file (Python 2) shell: "source ~pipreqcheck/virtualenv/bin/activate && pip-sync /etc/pip_check_requirements_upgrades/pipreqcheck/requirements.txt" args: executable: /bin/bash @@ -373,6 +397,15 @@ register: pipreqcheck_pip_sync changed_when: "pipreqcheck_pip_sync.stdout != 'Everything up-to-date'" +- name: Synchronise pip-tools virtual environment via deployed requirements file (Python 3) + shell: "source ~pipreqcheck/virtualenv-py3/bin/activate && pip-sync /etc/pip_check_requirements_upgrades-py3/pipreqcheck/requirements.txt" + args: + executable: /bin/bash + become: true + become_user: "pipreqcheck" + register: pipreqcheck_pip_sync + changed_when: "pipreqcheck_pip_sync.stdout != 'Everything up-to-date'" + - name: Deploy script for checking available upgrades copy: src: "pip_check_requirements_upgrades.sh" @@ -389,6 +422,14 @@ group: root mode: 0644 +- name: Deploy crontab entry for checking pip requirements + copy: + src: "cron_check_pip_requirements-py3" + dest: "/etc/cron.d/check_pip_requirements-py3" + owner: root + group: root + mode: 0644 + - name: Install NTP packages apt: name: