diff --git a/docs/releasenotes.rst b/docs/releasenotes.rst index 2a6d5a72ff8d72d8d63f6c1df66482d5db94f2e3..bd99173029c73755ef6338700ab721f28eec2918 100644 --- a/docs/releasenotes.rst +++ b/docs/releasenotes.rst @@ -20,6 +20,17 @@ run applications using Debian-only repositories. **New features/improvements** +* ``common`` role + + * Added parameters ``pip_check_requirements_in`` and + ``pip_check_requirements_py3_in`` that can be used for specifying + input requirements when checking for available package upgrades + for Python virtual environments that are used for the checks + themselves. This is particularly helpful in cases where Python + version gets deprecated and some packages do not correctly declare + the minimum version required, allowing to be more specific to + avoid unnecessary warning mails being sent out. + * ``xmpp_server`` role * Drop dependency on the external (Prosody) package diff --git a/docs/rolereference.rst b/docs/rolereference.rst index c9df994ed41c4032b17dbac032bb036569de2490..11db8e3aaaba9d947191c3b2a359ef4502e48f2f 100644 --- a/docs/rolereference.rst +++ b/docs/rolereference.rst @@ -416,13 +416,25 @@ Parameters - "2.debian.pool.ntp.org" - "3.debian.pool.ntp.org" +**pip_check_requirements_in** (list, optional, ``[pip, pip-tools, setuptools, wheel]``) + List of Python package requirements inputs to use for checking for + package upgrades for the Python 2 virtual environment used to run + the check itself. For Python 3, see the dedicated parameter + ``pip_check_requirements_py3`` below. + **pip_check_requirements** (list, optional, ``[click==7.0, pip-tools==4.0.0, pip==19.2.3, setuptools==41.2.0, six==1.12.0, wheel==0.33.6]``) List of Python package requirements to install in Python 2 virtual environment in order to be able to run the ``pip-tools`` applications as part of pip requirements upgrade checks. This list needs to be updated from time to time as the new releases of ``pip-tools`` and related packages are coming out. For Python 3, see - the dedicatd parameter ``pip_check_requirements_py3`` below. + the dedicated parameter ``pip_check_requirements_py3`` below. + +**pip_check_requirements_py3_in** (list, optional, ``[pip, pip-tools, setuptools, wheel]``) + List of Python package requirements inputs to use for checking for + package upgrades for the Python 3 virtual environment used to run + the check itself. For Python 2, see the dedicated parameter + ``pip_check_requirements`` above. **pip_check_requirements_py3** (list, optional, ``[click==7.0, pip-tools==3.9.0, pip==19.1.1, setuptools==41.2.0, six==1.12.0, wheel==0.33.6]``) List of Python package requirements to install in Python 3 virtual @@ -430,7 +442,7 @@ Parameters applications as part of pip requirements upgrade checks. This list needs to be updated from time to time as the new releases of ``pip-tools`` and related packages are coming out. For Python 2, see - the dedicatd parameter ``pip_check_requirements`` above. + the dedicated parameter ``pip_check_requirements`` above. **pipreqcheck_uid** (integer, optional, ``whatever OS picks``) UID for user running the pip requirements upgrade checks. User is created with diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index 4e06b099918a64640150f0479bf1454644cf1e96..b600d48294c0cc7d46bafb02cd5afd1bd30ed8f9 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -12,6 +12,11 @@ prompt_id: null extra_backup_patterns: - "/root" - "/home" +pip_check_requirements_in: + - pip + - pip-tools + - setuptools + - wheel pip_check_requirements: - click==7.1.2 - pip-tools==5.3.1 @@ -19,6 +24,11 @@ pip_check_requirements: - setuptools==44.1.1 - six==1.15.0 - wheel==0.35.1 +pip_check_requirements_py3_in: + - pip + - pip-tools + - setuptools + - wheel pip_check_requirements_py3: - click==7.1.2 - pip-tools==5.3.1 diff --git a/roles/common/files/pipreqcheck_requirements.in b/roles/common/files/pipreqcheck_requirements.in deleted file mode 100644 index 29fbfc8bab205d660a5e8578723812283f602c09..0000000000000000000000000000000000000000 --- a/roles/common/files/pipreqcheck_requirements.in +++ /dev/null @@ -1,4 +0,0 @@ -pip -pip-tools -setuptools -wheel \ No newline at end of file diff --git a/roles/common/molecule/default/group_vars/parameters-optional.yml b/roles/common/molecule/default/group_vars/parameters-optional.yml index 6a2cf61badca800a9b8a1764299249ef722c1e9a..347d85e99d96e0c386197303afd709822b25b834 100644 --- a/roles/common/molecule/default/group_vars/parameters-optional.yml +++ b/roles/common/molecule/default/group_vars/parameters-optional.yml @@ -53,6 +53,16 @@ ntp_servers: maintenance: true maintenance_allowed_hosts: - client1 +pip_check_requirements_in: + - pip >= 0.2.1 + - pip-tools >= 0.2.2 + - setuptools >= 0.2.3 + - wheel >= 0.2.4 +pip_check_requirements_py3_in: + - pip >= 0.3.1 + - pip-tools >= 0.3.2 + - setuptools >= 0.3.3 + - wheel >= 0.3.4 # From backup_client role meta dependency. backup_encryption_key: "{{ lookup('file', 'tests/data/gnupg/backup_encryption_key') }}" diff --git a/roles/common/molecule/default/tests/test_parameters_mandatory.py b/roles/common/molecule/default/tests/test_parameters_mandatory.py index f115220caffee25323e7d523967874a272787a3e..b3af54e64edfc0457db21ea78a1dfb7980aacbc6 100644 --- a/roles/common/molecule/default/tests/test_parameters_mandatory.py +++ b/roles/common/molecule/default/tests/test_parameters_mandatory.py @@ -5,6 +5,8 @@ import paramiko import testinfra.utils.ansible_runner +import pytest + testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('parameters-mandatory') @@ -131,3 +133,33 @@ def test_ntp_listening_interfaces(host): """ assert not host.socket('udp://:::123').is_listening + + +@pytest.mark.parametrize("requirements_path, expected_requirements", [ + ('/etc/pip_check_requirements_upgrades/pipreqcheck/requirements.in', [ + "pip", + "pip-tools", + "setuptools", + "wheel" + ]), + ('/etc/pip_check_requirements_upgrades-py3/pipreqcheck/requirements.in', [ + "pip", + "pip-tools", + "setuptools", + "wheel" + ]) +]) +def test_pipreqcheck_input_content(host, requirements_path, expected_requirements): + """ + Tests content of requirements input file used for virtual + environment utilised by script that performs pip requirements + upgrade checks. + """ + + with host.sudo(): + deployed_requirements = host.file(requirements_path).content_string + + expected_requirements = sorted([line.lower() for line in expected_requirements]) + actual_requirements = sorted(deployed_requirements.lower().strip().split("\n")) + + assert actual_requirements == expected_requirements diff --git a/roles/common/molecule/default/tests/test_parameters_optional.py b/roles/common/molecule/default/tests/test_parameters_optional.py index 7672bb146b6cc33074a25408b970f994a6e9952f..9b3c3bc7880009fd9c644580e7aa11c8806be117 100644 --- a/roles/common/molecule/default/tests/test_parameters_optional.py +++ b/roles/common/molecule/default/tests/test_parameters_optional.py @@ -323,3 +323,33 @@ def test_ntp_listening_interfaces(host): """ assert host.socket('udp://:::123').is_listening + + +@pytest.mark.parametrize("requirements_path, expected_requirements", [ + ('/etc/pip_check_requirements_upgrades/pipreqcheck/requirements.in', [ + "pip >= 0.2.1", + "pip-tools >= 0.2.2", + "setuptools >= 0.2.3", + "wheel >= 0.2.4" + ]), + ('/etc/pip_check_requirements_upgrades-py3/pipreqcheck/requirements.in', [ + "pip >= 0.3.1", + "pip-tools >= 0.3.2", + "setuptools >= 0.3.3", + "wheel >= 0.3.4" + ]) +]) +def test_pipreqcheck_input_content(host, requirements_path, expected_requirements): + """ + Tests content of requirements input file used for virtual + environment utilised by script that performs pip requirements + upgrade checks. + """ + + with host.sudo(): + deployed_requirements = host.file(requirements_path).content_string + + expected_requirements = sorted([line.lower() for line in expected_requirements]) + actual_requirements = sorted(deployed_requirements.lower().strip().split("\n")) + + assert actual_requirements == expected_requirements diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 8fba3efb32bad70acf9ee69b59fce4bc489069a5..8aeed417f2087920bd0b7a044bf0387947860bb5 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -353,15 +353,17 @@ - "/etc/pip_check_requirements_upgrades-py3/pipreqcheck" - name: Deploy .in file for pip requirements in pip-tools virtual environment - copy: - src: "pipreqcheck_requirements.in" - dest: "{{ item }}" + template: + src: "pipreqcheck_requirements.in.j2" + dest: "{{ item.path }}" owner: root group: pipreqcheck mode: 0640 with_items: - - "/etc/pip_check_requirements_upgrades/pipreqcheck/requirements.in" - - "/etc/pip_check_requirements_upgrades-py3/pipreqcheck/requirements.in" + - path: "/etc/pip_check_requirements_upgrades/pipreqcheck/requirements.in" + requirements: "{{ pip_check_requirements_in }}" + - path: "/etc/pip_check_requirements_upgrades-py3/pipreqcheck/requirements.in" + requirements: "{{ pip_check_requirements_py3_in }}" - name: Deploy requirements file for pipreqcheck virtual environment template: diff --git a/roles/common/templates/pipreqcheck_requirements.in.j2 b/roles/common/templates/pipreqcheck_requirements.in.j2 new file mode 100644 index 0000000000000000000000000000000000000000..85e491c7b4c392f90ec14b97df10a6c0f7f88ed2 --- /dev/null +++ b/roles/common/templates/pipreqcheck_requirements.in.j2 @@ -0,0 +1,3 @@ +{% for requirement in item.requirements %} +{{ requirement }} +{% endfor %}