From 380556b776cd22261cb0e336adc63be0a82af42b 2018-12-10 20:58:52 From: Branko Majic Date: 2018-12-10 20:58:52 Subject: [PATCH] MAR-141: Fix handling of pipreqcheck virtualenv package installation: - Deploy all requirements through requirements file, even if using pip. Improves reproducability of installed packages and solves some double-versioning issues for packages like pip etc. - Updated test for validating installed packages. --- diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index 2a368479875c89dc631a01be84421b01c30c2410..2904099a56d1bfaa80466ada0b1a80a34248e3ad 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -13,10 +13,12 @@ extra_backup_patterns: - "/root" - "/home" pip_check_requirements: - - click==6.7 - - first==2.0.1 - - pip-tools==2.0.2 + - click==7.0 + - pip-tools==3.1.0 + - pip==18.1 + - setuptools==40.6.2 - six==1.11.0 + - wheel==0.32.3 ntp_servers: [] # Internal use only. diff --git a/roles/common/files/pip_check_requirements_upgrades.sh b/roles/common/files/pip_check_requirements_upgrades.sh index c87a317a60c292d7a546bed0c7f39830756a9316..c2504df1f4026ddd7520eacf5ebecaedd5779c02 100755 --- a/roles/common/files/pip_check_requirements_upgrades.sh +++ b/roles/common/files/pip_check_requirements_upgrades.sh @@ -254,7 +254,7 @@ for environment in "$config_dir"/*; do # Deploy the existing requirements file and the new one. sed -e 's/[[:blank:]]*#.*//' "$req_txt" | grep -v "^$" | sort -u > "$tmp_current" - if ! pip-compile --no-header --no-annotate --no-index --output-file "$tmp_new" --upgrade "$req_in" > /dev/null; then + if ! pip-compile --allow-unsafe --no-header --no-annotate --no-index --output-file "$tmp_new" --upgrade "$req_in" > /dev/null; then error "Failed while running pip-compile command against (see error stack trace above): $req_in" continue fi diff --git a/roles/common/files/pipreqcheck_requirements.in b/roles/common/files/pipreqcheck_requirements.in index 2c362e602b6b13e495e9b08fc2afbc41b9e9fc84..d42936a4179da7e2829f20254835a126513c858c 100644 --- a/roles/common/files/pipreqcheck_requirements.in +++ b/roles/common/files/pipreqcheck_requirements.in @@ -1 +1,3 @@ -pip-tools \ No newline at end of file +pip +pip-tools +setuptools \ No newline at end of file diff --git a/roles/common/molecule/default/tests/test_default.py b/roles/common/molecule/default/tests/test_default.py index e23f5d53bc568d4d7b0009afea35c30593583893..6cc70df8d3f9b2a5c84c76a7402027c78b42a13e 100644 --- a/roles/common/molecule/default/tests/test_default.py +++ b/roles/common/molecule/default/tests/test_default.py @@ -305,21 +305,41 @@ def test_pipreqcheck_requirements(host, requirements_in_path, requirements_txt_p assert requirements_txt.mode == 0o640 -@pytest.mark.parametrize('pip_path', [ - '/var/lib/pipreqcheck/virtualenv/bin/pip', - '/var/lib/pipreqcheck/virtualenv-py3/bin/pip', +@pytest.mark.parametrize("pip_path, expected_packages", [ + ('/var/lib/pipreqcheck/virtualenv/bin/pip', [ + "Click==7.0", + "pip==18.1", + "pip-tools==3.1.0", + "setuptools==40.6.2", + "six==1.11.0", + "wheel==0.32.3" + ]), + ('/var/lib/pipreqcheck/virtualenv-py3/bin/pip', [ + "Click==7.0", + "pip==18.1", + "pip-tools==3.1.0", + "setuptools==40.6.2", + "six==1.11.0", + "wheel==0.32.3", + ]), ]) -def test_pipreqcheck_packages(host, pip_path): +def test_pipreqcheck_virtualenv_packages(host, pip_path, expected_packages): """ - Tests if Python virtual environment used for running the pip requirements - upgrade checks has correct version of pip installed. + Tests if correct packages are installed in virtualenv used for pip + requirements checks.. """ - with host.sudo(): - packages = host.pip_package.get_packages(pip_path=pip_path) + packages = host.run("sudo -u %s %s freeze --all" % ('pipreqcheck', pip_path)) + + # Normalise package names and order. + expected_packages = sorted([unicode(p.lower()) for p in expected_packages]) + actual_packages = sorted(packages.stdout.lower().split("\n")) + + # This is a dummy distro-provided package ignored by the pip-tools. + if "pkg-resources==0.0.0" in actual_packages: + actual_packages.remove("pkg-resources==0.0.0") - assert packages['pip']['version'].rsplit('.', 1)[0] == '18' - assert 'pip-tools' in packages + assert actual_packages == expected_packages def test_pipreqcheck_script(host): diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 827e9db9849787954a352fdc9fb5073eb55f334d..f090213a56a08ee81b44e4c41d9513130f5e5c99 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -368,27 +368,17 @@ - "/etc/pip_check_requirements_upgrades/pipreqcheck/requirements.txt" - "/etc/pip_check_requirements_upgrades-py3/pipreqcheck/requirements.txt" -- name: Install latest pip in pip-tools virtual environment +- name: Install requirements in the pipreqcheck virtual environment pip: - name: - - "pip>=18.0.0,<19.0.0" - virtualenv: "{{ item }}" - become: true - become_user: "pipreqcheck" - with_items: - - "~pipreqcheck/virtualenv" - - "~pipreqcheck/virtualenv-py3" - -- name: Install pip-tools if not present - pip: - name: pip-tools>=2.0.2 - state: present - virtualenv: "{{ item }}" + requirements: "{{ item.requirements }}" + virtualenv: "{{ item.virtualenv }}" become: true - become_user: "pipreqcheck" + become_user: pipreqcheck with_items: - - "~pipreqcheck/virtualenv" - - "~pipreqcheck/virtualenv-py3" + - virtualenv: "~pipreqcheck/virtualenv" + requirements: "/etc/pip_check_requirements_upgrades/pipreqcheck/requirements.txt" + - virtualenv: "~pipreqcheck/virtualenv-py3" + requirements: "/etc/pip_check_requirements_upgrades-py3/pipreqcheck/requirements.txt" - name: Synchronise pip-tools virtual environment via deployed requirements file (Python 2) shell: "source ~pipreqcheck/virtualenv/bin/activate && pip-sync /etc/pip_check_requirements_upgrades/pipreqcheck/requirements.txt"