Changeset - 71bc6c4991f8
[Not reviewed]
0 3 0
Branko Majic (branko) - 7 years ago 2018-08-05 14:13:04
branko@majic.rs
MAR-132: Fix virtual environment set-up in wsgi_website role:

- Install more up-to-date version of pip. This fixes some issues
related to pip freeze detecting argparse and wsgiref in virtualenv
as separate packages.
- Remove the pkg-resources package from virtualenv. Workaround for
Debian-specific behaviour. More details at:
- https://github.com/pypa/pip/issues/4022
- https://bugs.launchpad.net/ubuntu/+source/python-pip/+bug/1635463
- https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=871790
- Deploy Gunicorn requirements file without switching to website admin
user (otherwise Ansible fails to template the file due to
permissions restrictions).
- Updated test for checking packages installed in the virtualenv.
3 files changed with 21 insertions and 6 deletions:
0 comments (0 inline, 0 general)
roles/wsgi_website/molecule/default/tests/test_default.py
Show inline comments
 
@@ -279,81 +279,77 @@ def test_python_virtualenv_project_directory_config(host, project_file, expected
 
@pytest.mark.parametrize("wrapper_script, expected_owner, expected_group", [
 
    ('/var/www/parameters-mandatory/virtualenv/bin/exec', 'admin-parameters-mandatory', 'web-parameters-mandatory'),
 
    ('/var/www/parameters-optional.local/virtualenv/bin/exec', 'admin-parameters-optional_local', 'web-parameters-optional_local'),
 
    ('/var/www/parameters-paste-req/virtualenv/bin/exec', 'admin-parameters-paste-req', 'web-parameters-paste-req'),
 
])
 
def test_python_virtualenv_wrapper_script(host, wrapper_script, expected_owner, expected_group):
 
    """
 
    Tests if Python virtualenv wrapper script is functioning correctly.
 
    """
 

	
 
    with host.sudo():
 

	
 
        wrapper = host.file(wrapper_script)
 
        assert wrapper.is_file
 
        assert wrapper.user == expected_owner
 
        assert wrapper.group == expected_group
 
        assert wrapper.mode == 0o750
 

	
 
        command = host.run("sudo -u %s %s python -c 'import gunicorn'" % (expected_owner, wrapper_script))
 
        assert command.rc == 0
 

	
 

	
 
@pytest.mark.parametrize("admin_user, pip_path, expected_packages",  [
 
    ('admin-parameters-mandatory', '/var/www/parameters-mandatory/virtualenv/bin/pip', [
 
        "argparse==1.2.1",
 
        "futures==3.2.0",
 
        "gunicorn==19.9.0",
 
        "wsgiref==0.1.2"
 
    ]),
 
    ('admin-parameters-optional_local', '/var/www/parameters-optional.local/virtualenv/bin/pip', [
 
        "Pygments==2.2.0",
 
        "dnspython==1.15.0",
 
        "docopt==0.6.2",
 
        "futures==3.1.0",
 
        "gunicorn==19.7.0",
 
        "jedi==0.12.1",
 
        "parso==0.3.1",
 
        "prompt-toolkit==1.0.15",
 
        "ptpython==0.41",
 
        "six==1.11.0",
 
        "wcwidth==0.1.7",
 
    ]),
 
    ('admin-parameters-paste-req', '/var/www/parameters-paste-req/virtualenv/bin/pip', [
 
        "Flask==0.12.2",
 
        "Jinja2==2.9.6",
 
        "MarkupSafe==1.0",
 
        "Paste==2.0.3",
 
        "PasteDeploy==1.5.2",
 
        "PasteScript==2.0.2",
 
        "Werkzeug==0.12.2",
 
        "argparse==1.2.1",
 
        "click==6.7",
 
        "futures==3.1.0",
 
        "gunicorn==19.7.0",
 
        "itsdangerous==0.24",
 
        "six==1.10.0",
 
        "wsgiref==0.1.2",
 
    ]),
 
])
 
def test_virtualenv_packages(host, admin_user, pip_path, expected_packages):
 
    """
 
    Tests if correct packages are installed in virtualenv.
 
    """
 

	
 
    packages = host.run("sudo -u %s %s freeze" % (admin_user, pip_path))
 

	
 
    # Normalise package names and order.
 
    expected_packages = sorted([p.lower() for p in expected_packages])
 
    actual_packages = sorted(packages.stdout.lower().split("\n"))
 

	
 
    assert actual_packages == expected_packages
 

	
 

	
 
@pytest.mark.parametrize('check_config_dir', [
 
    '/etc/pip_check_requirements_upgrades/parameters-mandatory',
 
    '/etc/pip_check_requirements_upgrades/parameters-optional.local',
 
])
 
def test_wsgi_requirements_upgrade_checks(host, check_config_dir):
 
    """
 
    Tests if Python requirements files for upgrade checks are set-up
 
    correctly (absent when not configured).
roles/wsgi_website/tasks/main.yml
Show inline comments
 
@@ -101,48 +101,69 @@
 
  when: "current_python_version.rc == 0 and not current_python_version.stdout.startswith(python_version | string)"
 
  notify:
 
    - Restart WSGI services
 

	
 
- name: Create directory for storing the Python virtual environment
 
  file:
 
    path: "{{ home }}/virtualenv"
 
    state: directory
 
    owner: "{{ admin }}"
 
    group: "{{ user }}"
 
    mode: 02750
 

	
 
- name: Create Python virtual environment
 
  command: '/usr/bin/virtualenv --python "{{ python_interpreter }}" --prompt "({{ fqdn }})" "{{ home }}/virtualenv"'
 
  args:
 
    creates: "{{ home }}/virtualenv/bin/{{ python_interpreter | basename }}"
 
  become: true
 
  become_user: "{{ admin }}"
 
  tags:
 
    # [ANSIBLE0012] Commands should not change things if nothing needs doing
 
    #   This task will not fire if the virtual environment has already bene
 
    #   created (thanks to 'creates' parameter).
 
    - skip_ansible_lint
 

	
 
- name: Install latest pip in virtual environment
 
  pip:
 
    name:
 
      - "pip>=18.0.0,<19.0.0"
 
    virtualenv: "{{ home }}/virtualenv"
 
  become: true
 
  become_user: "{{ admin }}"
 

	
 
# Workaround for:
 
#
 
# - https://github.com/pypa/pip/issues/4022
 
# - https://bugs.launchpad.net/ubuntu/+source/python-pip/+bug/1635463
 
# - https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=871790
 
- name: Remove the pkg-resources package from virtual environment (see comments above for details)
 
  pip:
 
    name: pkg-resources
 
    virtualenv: "{{ home }}/virtualenv"
 
    state: absent
 
  become: true
 
  become_user: "{{ admin }}"
 

	
 
- name: Configure project directory for the Python virtual environment
 
  template:
 
    src: "venv_project.j2"
 
    dest: "{{ home }}/virtualenv/.project"
 
    owner: "{{ admin }}"
 
    group: "{{ user }}"
 
    mode: 0640
 

	
 
- name: Deploy virtualenv wrapper
 
  template:
 
    src: "venv_exec.j2"
 
    dest: "{{ home }}/virtualenv/bin/exec"
 
    owner: "{{ admin }}"
 
    group: "{{ user }}"
 
    mode: 0750
 

	
 
- name: Install WSGI server
 
  become: true
 
  become_user: "{{ admin }}"
 
  pip:
 
    name: "{{ item.package }}"
 
    version: "{{ item.version }}"
 
    state: present
 
    virtualenv: "{{ home }}/virtualenv"
roles/wsgi_website/tasks/requirements.yml
Show inline comments
 
---
 

	
 
- name: Set-up directory for storing requirements file for upgrade checks
 
  file:
 
    path: "/etc/pip_check_requirements_upgrades/{{ fqdn }}"
 
    state: directory
 
    owner: root
 
    group: pipreqcheck
 
    mode: 0750
 

	
 
- name: Deploy WSGI requirements files for upgrade checks
 
  template:
 
    src: "{{ item }}.j2"
 
    dest: "/etc/pip_check_requirements_upgrades/{{ fqdn }}/{{ item }}"
 
    owner: root
 
    group: pipreqcheck
 
    mode: 0640
 
  with_items:
 
    - wsgi_requirements.in
 
    - wsgi_requirements.txt
 

	
 
- name: Deploy Gunicorn requirements file for installation purposes
 
  become: true
 
  become_user: "{{ admin }}"
 
  template:
 
    src: "wsgi_requirements.txt.j2"
 
    dest: "{{ home }}/.wsgi_requirements.txt"
 
    owner: "{{ admin }}"
 
    group: "{{ user }}"
 
    mode: 0640
 

	
 
- name: Install Gunicorn via requirements file
 
  become: true
 
  become_user: "{{ admin }}"
 
  pip:
 
    requirements: "{{ home }}/.wsgi_requirements.txt"
 
    state: present
 
    virtualenv: "{{ home }}/virtualenv"
 
  register: install_gunciron_via_requirements
 
  notify:
 
    - Restart WSGI services
0 comments (0 inline, 0 general)