Changeset - 36ce706cb123
[Not reviewed]
0 8 0
Branko Majic (branko) - 20 days ago 2024-08-30 13:34:36
branko@majic.rs
MAR-239: Dropped support for Debian 11 Bullseye from the backup_client role:

- Switch to using Paramiko instead of pexpect backend (therefore
avoiding using the external SSH client binary).
8 files changed with 16 insertions and 56 deletions:
0 comments (0 inline, 0 general)
docs/releasenotes.rst
Show inline comments
 
Release notes
 
=============
 

	
 

	
 
x.y.z
 
-----
 

	
 
**New features/improvements**
 

	
 
* ``backup_client`` role
 

	
 
  * Switched to using Paramiko + SFTP backend (instead of pexpect +
 
    SFTP), which should improve the backup performance.
 

	
 
**Bug fixes:**
 

	
 
* ``common`` role
 

	
 
  * Fixed permission errors with Python cache directories in the pip
 
    requirements upgrade checks virtual environment that can happen if
 
    the initial virtual environment set-up fails.
 

	
 

	
 
8.0.0
 
-----
 

	
 
Dropped support for Python 2.7 and Debian 10 Buster. Added support for
 
Debian 12 Bookworm. Numerous minor improvements and fixes.
roles/backup_client/molecule/default/molecule.yml
Show inline comments
 
@@ -21,52 +21,24 @@ platforms:
 
  - name: backup-server
 
    box: debian/bookworm64
 
    memory: 512
 
    cpus: 1
 
    provider_raw_config_args:
 
      - "customize ['modifyvm', :id, '--paravirtprovider', 'minimal']"
 
    interfaces:
 
      - auto_config: true
 
        ip: 192.168.56.10
 
        network_name: private_network
 
        type: static
 

	
 
  - name: param-mandatory-bullseye
 
    groups:
 
      - parameters-mandatory
 
    box: debian/bullseye64
 
    memory: 256
 
    cpus: 1
 
    provider_raw_config_args:
 
      - "customize ['modifyvm', :id, '--paravirtprovider', 'minimal']"
 
    interfaces:
 
      - auto_config: true
 
        ip: 192.168.56.30
 
        network_name: private_network
 
        type: static
 

	
 
  - name: param-optional-bullseye
 
    groups:
 
      - parameters-optional
 
    box: debian/bullseye64
 
    memory: 256
 
    cpus: 1
 
    provider_raw_config_args:
 
      - "customize ['modifyvm', :id, '--paravirtprovider', 'minimal']"
 
    interfaces:
 
      - auto_config: true
 
        ip: 192.168.56.31
 
        network_name: private_network
 
        type: static
 

	
 
  - name: param-mandatory-bookworm
 
    groups:
 
      - parameters-mandatory
 
    box: debian/bookworm64
 
    memory: 256
 
    cpus: 1
 
    provider_raw_config_args:
 
      - "customize ['modifyvm', :id, '--paravirtprovider', 'minimal']"
 
    interfaces:
 
      - auto_config: true
 
        ip: 192.168.56.20
 
        network_name: private_network
roles/backup_client/molecule/default/prepare.yml
Show inline comments
 
@@ -109,18 +109,16 @@
 
        group: backup-users
 
        mode: 0770
 
      with_items: "{{ backup_users }}"
 

	
 
  handlers:
 
    - name: Restart ssh
 
      service:
 
        name: ssh
 
        state: restarted
 

	
 
  vars:
 
    backup_users:
 
      - name: bak-param-mandatory-bullseye
 
        key: "{{ lookup('file', 'tests/data/ssh/parameters-mandatory.pub') }}"
 
      - name: bak-param-mandatory-bookworm
 
        key: "{{ lookup('file', 'tests/data/ssh/parameters-mandatory.pub') }}"
 
      - name: backupuser
 
        key: "{{ lookup('file', 'tests/data/ssh/parameters-optional.pub') }}"
roles/backup_client/molecule/default/tests/test_default.py
Show inline comments
 
@@ -3,27 +3,24 @@ import os
 
import testinfra.utils.ansible_runner
 

	
 

	
 
testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
 
    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('parameters-*')
 

	
 

	
 
def test_installed_packages(host):
 
    """
 
    Tests if the necessary packages are installed.
 
    """
 

	
 
    expected_package_name = "python3-pexpect"
 

	
 
    assert host.package(expected_package_name).is_installed
 
    assert host.package('duply').is_installed
 
    assert host.package('duplicity').is_installed
 

	
 

	
 
def test_duply_directories(host):
 
    """
 
    Tests if Duply directories have been set-up correctly.
 
    """
 

	
 
    with host.sudo():
 

	
 
        for directory_path in ["/etc/duply",
roles/backup_client/molecule/default/tests/test_parameters_mandatory.py
Show inline comments
 
@@ -61,26 +61,26 @@ def test_duply_configuration_content(host):
 
    """
 
    Tests if duply configuration has been set-up correctly.
 
    """
 

	
 
    hostname = host.run('hostname').stdout.strip()
 

	
 
    with host.sudo():
 

	
 
        duply_configuration = host.file('/etc/duply/main/conf')
 

	
 
        assert "GPG_KEYS_ENC='59C26F031A129C54'" in duply_configuration.content_string
 
        assert "GPG_KEY_SIGN='59C26F031A129C54'" in duply_configuration.content_string
 
        assert "TARGET='pexpect+sftp://bak-%s@192.168.56.10:2222//duplicity'" % hostname in duply_configuration.content_string
 
        assert "DUPL_PARAMS=\"$DUPL_PARAMS --ssh-options='-oLogLevel=ERROR -oUserKnownHostsFile=/dev/null " \
 
        assert "TARGET='paramiko+sftp://bak-%s@192.168.56.10:2222//duplicity'" % hostname in duply_configuration.content_string
 
        assert "DUPL_PARAMS=\"$DUPL_PARAMS --ssh-options='-oUserKnownHostsFile=/dev/null " \
 
            "-oGlobalKnownHostsFile=/etc/duply/main/ssh/known_hosts -oIdentityFile=/etc/duply/main/ssh/identity'\"" in duply_configuration.content_string
 

	
 

	
 
def test_duply_gnupg_keyring_private_keys(host):
 
    """
 
    Tests if private key used for encryption/signing has been correctly
 
    imporeted into Duply GnuPG keyring.
 
    """
 

	
 
    with host.sudo():
 
        private_key_listing = host.run('gpg --homedir /etc/duply/main/gnupg --list-public-keys')
 

	
roles/backup_client/molecule/default/tests/test_parameters_optional.py
Show inline comments
 
@@ -60,26 +60,26 @@ def test_duply_configuration_content(host):
 
    """
 
    Tests if duply configuration has been set-up correctly.
 
    """
 

	
 
    hostname = host.run('hostname').stdout.strip()
 

	
 
    with host.sudo():
 

	
 
        duply_configuration = host.file('/etc/duply/main/conf')
 

	
 
        assert "GPG_KEYS_ENC='C4B2AE9F7A4F400A,3093C91BC3A9444B,86816FD928063B3F,8A14CD6C71223B72'" in duply_configuration.content_string
 
        assert "GPG_KEY_SIGN='C4B2AE9F7A4F400A'" in duply_configuration.content_string
 
        assert "TARGET='pexpect+sftp://backupuser@192.168.56.10:3333//duplicity/%s'" % hostname in duply_configuration.content_string
 
        assert "DUPL_PARAMS=\"$DUPL_PARAMS --ssh-options='-oLogLevel=ERROR -oUserKnownHostsFile=/dev/null " \
 
        assert "TARGET='paramiko+sftp://backupuser@192.168.56.10:3333//duplicity/%s'" % hostname in duply_configuration.content_string
 
        assert "DUPL_PARAMS=\"$DUPL_PARAMS --ssh-options='-oUserKnownHostsFile=/dev/null " \
 
            "-oGlobalKnownHostsFile=/etc/duply/main/ssh/known_hosts -oIdentityFile=/etc/duply/main/ssh/identity'\"" in duply_configuration.content_string
 

	
 

	
 
def test_duply_gnupg_keyring_private_keys(host):
 
    """
 
    Tests if private key used for encryption/signing has been correctly
 
    imporeted into Duply GnuPG keyring.
 
    """
 

	
 
    with host.sudo():
 

	
 
        private_key_listing = host.run('gpg --homedir /etc/duply/main/gnupg --list-public-keys')
roles/backup_client/tasks/main.yml
Show inline comments
 
---
 

	
 
# See duply_main_conf.j2 for details on why this is required (at least
 
# on Debian 11 Bullseye). With newer versions of Debian it might be
 
# possible to switch to Paramiko backend.
 
- name: Install pexpect for pexpect+sftp Duplicity backend
 
  apt:
 
    name: python3-pexpect
 
    state: present
 

	
 
- name: Install backup software
 
  apt:
 
    name:
 
      - duplicity
 
      - duply
 
    state: present
 

	
 
- name: Set-up Duply directories
 
  file:
 
    path: "{{ item }}"
 
    state: directory
 
    owner: root
roles/backup_client/templates/duply_main_conf.j2
Show inline comments
 
# GnuPG keys that should be used for encryption. Normally the encryption key is
 
# not available locally.
 
GPG_KEYS_ENC='{{ backup_encryption_key_id.stdout }}{% if backup_additional_encryption_keys %},{{ backup_additional_encryption_keys_ids.stdout }}{% endif %}'
 

	
 
# GnuPG key used for signing.
 
GPG_KEY_SIGN='{{ backup_encryption_key_id.stdout }}'
 

	
 
# Trust all keys available in the GnuPG keyring.
 
GPG_OPTS="--homedir /etc/duply/main/gnupg/ --trust-model always"
 

	
 
# Destination where the backups are stored at.
 
#
 
# Use the pexpect+sftp backend for Duplicity so we can (see also
 
# DUPL_PARAMS and --ssh-options):
 
#
 
#   - Pass in custom options for user/global known_hosts files (not
 
#     possible with Duplicity shipping with Debian 11 Bullseye).
 
#   - Reduce logging verbosity (avoiding output from sftp that mentions
 
#     updates of user's known_hosts file with IP addresses).
 
TARGET='pexpect+sftp://{{ backup_client_username }}@{{ backup_server }}:{{ backup_server_port }}/{{ backup_server_destination }}'
 
TARGET='paramiko+sftp://{{ backup_client_username }}@{{ backup_server }}:{{ backup_server_port }}/{{ backup_server_destination }}'
 

	
 
# Base directory to backup (root). File selection is done via include/exclude
 
# patterns.
 
SOURCE='/'
 

	
 
# Maximum age for preserving old backups. Used when running the "purge"
 
# command.
 
MAX_AGE=6M
 

	
 
# Maximum age of the last full backup performed before a new full backup is
 
# taken.
 
MAX_FULLBKP_AGE=1M
 
DUPL_PARAMS="$DUPL_PARAMS --full-if-older-than $MAX_FULLBKP_AGE " 
 

	
 
# Duplicity volume size in megabytes.
 
VOLSIZE=1024
 
DUPL_PARAMS="$DUPL_PARAMS --volsize $VOLSIZE "
 

	
 
# Output verbosity (error 0, warning 1-2, notice 3-4, info 5-8, debug 9)
 
VERBOSITY=4
 
VERBOSITY=notice
 

	
 
# Path to a directory used for restoring files from backups. The file is stored
 
# there temporarily.
 
TEMP_DIR="/tmp"
 

	
 
# Directory for storing (caching) unencrypted metadata. This metadata is used
 
# for producting incremental backups.
 
ARCH_DIR="/var/cache/duply/main/"
 

	
 
# Use the GnuPG agent for passwords prompts. Since we deploy the signing key
 
# without any encryption, this effectively means no prompts.
 
DUPL_PARAMS="$DUPL_PARAMS --use-agent"
 

	
 
# Rely only on global known_hosts file (which should only contain
 
# resolvable names), bypassing addition of IP addresses to root's
 
# known_hosts file. Log level is configured to reduce verbosity
 
# (mentions of IP address additions to user's known_hosts file). Use
 
# dedicated private key for performing logins towards the backup
 
# server.
 
DUPL_PARAMS="$DUPL_PARAMS --ssh-options='-oLogLevel=ERROR -oUserKnownHostsFile=/dev/null -oGlobalKnownHostsFile=/etc/duply/main/ssh/known_hosts -oIdentityFile=/etc/duply/main/ssh/identity'"
 
DUPL_PARAMS="$DUPL_PARAMS --ssh-options='-oUserKnownHostsFile=/dev/null -oGlobalKnownHostsFile=/etc/duply/main/ssh/known_hosts -oIdentityFile=/etc/duply/main/ssh/identity'"
 

	
 
# By default we exclude everything, and then include only specific patterns.
 
DUPL_PARAMS="$DUPL_PARAMS --include-filelist /etc/duply/main/include"
0 comments (0 inline, 0 general)