diff --git a/docs/releasenotes.rst b/docs/releasenotes.rst index e31d9566b7ecefe1c0170bf18148387b9a9410c1..6f251f9e3b2a0e22604c2f2be07fb94ca225e329 100644 --- a/docs/releasenotes.rst +++ b/docs/releasenotes.rst @@ -5,6 +5,15 @@ Release notes x.y.z ----- +**New features/improvements** + +* ``backup_client`` role + + * Switched to using Paramiko + SFTP backend (instead of pexpect + + SFTP), which should improve the backup performance. + +**Bug fixes:** + * ``common`` role * Fixed permission errors with Python cache directories in the pip diff --git a/roles/backup_client/molecule/default/molecule.yml b/roles/backup_client/molecule/default/molecule.yml index 1020d5d1f9817726e97aa4145444cabe5f9b17b7..0e36dbea6e2e58cdead63861dda26da8b4baa1db 100644 --- a/roles/backup_client/molecule/default/molecule.yml +++ b/roles/backup_client/molecule/default/molecule.yml @@ -30,34 +30,6 @@ platforms: network_name: private_network type: static - - name: param-mandatory-bullseye - groups: - - parameters-mandatory - box: debian/bullseye64 - memory: 256 - cpus: 1 - provider_raw_config_args: - - "customize ['modifyvm', :id, '--paravirtprovider', 'minimal']" - interfaces: - - auto_config: true - ip: 192.168.56.30 - network_name: private_network - type: static - - - name: param-optional-bullseye - groups: - - parameters-optional - box: debian/bullseye64 - memory: 256 - cpus: 1 - provider_raw_config_args: - - "customize ['modifyvm', :id, '--paravirtprovider', 'minimal']" - interfaces: - - auto_config: true - ip: 192.168.56.31 - network_name: private_network - type: static - - name: param-mandatory-bookworm groups: - parameters-mandatory diff --git a/roles/backup_client/molecule/default/prepare.yml b/roles/backup_client/molecule/default/prepare.yml index 40993baeabd93c6902428a9b9f016279da6d9d88..4976ea4c11f8a03203e5789d970ef439193d1256 100644 --- a/roles/backup_client/molecule/default/prepare.yml +++ b/roles/backup_client/molecule/default/prepare.yml @@ -118,8 +118,6 @@ vars: backup_users: - - name: bak-param-mandatory-bullseye - key: "{{ lookup('file', 'tests/data/ssh/parameters-mandatory.pub') }}" - name: bak-param-mandatory-bookworm key: "{{ lookup('file', 'tests/data/ssh/parameters-mandatory.pub') }}" - name: backupuser diff --git a/roles/backup_client/molecule/default/tests/test_default.py b/roles/backup_client/molecule/default/tests/test_default.py index 0659397c318963c813b27f3df10af41ea985b4ec..8dde3266ee697e926e27c27291af68a332697afe 100644 --- a/roles/backup_client/molecule/default/tests/test_default.py +++ b/roles/backup_client/molecule/default/tests/test_default.py @@ -12,9 +12,6 @@ def test_installed_packages(host): Tests if the necessary packages are installed. """ - expected_package_name = "python3-pexpect" - - assert host.package(expected_package_name).is_installed assert host.package('duply').is_installed assert host.package('duplicity').is_installed diff --git a/roles/backup_client/molecule/default/tests/test_parameters_mandatory.py b/roles/backup_client/molecule/default/tests/test_parameters_mandatory.py index 7da2d5a8e8024f013ed9c62fff82db238000dee2..4bf4c98ff531161d30da2cd61e519aea83ae1eff 100644 --- a/roles/backup_client/molecule/default/tests/test_parameters_mandatory.py +++ b/roles/backup_client/molecule/default/tests/test_parameters_mandatory.py @@ -70,8 +70,8 @@ def test_duply_configuration_content(host): assert "GPG_KEYS_ENC='59C26F031A129C54'" in duply_configuration.content_string assert "GPG_KEY_SIGN='59C26F031A129C54'" in duply_configuration.content_string - assert "TARGET='pexpect+sftp://bak-%s@192.168.56.10:2222//duplicity'" % hostname in duply_configuration.content_string - assert "DUPL_PARAMS=\"$DUPL_PARAMS --ssh-options='-oLogLevel=ERROR -oUserKnownHostsFile=/dev/null " \ + assert "TARGET='paramiko+sftp://bak-%s@192.168.56.10:2222//duplicity'" % hostname in duply_configuration.content_string + assert "DUPL_PARAMS=\"$DUPL_PARAMS --ssh-options='-oUserKnownHostsFile=/dev/null " \ "-oGlobalKnownHostsFile=/etc/duply/main/ssh/known_hosts -oIdentityFile=/etc/duply/main/ssh/identity'\"" in duply_configuration.content_string diff --git a/roles/backup_client/molecule/default/tests/test_parameters_optional.py b/roles/backup_client/molecule/default/tests/test_parameters_optional.py index 89b8d844fa6b958513288471641c13b76b9f553a..642ed89240e92dd0e72c0beee78bac7f08c24894 100644 --- a/roles/backup_client/molecule/default/tests/test_parameters_optional.py +++ b/roles/backup_client/molecule/default/tests/test_parameters_optional.py @@ -69,8 +69,8 @@ def test_duply_configuration_content(host): assert "GPG_KEYS_ENC='C4B2AE9F7A4F400A,3093C91BC3A9444B,86816FD928063B3F,8A14CD6C71223B72'" in duply_configuration.content_string assert "GPG_KEY_SIGN='C4B2AE9F7A4F400A'" in duply_configuration.content_string - assert "TARGET='pexpect+sftp://backupuser@192.168.56.10:3333//duplicity/%s'" % hostname in duply_configuration.content_string - assert "DUPL_PARAMS=\"$DUPL_PARAMS --ssh-options='-oLogLevel=ERROR -oUserKnownHostsFile=/dev/null " \ + assert "TARGET='paramiko+sftp://backupuser@192.168.56.10:3333//duplicity/%s'" % hostname in duply_configuration.content_string + assert "DUPL_PARAMS=\"$DUPL_PARAMS --ssh-options='-oUserKnownHostsFile=/dev/null " \ "-oGlobalKnownHostsFile=/etc/duply/main/ssh/known_hosts -oIdentityFile=/etc/duply/main/ssh/identity'\"" in duply_configuration.content_string diff --git a/roles/backup_client/tasks/main.yml b/roles/backup_client/tasks/main.yml index dc8c521002661743d0383e7f03d766650f1d6157..a818e9178e0a766be6a57bad1250a32556e55857 100644 --- a/roles/backup_client/tasks/main.yml +++ b/roles/backup_client/tasks/main.yml @@ -1,13 +1,5 @@ --- -# See duply_main_conf.j2 for details on why this is required (at least -# on Debian 11 Bullseye). With newer versions of Debian it might be -# possible to switch to Paramiko backend. -- name: Install pexpect for pexpect+sftp Duplicity backend - apt: - name: python3-pexpect - state: present - - name: Install backup software apt: name: diff --git a/roles/backup_client/templates/duply_main_conf.j2 b/roles/backup_client/templates/duply_main_conf.j2 index e19ec72df902180e3db2f79d616b59797e51a83c..228aa079209c39ac11c951daa3c91e8714a573ab 100644 --- a/roles/backup_client/templates/duply_main_conf.j2 +++ b/roles/backup_client/templates/duply_main_conf.j2 @@ -9,15 +9,7 @@ GPG_KEY_SIGN='{{ backup_encryption_key_id.stdout }}' GPG_OPTS="--homedir /etc/duply/main/gnupg/ --trust-model always" # Destination where the backups are stored at. -# -# Use the pexpect+sftp backend for Duplicity so we can (see also -# DUPL_PARAMS and --ssh-options): -# -# - Pass in custom options for user/global known_hosts files (not -# possible with Duplicity shipping with Debian 11 Bullseye). -# - Reduce logging verbosity (avoiding output from sftp that mentions -# updates of user's known_hosts file with IP addresses). -TARGET='pexpect+sftp://{{ backup_client_username }}@{{ backup_server }}:{{ backup_server_port }}/{{ backup_server_destination }}' +TARGET='paramiko+sftp://{{ backup_client_username }}@{{ backup_server }}:{{ backup_server_port }}/{{ backup_server_destination }}' # Base directory to backup (root). File selection is done via include/exclude # patterns. @@ -37,7 +29,7 @@ VOLSIZE=1024 DUPL_PARAMS="$DUPL_PARAMS --volsize $VOLSIZE " # Output verbosity (error 0, warning 1-2, notice 3-4, info 5-8, debug 9) -VERBOSITY=4 +VERBOSITY=notice # Path to a directory used for restoring files from backups. The file is stored # there temporarily. @@ -57,7 +49,7 @@ DUPL_PARAMS="$DUPL_PARAMS --use-agent" # (mentions of IP address additions to user's known_hosts file). Use # dedicated private key for performing logins towards the backup # server. -DUPL_PARAMS="$DUPL_PARAMS --ssh-options='-oLogLevel=ERROR -oUserKnownHostsFile=/dev/null -oGlobalKnownHostsFile=/etc/duply/main/ssh/known_hosts -oIdentityFile=/etc/duply/main/ssh/identity'" +DUPL_PARAMS="$DUPL_PARAMS --ssh-options='-oUserKnownHostsFile=/dev/null -oGlobalKnownHostsFile=/etc/duply/main/ssh/known_hosts -oIdentityFile=/etc/duply/main/ssh/identity'" # By default we exclude everything, and then include only specific patterns. DUPL_PARAMS="$DUPL_PARAMS --include-filelist /etc/duply/main/include"