From 4e137a40d922c4f6c59876cd7bba15a105425b25 2023-11-21 23:45:49 From: Branko Majic Date: 2023-11-21 23:45:49 Subject: [PATCH] MAR-183: Use local caching and serving of ClamAV database files: - Helps avoid getting stuck due to hitting the upstream rate limiting, and also speeds-up the database downloads. --- diff --git a/roles/mail_server/molecule/default/files/clamav-database-nginx.conf b/roles/mail_server/molecule/default/files/clamav-database-nginx.conf new file mode 100644 index 0000000000000000000000000000000000000000..d7ed23f8d9f0cf72594a751052320c01b7fceac9 --- /dev/null +++ b/roles/mail_server/molecule/default/files/clamav-database-nginx.conf @@ -0,0 +1,28 @@ +server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + + root /vagrant/clamav-database; + + location / { + autoindex on; + try_files $uri $uri/ =404; + } +} + +server { + listen 443 ssl default_server; + listen [::]:443 ssl default_server; + server_name _; + + ssl_certificate_key /etc/ssl/private/nginx_https.key; + ssl_certificate /etc/ssl/certs/nginx_https.pem; + + root /vagrant/clamav-database; + + location / { + autoindex on; + try_files $uri $uri/ =404; + } +} diff --git a/roles/mail_server/molecule/default/files/cvdupdate-requirements.in b/roles/mail_server/molecule/default/files/cvdupdate-requirements.in new file mode 100644 index 0000000000000000000000000000000000000000..71109b479581fa26eea3a543fa670f9856e58e59 --- /dev/null +++ b/roles/mail_server/molecule/default/files/cvdupdate-requirements.in @@ -0,0 +1 @@ +cvdupdate diff --git a/roles/mail_server/molecule/default/files/cvdupdate-requirements.txt b/roles/mail_server/molecule/default/files/cvdupdate-requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..07da3c08b13ff22eb57324cbd45b33e9b58b0928 --- /dev/null +++ b/roles/mail_server/molecule/default/files/cvdupdate-requirements.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --allow-unsafe +# +certifi==2023.11.17 # via requests +charset-normalizer==3.3.2 # via requests +click==8.1.7 # via cvdupdate +colorama==0.4.6 # via cvdupdate +coloredlogs==15.0.1 # via cvdupdate +cvdupdate==1.1.1 # via -r requirements.in +dnspython==2.3.0 # via cvdupdate +humanfriendly==10.0 # via coloredlogs +idna==3.4 # via requests +importlib-metadata==6.7.0 # via click +rangehttpserver==1.3.3 # via cvdupdate +requests==2.31.0 # via cvdupdate +typing-extensions==4.7.1 # via importlib-metadata +urllib3==2.0.7 # via requests +zipp==3.15.0 # via importlib-metadata diff --git a/roles/mail_server/molecule/default/host_vars/clamav-database.yml b/roles/mail_server/molecule/default/host_vars/clamav-database.yml new file mode 100644 index 0000000000000000000000000000000000000000..f0e0a9e6588c0fbc735452b4d411dc19d65ce881 --- /dev/null +++ b/roles/mail_server/molecule/default/host_vars/clamav-database.yml @@ -0,0 +1,4 @@ +--- + +clamav_database_http_server_tls_certificate: "{{ lookup('file', 'tests/data/x509/server/clamav-database_https.cert.pem') }}" +clamav_database_http_server_tls_key: "{{ lookup('file', 'tests/data/x509/server/clamav-database_https.key.pem') }}" diff --git a/roles/mail_server/molecule/default/molecule.yml b/roles/mail_server/molecule/default/molecule.yml index ffe849ff7b5ad402ab82db85b2d5b3459a7d2e07..9d404be9e6cb1cce67daa28e0bb7c28c265d1191 100644 --- a/roles/mail_server/molecule/default/molecule.yml +++ b/roles/mail_server/molecule/default/molecule.yml @@ -4,6 +4,11 @@ dependency: {} driver: name: vagrant + safe_files: + # Preserve the ClamAV database files from previous runs on the + # clamav-database helper machine. Meant to avoid hitting hard + # limits for database downloads and getting completely blocked. + - "*/clamav-database/*" provider: name: virtualbox @@ -14,13 +19,25 @@ lint: platforms: + - name: clamav-database + box: debian/contrib-buster64 + memory: 512 + cpus: 1 + interfaces: + - auto_config: true + ip: 192.168.56.10 + network_name: private_network + type: static + config_options: + synced_folder: True + - name: ldap-server box: debian/contrib-buster64 memory: 256 cpus: 1 interfaces: - auto_config: true - ip: 192.168.56.10 + ip: 192.168.56.11 network_name: private_network type: static diff --git a/roles/mail_server/molecule/default/prepare.yml b/roles/mail_server/molecule/default/prepare.yml index 45d8cab07a5d33ee2f654a3014fddd15316c32b6..99275761b1b857e24d670563d484267349340ced 100644 --- a/roles/mail_server/molecule/default/prepare.yml +++ b/roles/mail_server/molecule/default/prepare.yml @@ -24,6 +24,8 @@ - "{{ item.fqdn }}" - "{{ item.fqdn[:item.fqdn.rfind('-')] }}" with_items: + - name: clamav-database_https + fqdn: database.clamav.net - name: ldap-server_ldap fqdn: ldap-server - name: parameters-mandatory-buster64_imap @@ -66,6 +68,110 @@ - nmap state: present +- name: Set-up a local ClamAV database mirror to avoid hitting upstream rate limits + hosts: clamav-database + become: true + tasks: + + - name: Install system packages for hosting the ClamAV database + apt: + name: + - nginx + - virtualenv + state: present + + - name: Set-up directory for ClamAV database sync tool virtual environment + file: + path: /var/lib/cvdupdate + state: directory + owner: vagrant + group: vagrant + mode: 0755 + + - name: Create virtual environment for running ClamAV database sync tool + become_user: vagrant + command: + cmd: "/usr/bin/virtualenv --python /usr/bin/python3 --prompt '(cvdupdate) ' /var/lib/cvdupdate" + creates: "/var/lib/cvdupdate" + + - name: Deploy pip requirements file for running the ClamAV database sync tool + copy: + src: cvdupdate-requirements.txt + dest: /var/lib/cvdupdate/requirements.txt + owner: vagrant + group: vagrant + mode: 0644 + + - name: Install requirements in the pipreqcheck virtual environment + become_user: vagrant + pip: + requirements: /var/lib/cvdupdate/requirements.txt + virtualenv: /var/lib/cvdupdate + + - name: Allow traversal of Vagrant directory by the http server user + file: + path: /vagrant/ + mode: 0711 + + - name: Create directory for storing ClamAV database files + file: + path: /vagrant/clamav-database + state: directory + owner: vagrant + group: vagrant + mode: 0755 + + - name: Configure default location for storing ClamAV database files + become_user: vagrant + command: "/var/lib/cvdupdate/bin/cvd config set --dbdir /vagrant/clamav-database/" + + - name: Download/update the ClamAV database files + become_user: vagrant + command: "/var/lib/cvdupdate/bin/cvd update" + + - name: Allow all users to read ClamAV database files + file: + path: "/vagrant/clamav-database/" + mode: "g=u-w,o=u-w" + recurse: true + + - name: Deploy nginx TLS private key + copy: + dest: "/etc/ssl/private/nginx_https.key" + content: "{{ clamav_database_http_server_tls_key }}" + mode: 0640 + owner: root + group: root + notify: + - Restart nginx + + - name: Deploy nginx TLS certificate + copy: + dest: "/etc/ssl/certs/nginx_https.pem" + content: "{{ clamav_database_http_server_tls_certificate }}" + mode: 0644 + owner: root + group: root + notify: + - Restart nginx + + - name: Deploy nginx configuration for serving the ClamAV database files + copy: + src: clamav-database-nginx.conf + dest: /etc/nginx/sites-available/default + owner: root + group: root + mode: 0644 + notify: + - Restart nginx + + handlers: + + - name: Restart nginx + service: + name: nginx + state: restarted + - hosts: buster become: true tasks: @@ -80,7 +186,9 @@ mode: 0644 state: present with_dict: - 192.168.56.10: "ldap-server backup-server" + # Force mail servers to use local ClamAV database mirror. + 192.168.56.10: "db.local.clamav.net database.clamav.net" + 192.168.56.11: "ldap-server backup-server" 192.168.56.20: "client1 smtp-server-requiring-tls" 192.168.56.21: "client2 smtp-server-refusing-tls" 192.168.56.30: "parameters-mandatory parameters-mandatory-buster64"