Changeset - 4e137a40d922
[Not reviewed]
0 2 4
Branko Majic (branko) - 5 months ago 2023-11-21 23:45:49
branko@majic.rs
MAR-183: Use local caching and serving of ClamAV database files:

- Helps avoid getting stuck due to hitting the upstream rate limiting,
and also speeds-up the database downloads.
6 files changed with 181 insertions and 2 deletions:
0 comments (0 inline, 0 general)
roles/mail_server/molecule/default/files/clamav-database-nginx.conf
Show inline comments
 
new file 100644
 
server {
 
    listen 80 default_server;
 
    listen [::]:80 default_server;
 
    server_name _;
 

	
 
    root /vagrant/clamav-database;
 

	
 
    location / {
 
        autoindex on;
 
	try_files $uri $uri/ =404;
 
    }
 
}
 

	
 
server {
 
    listen 443 ssl default_server;
 
    listen [::]:443 ssl default_server;
 
    server_name _;
 

	
 
    ssl_certificate_key /etc/ssl/private/nginx_https.key;
 
    ssl_certificate /etc/ssl/certs/nginx_https.pem;
 

	
 
    root /vagrant/clamav-database;
 

	
 
    location / {
 
        autoindex on;
 
	try_files $uri $uri/ =404;
 
    }
 
}
roles/mail_server/molecule/default/files/cvdupdate-requirements.in
Show inline comments
 
new file 100644
 
cvdupdate
roles/mail_server/molecule/default/files/cvdupdate-requirements.txt
Show inline comments
 
new file 100644
 
#
 
# This file is autogenerated by pip-compile
 
# To update, run:
 
#
 
#    pip-compile --allow-unsafe
 
#
 
certifi==2023.11.17       # via requests
 
charset-normalizer==3.3.2  # via requests
 
click==8.1.7              # via cvdupdate
 
colorama==0.4.6           # via cvdupdate
 
coloredlogs==15.0.1       # via cvdupdate
 
cvdupdate==1.1.1          # via -r requirements.in
 
dnspython==2.3.0          # via cvdupdate
 
humanfriendly==10.0       # via coloredlogs
 
idna==3.4                 # via requests
 
importlib-metadata==6.7.0  # via click
 
rangehttpserver==1.3.3    # via cvdupdate
 
requests==2.31.0          # via cvdupdate
 
typing-extensions==4.7.1  # via importlib-metadata
 
urllib3==2.0.7            # via requests
 
zipp==3.15.0              # via importlib-metadata
roles/mail_server/molecule/default/host_vars/clamav-database.yml
Show inline comments
 
new file 100644
 
---
 

	
 
clamav_database_http_server_tls_certificate: "{{ lookup('file', 'tests/data/x509/server/clamav-database_https.cert.pem') }}"
 
clamav_database_http_server_tls_key: "{{ lookup('file', 'tests/data/x509/server/clamav-database_https.key.pem') }}"
roles/mail_server/molecule/default/molecule.yml
Show inline comments
 
@@ -4,6 +4,11 @@ dependency: {}
 

	
 
driver:
 
  name: vagrant
 
  safe_files:
 
    # Preserve the ClamAV database files from previous runs on the
 
    # clamav-database helper machine. Meant to avoid hitting hard
 
    # limits for database downloads and getting completely blocked.
 
    - "*/clamav-database/*"
 
  provider:
 
    name: virtualbox
 

	
 
@@ -14,13 +19,25 @@ lint:
 

	
 
platforms:
 

	
 
  - name: clamav-database
 
    box: debian/contrib-buster64
 
    memory: 512
 
    cpus: 1
 
    interfaces:
 
      - auto_config: true
 
        ip: 192.168.56.10
 
        network_name: private_network
 
        type: static
 
    config_options:
 
      synced_folder: True
 

	
 
  - name: ldap-server
 
    box: debian/contrib-buster64
 
    memory: 256
 
    cpus: 1
 
    interfaces:
 
      - auto_config: true
 
        ip: 192.168.56.10
 
        ip: 192.168.56.11
 
        network_name: private_network
 
        type: static
 

	
roles/mail_server/molecule/default/prepare.yml
Show inline comments
 
@@ -24,6 +24,8 @@
 
          - "{{ item.fqdn }}"
 
          - "{{ item.fqdn[:item.fqdn.rfind('-')] }}"
 
      with_items:
 
        - name: clamav-database_https
 
          fqdn: database.clamav.net
 
        - name: ldap-server_ldap
 
          fqdn: ldap-server
 
        - name: parameters-mandatory-buster64_imap
 
@@ -66,6 +68,110 @@
 
          - nmap
 
        state: present
 

	
 
- name: Set-up a local ClamAV database mirror to avoid hitting upstream rate limits
 
  hosts: clamav-database
 
  become: true
 
  tasks:
 

	
 
    - name: Install system packages for hosting the ClamAV database
 
      apt:
 
        name:
 
          - nginx
 
          - virtualenv
 
        state: present
 

	
 
    - name: Set-up directory for ClamAV database sync tool virtual environment
 
      file:
 
        path: /var/lib/cvdupdate
 
        state: directory
 
        owner: vagrant
 
        group: vagrant
 
        mode: 0755
 

	
 
    - name: Create virtual environment for running ClamAV database sync tool
 
      become_user: vagrant
 
      command:
 
        cmd: "/usr/bin/virtualenv --python /usr/bin/python3 --prompt '(cvdupdate) ' /var/lib/cvdupdate"
 
        creates: "/var/lib/cvdupdate"
 

	
 
    - name: Deploy pip requirements file for running the ClamAV database sync tool
 
      copy:
 
        src: cvdupdate-requirements.txt
 
        dest: /var/lib/cvdupdate/requirements.txt
 
        owner: vagrant
 
        group: vagrant
 
        mode: 0644
 

	
 
    - name: Install requirements in the pipreqcheck virtual environment
 
      become_user: vagrant
 
      pip:
 
        requirements: /var/lib/cvdupdate/requirements.txt
 
        virtualenv: /var/lib/cvdupdate
 

	
 
    - name: Allow traversal of Vagrant directory by the http server user
 
      file:
 
        path: /vagrant/
 
        mode: 0711
 

	
 
    - name: Create directory for storing ClamAV database files
 
      file:
 
        path: /vagrant/clamav-database
 
        state: directory
 
        owner: vagrant
 
        group: vagrant
 
        mode: 0755
 

	
 
    - name: Configure default location for storing ClamAV database files
 
      become_user: vagrant
 
      command: "/var/lib/cvdupdate/bin/cvd config set --dbdir /vagrant/clamav-database/"
 

	
 
    - name: Download/update the ClamAV database files
 
      become_user: vagrant
 
      command: "/var/lib/cvdupdate/bin/cvd update"
 

	
 
    - name: Allow all users to read ClamAV database files
 
      file:
 
        path: "/vagrant/clamav-database/"
 
        mode: "g=u-w,o=u-w"
 
        recurse: true
 

	
 
    - name: Deploy nginx TLS private key
 
      copy:
 
        dest: "/etc/ssl/private/nginx_https.key"
 
        content: "{{ clamav_database_http_server_tls_key }}"
 
        mode: 0640
 
        owner: root
 
        group: root
 
      notify:
 
        - Restart nginx
 

	
 
    - name: Deploy nginx TLS certificate
 
      copy:
 
        dest: "/etc/ssl/certs/nginx_https.pem"
 
        content: "{{ clamav_database_http_server_tls_certificate }}"
 
        mode: 0644
 
        owner: root
 
        group: root
 
      notify:
 
        - Restart nginx
 

	
 
    - name: Deploy nginx configuration for serving the ClamAV database files
 
      copy:
 
        src: clamav-database-nginx.conf
 
        dest: /etc/nginx/sites-available/default
 
        owner: root
 
        group: root
 
        mode: 0644
 
      notify:
 
        - Restart nginx
 

	
 
  handlers:
 

	
 
    - name: Restart nginx
 
      service:
 
        name: nginx
 
        state: restarted
 

	
 
- hosts: buster
 
  become: true
 
  tasks:
 
@@ -80,7 +186,9 @@
 
        mode: 0644
 
        state: present
 
      with_dict:
 
        192.168.56.10: "ldap-server backup-server"
 
        # Force mail servers to use local ClamAV database mirror.
 
        192.168.56.10: "db.local.clamav.net database.clamav.net"
 
        192.168.56.11: "ldap-server backup-server"
 
        192.168.56.20: "client1 smtp-server-requiring-tls"
 
        192.168.56.21: "client2 smtp-server-refusing-tls"
 
        192.168.56.30: "parameters-mandatory parameters-mandatory-buster64"
0 comments (0 inline, 0 general)