# -*- coding: utf-8 -*-

#

# Majic Ansible Roles documentation build configuration file, created by

# sphinx-quickstart on Sat Nov  8 14:03:39 2014.

#

# This file is execfile()d with the current directory set to its

# containing dir.

#

# Note that not all possible configuration values are present in this

# autogenerated file.

#

# All configuration values have a default; values that are commented out

# serve to show the default.

# import sys

# import os

# If extensions (or modules to document with autodoc) are in another directory,

# add these directories to sys.path here. If the directory is relative to the

# documentation root, use os.path.abspath to make it absolute, like shown here.

# sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.

# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be

# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom

# ones.

extensions = []

# Add any paths that contain templates here, relative to this directory.

templates_path = ['_templates']

# The suffix of source filenames.

source_suffix = '.rst'

# The encoding of source files.

# source_encoding = 'utf-8-sig'

# The master toctree document.

master_doc = 'index'

# General information about the project.

project = u'Majic Ansible Roles'

copyright = u'2018, Branko Majic'

# The version info for the project you're documenting, acts as replacement for

# |version| and |release|, also used in various other places throughout the

# built documents.

#

# The short X.Y version.

# The full version, including alpha/beta/rc tags.

# The language for content autogenerated by Sphinx. Refer to documentation

# for a list of supported languages.

#

# This is also used if you do content translation via gettext catalogs.

# Usually you set "language" from the command line for these cases.

language = None

# There are two options for replacing |today|: either, you set today to some

# non-false value, then it is used:

# today = ''

# Else, today_fmt is used as the format for a strftime call.

# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and

# directories to ignore when looking for source files.

exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all

# documents.

# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.

# add_function_parentheses = True

# If true, the current module name will be prepended to all description

# unit titles (such as .. function::).

# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the

# output. They are ignored by default.

# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.

pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.

# modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.

# keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for

# a list of builtin themes.

html_theme = 'sphinx_rtd_theme'

# Theme options are theme-specific and customize the look and feel of a theme

# further.  For a list of options available for each theme, see the

# documentation.

# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.

# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to

# "<project> v<release> documentation".

# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.

# html_short_title = None

# The name of an image file (relative to this directory) to place at the top

# of the sidebar.

# html_logo = None

# The name of an image file (within the static path) to use as favicon of the

# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32

# pixels large.

# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,

# relative to this directory. They are copied after the builtin static files,

# so a file named "default.css" will overwrite the builtin "default.css".

html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or

# .htaccess) here, relative to this directory. These files are copied

# directly to the root of the documentation.

# html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,

# using the given strftime format.

# html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to

# typographically correct entities.

# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.

# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to

# template names.

# html_additional_pages = {}

# If false, no module index is generated.

# html_domain_indices = True

# If false, no index is generated.

# html_use_index = True

# If true, the index is split into individual pages for each letter.

# html_split_index = False

# If true, links to the reST sources are added to the pages.

# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.

# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.

# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will

# contain a <link> tag referring to it.  The value of this option must be the

# base URL from which the finished HTML is served.

# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").

# html_file_suffix = None

# Language to be used for generating the HTML full-text search index.

# Sphinx supports the following languages:

#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'

#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'

# html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.

# Now only 'ja' uses this config value

# html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that

# implements a search results scorer. If empty, the default will be used.

# html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.

htmlhelp_basename = 'MajicAnsibleRolesdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {

    # The paper size ('letterpaper' or 'a4paper').

    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').

    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.

    # 'preamble': '',

    # Latex figure (float) alignment

    # 'figure_align': 'htbp',

}

# Grouping the document tree into LaTeX files. List of tuples

# (source start file, target name, title,

#  author, documentclass [howto, manual, or own class]).

latex_documents = [

  ('index', 'MajicAnsibleRoles.tex', u'Majic Ansible Roles Documentation',

   u'Branko Majic', 'manual'),

]

# The name of an image file (relative to this directory) to place at the top of

# the title page.

# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,

# not chapters.

# latex_use_parts = False

# If true, show page references after internal links.

# latex_show_pagerefs = False

# If true, show URL addresses after external links.

# latex_show_urls = False

# Documents to append as an appendix to all manuals.

# latex_appendices = []

# If false, no module index is generated.

# latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples

# (source start file, name, description, authors, manual section).

man_pages = [

    ('index', 'majicansibleroles', u'Majic Ansible Roles Documentation',

Release notes

=============

-----

**Bug fixes:**

* ``common`` role

  * Fix deprecation warnings for Python requirements upgrade checks

    when using pip-tools 7.3.0. This would result in unnecessary

    notifications being sent out to server administrator.

7.1.0

-----

Added support for Debian 11 (Bullseye), with some smaller bug fixes.

**New features/improvements**

* All roles

  * Added support for Debian 11 (Bullseye).

**Bug fixes:**

* ``xmpp_server`` role

  * Make sure to take care of deprecation-related package and

    configuration removals prior to running the rest of the tasks to

    avoid errors related to deprecated elements being invalid (like

    repository URLs for Prosody).

7.0.0

-----

Dropped support for Debian 9 (Stretch), moved away from using

non-Debian project repositories (like Prosody ones).

**Breaking changes:**

* All roles

  * Dropped support for Debian 9 (Stretch).

* ``xmpp_server`` role

  * Parameter ``xmpp_prosody_package`` has been dropped.

**New features/improvements**

* ``common`` role

  * Added parameters ``pip_check_requirements_in`` and

    ``pip_check_requirements_py3_in`` that can be used for specifying

    input requirements when checking for available package upgrades

    for Python virtual environments that are used for the checks

    themselves. This is particularly helpful in cases where Python

    version gets deprecated and some packages do not correctly declare

    the minimum version required, allowing to be more specific to

    avoid unnecessary warning mails being sent out.

  * Updated default package pins for virtual environments used to

    check for available pip package upgrades.

* ``mail_server`` role

  * Added parameter ``mail_server_smtp_additional_configuration`` that

    provides ability to include additional configuration directives

    for the SMTP server.

* ``xmpp_server`` role

  * Drop dependency on the external (Prosody) package

    repository. Install everything using official Debian

    repositories. This should help avoid future issues with Prosody

    project removing older versions of packages or dropping entire

    repository archives for older Debian releases.

  * Prosody package and some of its dependencies are installed from

    Debian backports to get more featureful release installed.

  * Role no longer depends on fetching external Prosody modules from

    project code repository, and instead relies on the prosody-modules

    package for LDAP authentication module.

6.0.0

-----

Added support for Debian 10 (Buster), alongside a couple of minor

changes and features/improvements.

**Breaking changes:**

* ``ldap_server`` role

  * Use 2048-bit Diffie-Hellman parameters for relevant TLS

    ciphers. This could introduce incompatibility with older

    clients/servers trying to connect to the LDAP server. This change

    is applicable only under Debian Buster.

**New features/improvements:**

* All roles

  * Added support for Debian 10 (Buster).

* ``common`` role

  * Added parameters ``maintenance`` and ``maintenance_allowed_hosts``

    for enabling maintenance mode. In maintenance mode only the listed

    hosts are allowed to connect to the server.

**Bug fixes:**

* ``ldap_server`` role

  * Allow use of DHE TLS ciphers by generating the necessary

    Diffie-Hellman parameters. This bug fix is applicable only under

    Debian Buster.

* ``wsgi_website_`` role

  * When the virtual environment is created, the ``setuptools`` and

    ``pip`` packages will not get pinned to any specific version,

    allowing roles that are based on ``wsgi_website`` to easily

    install preferred versions, and avoid idempotence problems in the

    process.

5.0.0

-----

Upgrade to Ansible 2.9.x, dropping support for Debian 8 Jessie,

upgrade to Python 3.x, dropping support for Python 2.7. A number of

parameters have been made mandatory or deprecated. Security has been

slightly improved in a number of roles, and there is plenty of

bug-fixes and minor improvements throughout as well.

**Breaking changes:**

* Switched to Ansible 2.9.x, removing support for older versions. All

  documentation has been updated.

* Switched to using Python 3 on both controller and managed server

  side. Python 2.7 can no longer be used for this purpose. Support for

  WSGI applications running on Python 2.7 remains.

* All roles

  * Support for Debian 8 Jessie has been dropped.

  * Common parameters ``tls_private_key_dir`` and

    ``tls_certificate_dir`` are no longer used.

  * TLS private key and certificate parameters are now mandatory.

* ``bootstrap`` role

  * Parameter ``ansible_key`` is now mandatory.

* ``common`` role``

  * Minimum version of ``pip-tools`` in the ``pip_check_requirements``

    and ``pip_check_requirements_py3`` is now 5.3.0. This change was

    required in order to fix the deprecation warnings being sent out

    when the ``pip_check_requirements_upgrades.sh`` script is run.

* ``database_server`` role

  * Parameter ``db_root_password`` has been deprecated. The root user

    can now login into the database (as the root database user) via

    unix socket authentication.

  * Role will drop the use of Debian system maintenance user

    (``debian-sys-maint``) in favour of using the root account with

    UNIX socket authentication if the database server has not already

    been set-up in that manner. This is the default behaviour starting

    from Debian Stretch, and the ``debian-sys-main`` will be present

    only if the server has been upgraded from older releases.

* ``ldap_server`` role

  * Parameter ``ldap_server_domain`` is now mandatory.

  * Updated default set of TLS ciphers used by server

    (``ldap_tls_ciphers`` parameter). All CBC ciphers have been

    dropped. This could introduce incompatibility with older clients

    trying to connect to the LDAP server.

* ``mail_forwarder`` role

  * Use 2048-bit Diffie-Hellman parameters for relevant TLS

    ciphers. This could introduce incompatibility with older

    clients/servers trying to connect to the SMTP server.

* ``mail_server`` role

.. _usage:

Usage

=====

Majic Ansible Roles are targeted at sysadmins who wish to deploy services for

their own, small-scale use. This chapter gives a simple tutorial-like set of

instructions for using all of the roles available.

.. contents:: :local:

Overview

--------

There is a number of different roles that can prove useful for setting-up a

small infrastructure of your own.

Some roles are suited for one-off operations during installation, like the

``preseed`` and ``bootstrap``, while some are better suited for periodic runs

for maintaining the users and integrity of the system.

By the end of following the instructions, you will have the following:

* Ansible server, used as controller for configuring and managing the

  remaining servers.

* Communications server, providing the LDAP, mail, and XMPP services.

* Web server, providing the web services.

* Backup server, used for storing all of the backups.

.. warning::

   Majic Ansible Roles support *only* Python 3 - both on the

   controller side and on the managed servers side.

   It is important to make sure that both the controller Python

   virtual environment used for Ansible *and* the interpreter for

   remote servers are *both* set-up to use Python 3.

   Python 3 is specified explicitly during virtual environment

   creation and in ``ansible.cfg`` configuration file

   (``interpreter_python`` option under ``defaults`` section).

Pre-requisites

--------------

For the set-up outlined in this usage guide you'll need the following:

* One server where Ansible will be installed at. Debian Bullseye will

  be installed on top of this server. The server will be set-up

  manually (this is currently out of scope for the *Majic Ansible

  Roles* automated set-up).

* Three servers where the services will be set-up. All servers must be able to

  communicate over network with each-other, the Ansible servers, and with

  Internet. Debian Bullseye will be installed on top of this server as part of the

  usage instructions.

* Debian Bullseye network install CD.

* All servers should be on the same network.

* IP addresses for all servers should be known.

* Netmask for all servers should be known.

* Gateway for all servers should be known.

In case of the servers listed above, it might be safest to have them as VMs -

this is cheapest thing to do, and simplest (who wants to deal with pesky hardware anyway?).

Usage instructions assume the following:

* Domain used for all servers is ``example.com``. If you wish to use a different

  domain, adjust the instructions accordingly.

* Server hostnames are ``ansible``, ``comms``, ``www``, and ``bak`` (for Ansible

  server, communications server, web server, and backup server, respectively).

Installing the OS on Ansible server

-----------------------------------

Start-off by installing the operating system on the Ansible server:

1. Fire-up the ``ansible`` server, and boot from the network installation CD.

2. Select the **Install** option.

3. Pick **English** as language.

4. Pick the country you are living in (or whatever else you want).

5. Pick the **en_US.UTF-8** locale.

6. Pick the **American English** keymap.

7. Configure the network if necessary.

8. Set the hostname to ``ansible``.

9. Set the domain to ``example.com``.

10. Set the root password.

11. Create a new user. For simplicity, call the user **Ansible user**, with

    username **ansible**.

12. Set-up partitioning in any way you want. You can go for **Guided - use

    entire disk** if you want to keep it simple and are just testing things.

13. Wait until the base system has been installed.

14. Pick whatever Debian archive mirror is closest to you.

15. If you have an HTTP proxy, provide its URL.

16. Pick if you want to participate in package survey or not.

17. Make sure that at least the **standard system utilities** and **SSH server**

    options are selected on task selection screen.

18. Wait for packages to be installed.

19. Install the GRUB boot loader on MBR.

20. Finalise the server install, and remove the installation media from server.

Installing required packages

----------------------------

With the operating system installed, it is necessary to install a couple of

packages, and to prepare the environment a bit on the Ansible server:

1. Install the necessary system packages (using the ``root`` account)::

     apt-get install -y virtualenv virtualenvwrapper git python3-pip python3-dev libffi-dev libssl-dev

2. Set-up loading of ``virtualenvwrapper`` via Bash completions (using the ``root`` account)::

     ln -s /usr/share/bash-completion/completions/virtualenvwrapper /etc/bash_completion.d/virtualenvwrapper

3. Set-up the virtual environment (using the ``ansible`` account):

   .. warning::

      If you are already logged-in as user ``ansible`` in the server, you will

      need to log-out and log-in again in order to be able to use

      ``virtualenvwrapper`` commands!

   ::

     mkdir ~/mysite/

     mkvirtualenv -p /usr/bin/python3 -a ~/mysite/ mysite

     pip install -U pip setuptools

     pip install 'ansible~=2.9.0' dnspython

.. warning::

   The ``dnspython`` package is important since it is used internally via

   ``dig`` lookup plugin.

Cloning the *Majic Ansible Roles*

---------------------------------

With most of the software pieces in place, the only missing thing is the Majic

Ansible Roles:

1. Clone the git repository::

     git clone https://code.majic.rs/majic-ansible-roles ~/majic-ansible-roles

2. Checkout the correct version of the roles::

     cd ~/majic-ansible-roles/

Preparing the basic site configuration

--------------------------------------

Phew... Now that was a bit tedious and boring... But at least you are now ready

to set-up your own site :)

First of all, let's set-up some basic directory structure and configuration:

1. Create Ansible configuration file.

   .. warning::

      Since Ansible 2.x has introduced much stricter controls over security of

      deployed Python scripts, it is recommended (as in this example) to use the

      ``pipelining`` option (which should also improve performance). This is in

      particular necessary in cases where the SSH user connecting to remote

      machine is *not* ``root``, but there are tasks that use ``become`` with

      non-root ``become_user`` (which is the case in Majic Ansible Roles). See

      `official documentation

      <https://docs.ansible.com/ansible/latest/become.html#becoming-an-unprivileged-user>`_

      and other alternatives to this.

   :file:`~/mysite/ansible.cfg`

   ::

     [defaults]

     roles_path=/home/ansible/majic-ansible-roles/roles:/home/ansible/mysite/roles

     force_handlers = True

     inventory = /home/ansible/mysite/hosts

     interpreter_python = /usr/bin/python3

     [ssh_connection]

     pipelining = True

2. Create directory where retry files will be stored at (so they woudln't

   pollute your home directory)::

     mkdir ~/mysite/retry

3. Create the inventory file.

   :file:`~/mysite/hosts`

   ::

     [preseed]

     localhost ansible_connection=local

     [communications]

     comms.example.com

     [web]

     www.example.com

     [backup]

     bak.example.com

4. Create a number of directories for storing playbooks, group

   variables, SSH keys, X.509 artefacts (for TLS), and GnuPG keyring

   (we'll get to this later)::

     mkdir ~/mysite/playbooks/

     mkdir ~/mysite/group_vars/

     mkdir ~/mysite/ssh/

     mkdir ~/mysite/tls/

     mkdir ~/mysite/gnupg/

5. Create SSH private/public key pair that will be used by Ansible for

   connecting to destination servers, as well as for some roles::

     ssh-keygen -f ~/.ssh/id_rsa -N ''

Protecting communications using TLS

-----------------------------------

In order to protect the communications between users and servers, as

well as between servers themselves, it is important to set-up and

properly configure TLS for each role.

*Majic Ansible Roles* mandates use of TLS wherever possible. In other

words, *you must* have TLS private keys and certificates issued by

some CA for all servers in order to be able to use most of the

roles. The private keys and certificates are primarily meant to be

generated *per service*, and that is the approach we will pursue here

as well.

TLS private keys should be ideally generated locally and kept in a

safe environment (possibly encrypted until needed), while the X.509

certificates should be issued by a relevant certification

authority. You can choose to roll-out your own CA, use one of the

public CAs, or perhaps go for a mix of both.

For the purpose of this guide, we'll set-up a small simple local CA to

issue all the necessary certificates, and we'll generate the private

keys and issue server certificates on the go as needed, storing them

all under the ``~/mysite/tls/`` directory.

So, let us make a slight detour to create a CA of our own:

1. First off, install a couple more tools on the Ansible server. We

   will be using ``certtool`` for our improvised CA needs (run this as

   ``root``)::

     apt-get install -y gnutls-bin

2. Create a template for the ``certtool`` so it would know what

   extensions and content to have in the CA certificate:

   :file:`~/mysite/tls/ca.cfg`

   ::

      organization = "Example Inc."

      country = "SE"

      cn = "Example Inc. Test Site CA"

      expiration_days = 1825

      ca

      cert_signing_key

      crl_signing_key

3. Almost there... Now let us generate the CA private key and

   self-signed certificate::

     certtool --sec-param high --generate-privkey --outfile ~/mysite/tls/ca.key

     certtool --template ~/mysite/tls/ca.cfg --generate-self-signed --load-privkey ~/mysite/tls/ca.key --outfile ~/mysite/tls/ca.pem

4. And just one more small tweak - we need to provide a truststore PEM

   file containing all CA certificates in the chain for services to be

   able to connect to each-other (where necessary). In this particular

   case we have a super-simple hierarchy (root CA is also issuing the

   end entity certificates), so simply make a copy of the ``ca.pem``::

     cp ~/mysite/tls/ca.pem ~/mysite/tls/truststore.pem

.. note::

   A useful feature that all roles implement is a check to see if

   certificates will expire within the next 30 days. This check is

   performed via cronjob at midnight, and failing results will end-up

   being delivered to the ``root`` user on local server. Later on,

   once you have configured the mail server, you should be able to

   set-up the necessary aliases to have the mails delivered to

   non-local accounts too.

Preseed files

-------------

The ``preseed`` role is useful for generating Debian preseed files. Preseed

files can be used for automating the Debian installation process.

Preseed files are created on the Ansible controller, and then supplied

to Debian installer.

So, let's set this up for start:

1. First of all, create the playbook for generating the preseed files locally.

   :file:`~/mysite/playbooks/preseed.yml`

   ::

      ---

      - hosts: preseed

        roles:

          - preseed

2. Now we need to configure the role. Two parameters are mandatory -

   one that specifies where the preseed files are to be stored, and

   one that specifies the public key that should be used to

   pre-populate the SSH authorized keys for the ``root`` account. This

   is required for the initial bootstrap of servers because Debian

   GNU/Linux does not by default allow the ``root`` user to

   authenticate via SSH using a password. We will use the SSH public

   key generated earlier via the ``ssh-keygen`` command. Create the

   configuration file:

   :file:`~/mysite/group_vars/preseed.yml`

   ::

      ---

      # Public key used to authenticate remote logins via SSH for the

      # root account.

      ansible_key: "{{ lookup('file', '~/.ssh/id_rsa.pub') }}"

      # Directory where the preseed files will be output to.

      preseed_directory: "~/mysite/preseed_files/"