# -*- coding: utf-8 -*-

#

# Majic Ansible Roles documentation build configuration file, created by

# sphinx-quickstart on Sat Nov  8 14:03:39 2014.

#

# This file is execfile()d with the current directory set to its

# containing dir.

#

# Note that not all possible configuration values are present in this

# autogenerated file.

#

# All configuration values have a default; values that are commented out

# serve to show the default.

import sys

import os

# If extensions (or modules to document with autodoc) are in another directory,

# add these directories to sys.path here. If the directory is relative to the

# documentation root, use os.path.abspath to make it absolute, like shown here.

#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.

#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be

# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom

# ones.

extensions = []

# Add any paths that contain templates here, relative to this directory.

templates_path = ['_templates']

# The suffix of source filenames.

source_suffix = '.rst'

# The encoding of source files.

#source_encoding = 'utf-8-sig'

# The master toctree document.

master_doc = 'index'

# General information about the project.

project = u'Majic Ansible Roles'

copyright = u'2016, Branko Majic'

# The version info for the project you're documenting, acts as replacement for

# |version| and |release|, also used in various other places throughout the

# built documents.

#

# The short X.Y version.

# The full version, including alpha/beta/rc tags.

# The language for content autogenerated by Sphinx. Refer to documentation

# for a list of supported languages.

#

# This is also used if you do content translation via gettext catalogs.

# Usually you set "language" from the command line for these cases.

language = None

# There are two options for replacing |today|: either, you set today to some

# non-false value, then it is used:

#today = ''

# Else, today_fmt is used as the format for a strftime call.

#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and

# directories to ignore when looking for source files.

exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all

# documents.

#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.

#add_function_parentheses = True

# If true, the current module name will be prepended to all description

# unit titles (such as .. function::).

#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the

# output. They are ignored by default.

#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.

pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.

#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.

#keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for

# a list of builtin themes.

html_theme = 'classic'

# Theme options are theme-specific and customize the look and feel of a theme

# further.  For a list of options available for each theme, see the

# documentation.

#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.

#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to

# "<project> v<release> documentation".

#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.

#html_short_title = None

# The name of an image file (relative to this directory) to place at the top

# of the sidebar.

#html_logo = None

# The name of an image file (within the static path) to use as favicon of the

# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32

# pixels large.

#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,

# relative to this directory. They are copied after the builtin static files,

# so a file named "default.css" will overwrite the builtin "default.css".

html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or

# .htaccess) here, relative to this directory. These files are copied

# directly to the root of the documentation.

#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,

# using the given strftime format.

#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to

# typographically correct entities.

#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.

#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to

# template names.

#html_additional_pages = {}

# If false, no module index is generated.

#html_domain_indices = True

# If false, no index is generated.

#html_use_index = True

# If true, the index is split into individual pages for each letter.

#html_split_index = False

# If true, links to the reST sources are added to the pages.

#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.

#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.

#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will

# contain a <link> tag referring to it.  The value of this option must be the

# base URL from which the finished HTML is served.

#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").

#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.

# Sphinx supports the following languages:

#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'

#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'

#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.

# Now only 'ja' uses this config value

#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that

# implements a search results scorer. If empty, the default will be used.

#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.

htmlhelp_basename = 'MajicAnsibleRolesdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {

# The paper size ('letterpaper' or 'a4paper').

#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').

#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.

#'preamble': '',

# Latex figure (float) alignment

#'figure_align': 'htbp',

}

# Grouping the document tree into LaTeX files. List of tuples

# (source start file, target name, title,

#  author, documentclass [howto, manual, or own class]).

latex_documents = [

  ('index', 'MajicAnsibleRoles.tex', u'Majic Ansible Roles Documentation',

   u'Branko Majic', 'manual'),

]

# The name of an image file (relative to this directory) to place at the top of

# the title page.

#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,

# not chapters.

#latex_use_parts = False

# If true, show page references after internal links.

#latex_show_pagerefs = False

# If true, show URL addresses after external links.

#latex_show_urls = False

# Documents to append as an appendix to all manuals.

#latex_appendices = []

# If false, no module index is generated.

#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples

# (source start file, name, description, authors, manual section).

man_pages = [

    ('index', 'majicansibleroles', u'Majic Ansible Roles Documentation',

     [u'Branko Majic'], 1)

]

# If true, show URL addresses after external links.

#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples

# (source start file, target name, title, author,

#  dir menu entry, description, category)

texinfo_documents = [

  ('index', 'MajicAnsibleRoles', u'Majic Ansible Roles Documentation',

   u'Branko Majic', 'MajicAnsibleRoles', 'One line description of project.',

   'Miscellaneous'),

]

# Documents to append as an appendix to all manuals.

#texinfo_appendices = []

# If false, no module index is generated.

#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.

#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.

#texinfo_no_detailmenu = False

.. _usage:

Usage

=====

Majic Ansible Roles are targeted at sysadmins who wish to deploy services for

their own, small-scale use. This chapter gives a simple tutorial-like set of

instructions for using all of the roles available.

.. contents:: :local:

Overview

--------

There is a number of different roles that can prove useful for setting-up a

small infrastructure of your own.

Some roles are suited for one-off operations during installation, like the

``preseed`` and ``bootstrap``, while some are better suited for periodic runs

for maintaining the users and integrity of the system.

By the end of the instructions you will have the following:

* Ansible server, used for configuring the remaining servers.

* Communications server, providing the LDAP, mail, and XMPP services.

* Web server, providing the web services.

* Backup server, where the backups will be stored at.

Pre-requisites

--------------

For the set-up outlined in this usage guide you'll need the following:

* One server where Ansible will be installed at. Debian Jessie will be installed

  on top of this server. The server will be set-up manually (this is currently

  out of scope for the *Majic Ansible Roles* automated set-up).

* Three servers where the services will be set-up. All servers must be able to

  communicate over network with each-other, the Ansible servers, and with

  Internet. Debian Jessie will be installed on top of this server as part of the

  usage instructions.

* Debian Jessie network install CD.

* All servers should be on the same network.

* IP addresses for all servers should be known.

* Netmask for all servers should be known.

* Gateway for all servers should be known.

In case of the servers listed above, it might be safest to have them as VMs -

this is cheapest thing to do, and simplest (who wants to deal with pesky hardware anyway?).

Usage instructions assume the following:

* Domain used for all servers is ``example.com``. If you wish to use a different

  domain, adjust the instructions accordingly.

* Server hostnames are ``ansible``, ``comms``, ``www``, and ``bak`` (for Ansible

  server, communications server, web server, and backup server, respectively).

Installing the OS on Ansible server

-----------------------------------

Start-off by installing the operating system on the Ansible server:

1. Fire-up the ``ansible`` server, and boot from the network installation CD.

2. Select the **Install** option.

3. Pick **English** as language.

4. Pick the country you are living in (or whatever else you want).

5. Pick the **en_US.UTF-8** locale.

6. Pick the **American English** keymap.

7. Configure the network if necessary.

8. Set the hostname to ``ansible``.

9. Set the domain to ``example.com``.

10. Set the root password.

11. Create a new user. For simplicity, call the user **Ansible user**, with

    username **ansible**.

12. Set-up partitioning in any way you want. You can go for **Guided - use

    entire disk** if you want to keep it simple and are just testing things.

13. Wait until the base system has been installed.

14. Pick whatever Debian archive mirror is closest to you.

15. If you have an HTTP proxy, provide its URL.

16. Pick if you want to participate in package survey or not.

17. Make sure that at least the **standard system utilities** and **SSH server**

    options are selected on task selection screen.

18. Wait for packages to be installed.

19. Install the GRUB boot loader on MBR.

20. Finalise the server install, and remove the installation media from server.

Installing required packages

----------------------------

With the operating system installed, it is necessary to install a couple of

packages, and to prepare the environment a bit on the Ansible server:

1. Install the necessary system packages (using the ``root`` account)::

     apt-get install -y virtualenv virtualenvwrapper git python-pip python-dev libffi-dev libssl-dev

2. Set-up the virtual environment (using the ``ansible`` account):

   .. warning::

      If you are already logged-in as user ``ansible`` in the server, you will

      need to log-out and log-in again in order to be able to use

      ``virtualenvwrapper`` commands!

   ::

     mkdir ~/mysite/

     mkvirtualenv -a ~/mysite/ mysite

     pip install -U pip setuptools

     pip install 'ansible~=1.9.0' dnspython

.. warning::

   The ``dnspython`` package is important since it is used internally via

   ``dig`` lookup plugin.

Cloning the *Majic Ansible Roles*

---------------------------------

With most of the software pieces in place, the only missing thing is the Majic

Ansible Roles:

1. Clone the git repository::

     git clone http://code.majic.rs/majic-ansible-roles ~/majic-ansible-roles

2. Checkout the correct version of the roles::

     cd ~/majic-ansible-roles/

Preparing the basic site configuration

--------------------------------------

Phew... Now that was a bit tedious and boring... But at least you are now ready

to set-up your own site :)

First of all, let's set-up some basic directory structure and configuration:

1. Create Ansible configuration file.

   :file:`~/mysite/ansible.cfg`

   ::

     [defaults]

     roles_path=/home/ansible/majic-ansible-roles/roles:/home/ansible/mysite/roles

     force_handlers = True

     retry_files_save_path = /home/ansible/mysite/retry

     inventory = /home/ansible/mysite/hosts

2. Create directory where retry files will be stored at (so they woudln't

   pollute your home directory)::

     mkdir ~/mysite/retry

3. Create the hosts file.

   :file:`~/mysite/hosts`

   ::

     [preseed]

     localhost ansible_connection=local

     [communications]

     comms.example.com

     [web]

     www.example.com

     [backup]

     bak.example.com

4. Create a number of directories for storing playbooks, group variables, SSH

   keys, and GnuPG keyring (we'll get to this later)::

     mkdir ~/mysite/playbooks/

     mkdir ~/mysite/group_vars/

     mkdir ~/mysite/ssh/

     mkdir ~/mysite/gnupg/

5. Before moving ahead, we should also create SSH private/public key pair that

   will be used by Ansible for connecting to destination servers, as well as

   for some roles::

     ssh-keygen -f ~/.ssh/id_rsa -N ''

Preseed files

-------------

The ``preseed`` role is useful for generating Debian preseed files. Preseed

files can be used for automating the Debian installation process.

Preseed files are commonly created on the Ansible host, and then in some way

served to the servers using them during install.

So, let's set this up for start:

1. First of all, create the playbook for generating the preseed files locally.

   :file:`~/mysite/playbooks/preseed.yml`

   ::

      ---

      - hosts: preseed

        roles:

          - preseed

2. And that is about it to be able to actually use this particular role! So

   let's try running it::

     workon mysite && ansible-playbook playbooks/preseed.yml

3. If all went well, you should have the following files created:

   * :file:`~/mysite/preseed_files/comms.example.com.cfg`

   * :file:`~/mysite/preseed_files/www.example.com.cfg`

   * :file:`~/mysite/preseed_files/bak.example.com.cfg`

4. You can have a look at them, but you might notice the settings in the file

   might not be to your liking. In particular, it could be using wrong timezone,

   defaulting to DHCP for network configuration etc. Let's concentrate on making

   the network configuration changes - this is the main thing that will probably

   differ in your environment. Create a new configuration file:

   :file:`~/mysite/group_vars/preseed.yml`

   ::

      ---

      # Set your default (initial) root password.

      preseed_root_password: changeit

      # Use manual network configuration (no DHCP).

      preseed_network_auto: no

      # Set the gateway for all servers.

      preseed_gateway: 10.32.64.1

      # Set the netmask for all servers.

      preseed_netmask: 255.255.255.0

      # Set the DNS for all servers.

      preseed_dns: 10.32.64.1

      # Set the domain for all servers.

      preseed_domain: example.com

      # Set the server-specific options.

      preseed_server_overrides:

        comms.example.com:

          hostname: comms

          ip: 10.32.64.19

        www.example.com:

          hostname: www

          ip: 10.32.64.20

        bak.example.com:

          hostname: bak

          ip: 10.32.64.23

5. Now re-run the preseed playbook::

     workon mysite && ansible-playbook playbooks/preseed.yml

6. The preseed files should have been updated now, and you should have the new

   customised configuration files in the ``preseed_files`` directory. You can

   now use these to install the servers.

Installing the servers with preseed files

-----------------------------------------

You have your preseed files now, so you can go ahead and install the servers

``comms.example.com``, ``www.example.com``, and ``bak.example.com`` using

them with network install CD. Have a look at `Debian

<https://www.debian.org/releases/stable/amd64/apbs02.html.en>`_ instructions for

more details.

If you need to, you can easily serve the preseed files from the Ansible server

with Python's built-in HTTP server::

  cd ~/mysite/preseed_files/

  python -m SimpleHTTPServer 8000

Bootstrapping servers for Ansible set-up

----------------------------------------

In order to effectively use Ansible, a small initial bootstrap always has to be

done for managed servers. This mainly involves set-up of Ansible users on the

destination machine, and distributing the SSH public keys for authorisation.

When you use the preseed configuration files to deploy a server, you get the

benefit of having the authorized_keys set-up for the root operating system user,

making it easier to bootstrap the machines subsequently via Ansible.

Let's bootstrap our machines now:

1. For start, create a dedicated playbook for the bootstrap process.

   :file:`~/mysite/playbooks/bootstrap.yml`

   ::

      ---

      - hosts: [communications, web, backup]

        remote_user: root

        roles:

          - bootstrap

2. The ``bootstrap`` role has only one parameter - an SSH key which should be

   deployed for the Ansible user on managed server (in the ``authorized_keys``

   file). This defaults to content of local file ``~/.ssh/id_rsa.pub``, so no

   need to make any changes so far.

3. SSH into all machines at least once from the Ansible server in order to

   store the SSH fingerprints into known hosts file::

     ssh root@comms.example.com date

     ssh root@www.example.com date

     ssh root@bak.example.com date

4. Now, simply run the bootstrap role against the servers::

     workon mysite && ansible-playbook playbooks/bootstrap.yml

6. At this point you won't be able to ssh into the machines with root account

   anymore. You will be able to ssh into the machines using the private key of

   the ``ansible`` user (from the Ansible server). The ``ansible`` user will

   also be granted ability to run the ``sudo`` commands without providing

   password.

7. After this you can finally move on to configuring what you really want -

   common configuration and services for your site.

Common server configuration

---------------------------

Each server needs to share some common configuration in order to be functioning

properly. This includes set-up of some shared accounts, perhaps some hardening

etc.

Let's take care of this common configuration right away:

1. Create playbook for the communications server:

   :file:`~/mysite/playbooks/communications.yml`

   ::

      ---

      - hosts: communications

        remote_user: ansible

        become: yes

        roles:

          - common

2. Create playbook for the web server:

   :file:`~/mysite/playbooks/web.yml`

   ::

      ---

      - hosts: web

        remote_user: ansible

        become: yes

        roles:

          - common

3. Create playbook for the backup server:

   :file:`~/mysite/playbooks/backup.yml`

   ::

      ---

      - hosts: backup

        remote_user: ansible

        become: yes

        roles:

          - common

4. Create the global site playbook:

   :file:`~/mysite/playbooks/site.yml`

   ::

      ---

      - include: preseed.yml

      - include: communications.yml

      - include: web.yml

      - include: backup.yml

5. Time to create configuration for the role. Since this role is supposed to

   set-up a common base, we'll set-up the variables file that applies to all

   roles:

   :file:`~/mysite/group_vars/all.yml`

   ::

      ---

      os_users:

        - name: admin

          uid: 1000

          additional_groups:

            - sudo

          authorized_keys:

            - "{{ lookup('file', '~/.ssh/id_rsa.pub') }}"

          password: "{{ 'admin' | password_hash('sha512') }}"

      common_packages:

        - emacs24-nox

6. That's all for configuration, time to apply the changes::

     workon mysite && ansible-playbook playbooks/site.yml

7. After this you should be able to *ssh* from Ansible server onto the managed

   servers as user ``admin`` using the *SSH* private key. The ``admin`` user's

   password has also been set to ``admin``, and the user will be member of

   ``sudo`` group.

Introducing LDAP

----------------

Since some of the services actually depend on LDAP, we'll go ahead and set that

one up first. This includes both the LDAP *server* and *client* configuration.

1. Update the playbook for communications server to include the LDAP client and

   server roles (``ldap_client`` and ``ldap_server``, respectively).

   :file:`~/mysite/playbooks/communications.yml`

   ::

      ---

      - hosts: communications

        remote_user: ansible

        become: yes

        roles:

          - common

          - ldap_client

          - ldap_server

2. Update the playbook for web server to include the LDAP client role

   (``ldap_client``). You never know when it might come in handy :)

   :file:`~/mysite/playbooks/web.yml`

   ::

      ---

      - hosts: web

        remote_user: ansible

        become: yes

        roles:

          - common

          - ldap_client

3. Time to configure the roles. For start, let us configure the LDAP server

   role. Keep in mind that there is a lot of default variables set-up by the

   role itself, making our config rather short.

   :file:`~/mysite/group_vars/communications.yml`

   ::

      ---

      ldap_admin_password: admin

      ldap_server_organization: "Example Inc."

4. Phew. That was... Well, actually, easy :) Technically, only the LDAP admin

   password *must* be set, but it's nice to have better organisation specified

   than the default one. Let's add the LDAP client configuration next. We will

   start off with global LDAP client configuration. In case of LDAP client,

   we've got to be a bit more explicit.

   :file:`~/mysite/group_vars/all.yml`

   ::

      # Observe how we set the base DN. By default the ldap_server role

      # (defined up there) will use server's domain to form the base for LDAP.

      ldap_client_config:

        - comment: Set the base DN

          option: BASE

          value: dc=example,dc=com

        - comment: Set the default URI

          option: URI

          value: ldap://comms.example.com/

        - comment: Set the LDAP TLS truststore

          option: TLS_CACERT

          value: /etc/ssl/certs/truststore.pem

        - comment: Enforce TLS

          option: TLS_REQCERT

          value: demand

5. Ok, so this looks nice and dandy... But, let's have a bit better

   configuration on the communications server itself. Namely, on that one we

   should be able to connect to socket with LDAP clients instead over TCP port.

   :file:`~/mysite/group_vars/communications.yml`

   ::

      ldap_client_config:

        - comment: Set the base DN

          option: BASE

          value: dc=example,dc=com

        - comment: Set the default URI

          option: URI

          value: ldapi:///

        - comment: Set the default bind DN, useful for administration.

          option: BINDDN

          value: cn=admin,dc=example,dc=com

        - comment: Set the LDAP TLS truststore

          option: TLS_CACERT

          value: /etc/ssl/certs/truststore.pem

        - comment: Enforce TLS

          option: TLS_REQCERT

          value: demand

6. Ok, time to re-run the playbooks again... Wait a minute, something is missing

   here... Oh, right, forgot to mention one thing - Majic Ansible Roles use TLS

   throughout wherever possible. In other words, you *must* have TLS private

   keys and certificates issued by some CA for all servers in order to be able

   to use most of the roles (actually, you need them issued per *service*). This

   includes ``ldap_server`` too. So, let's make a slight detour to create a CA

   of our own, plus the necessary server certificate for the LDAP service...

   .. note::

      Another useful feature the roles implement is a check to see if

      certificates will expire within the next 30 days. This check is performed

      via cronjob at midnight, and results will end-up being delivered to the

      ``root`` user on local server. Later on, once you have configured the mail

      server, you should be able to set-up the necessary aliases to have the

      mails delivered to non-local accounts too.

   1. Let's first install a couple of more tools on the Ansible server, since we

      will be using ``certtool`` for our improvised CA needs (run this as

      ``root``)::

        apt-get install -y gnutls-bin

   2. Create directory where the private keys and certificates will be stored

      at (you can switch back to the ``ansible`` user now)::

        mkdir ~/mysite/tls/

   3. It is time to create a couple of templates for the ``certtool`` so it

      would know what extensions and content to have in the certificates:

      :file:`~/mysite/tls/ca.cfg`

      ::

         organization = "Example Inc."

         country = "SE"

         cn = "Example Inc. Test Site CA"

         expiration_days = 1825

         ca

         cert_signing_key

         crl_signing_key

      :file:`~/mysite/tls/comms.example.com_ldap.cfg`

      ::

         organization = "Example Inc."

         country = SE

         cn = "Exampe Inc. LDAP Server"

         expiration_days = 365

         dns_name = "comms.example.com"

         tls_www_server

         signing_key