diff --git a/.gitignore b/.gitignore
index 785a5dfa7358d3e79b18c56de52156f738080686..c316ba7632240ad7ac6c495a8f53bb385f01aa51 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,4 +19,7 @@ testsite/backup_keyring/
# Ignore Molecule artefacts.
.molecule
.vagrant
-.cache
\ No newline at end of file
+.cache
+
+# Ignore test report artefacts
+test_report*
\ No newline at end of file
diff --git a/docs/development.rst b/docs/development.rst
index 4d24a8d48252bdfa9a7a79dc9b50b4cccb70b91b..c8e79719effaebf56d163994f6664303c2947b04 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -57,8 +57,8 @@ performing the following steps:
pip-sync
-Running role tests
-------------------
+Running role tests directly
+---------------------------
Role tests are implemented using `Molecule `_,
`Testinfra `_, `VirtualBox
@@ -68,6 +68,10 @@ of Pyhton virtual environment, while *VirtualBox* and *Vagrant* need to be
installed distribution-wide, following instructions outlined on their
corresponding websites.
+Tests can be run directly for a single role, or for one or more roles using a
+dedicated shell script (see below). The shell script can also be used for
+generating reports in an automated environment.
+
In order to run tests for a specific role, perform the following steps:
1. Switch to Python virtual environment::
@@ -91,6 +95,36 @@ In order to run tests for a specific role, perform the following steps:
molecule test --platform debian-stretch64
+Running role tests via shell script
+-----------------------------------
+
+In order to make it easier to run tests for all roles, and eventually produce
+reports of such runs, a dedicated shell script is provided for running the
+tests.
+
+In order to run tests, perform the following steps:
+
+1. Switch to Python virtual environment::
+
+ workon majic-ansible-roles
+
+2. Make sure you are within the root directory of Git repository.
+
+3. Run tests for all roles and generate report::
+
+ ./scripts/run_tests.sh -r all
+
+ .. note::
+ Optionally you can run tests for a specific set of roles, or without
+ generating the report, for example ``./scripts/run_tests.sh web_server
+ common``
+
+4. Check results either from script output, or within directory
+ ``test_report-YYYY_MM_DD-hh_mm_ss``. For overview of what roles have failed,
+ have a look at ``summary.txt``. For details have a look at each role's
+ individual report.
+
+
.. _testsite:
Test Site
diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh
new file mode 100755
index 0000000000000000000000000000000000000000..edaa2a27a81fb918ebbc5b806721c1a15a5eb95a
--- /dev/null
+++ b/scripts/run_tests.sh
@@ -0,0 +1,300 @@
+#!/bin/bash
+#
+# run_tests.sh
+#
+# Copyright (C) 2017, Branko Majic
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+
+program="run_tests.sh"
+
+function usage() {
+ cat <.
+EOF
+}
+
+function version() {
+ cat < |
+| |
+| This program is free software: you can redistribute it and/or modify |
+| it under the terms of the GNU General Public License as published by |
+| the Free Software Foundation, either version 3 of the License, or |
+| (at your option) any later version. |
+| |
+| This program is distributed in the hope that it will be useful, |
+| but WITHOUT ANY WARRANTY; without even the implied warranty of |
+| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
+| GNU General Public License for more details. |
+| |
+| You should have received a copy of the GNU General Public License |
+| along with this program. If not, see . |
++-----------------------------------------------------------------------+
+
+EOF
+}
+
+# Set-up colours for message printing if we're not piping and terminal is
+# capable of outputting the colors.
+_color_terminal=$(tput colors 2>&1)
+if [[ -t 1 ]] && (( ${_color_terminal} > 0 )); then
+ _text_bold=$(tput bold)
+ _text_white=$(tput setaf 7)
+ _text_blue=$(tput setaf 6)
+ _text_green=$(tput setaf 2)
+ _text_yellow=$(tput setaf 3)
+ _text_red=$(tput setaf 1)
+ _text_reset=$(tput sgr0)
+else
+ _text_bold=""
+ _text_white=""
+ _text_blue=""
+ _text_green=""
+ _text_yellow=""
+ _text_red=""
+ _text_reset=""
+fi
+
+# Set-up functions for printing coloured messages.
+function debug() {
+ if [[ $debug != 0 ]]; then
+ echo "${_text_bold}${_text_blue}[DEBUG]${_text_reset}" "$@"
+ fi
+}
+
+function info() {
+ echo "${_text_bold}${_text_white}[INFO] ${_text_reset}" "$@"
+}
+
+function success() {
+ echo "${_text_bold}${_text_green}[OK] ${_text_reset}" "$@"
+}
+
+function warning() {
+ echo "${_text_bold}${_text_yellow}[WARN] ${_text_reset}" "$@"
+}
+
+function error() {
+ echo "${_text_bold}${_text_red}[ERROR]${_text_reset}" "$@" >&2
+}
+
+# Define error codes.
+SUCCESS=0
+ERROR_ARGUMENTS=1
+ERROR_NO_ROLES=2
+ERROR_MISSING_BINARY=3
+ERROR_FAILED_ROLES=4
+ERROR_REPORT=5
+
+# Disable debug and quiet modes by default.
+debug=0
+quiet=0
+
+# Default values.
+report=0
+
+# If no arguments were given, just show usage help.
+if [[ -z $1 ]]; then
+ usage
+ exit "$SUCCESS"
+fi
+
+# Parse the arguments
+while getopts "rqdvh" opt; do
+ case "$opt" in
+ r) report=1;;
+ q) quiet=1;;
+ d) debug=1;;
+ v) version
+ exit "$SUCCESS";;
+ h) usage
+ exit "$SUCCESS";;
+ *) usage
+ exit "$ERROR_ARGUMENTS";;
+ esac
+done
+i=$OPTIND
+shift $(($i-1))
+
+# Test if the necessary binaries are available.
+if ! type molecule > /dev/null 2>&1; then
+ error "Could not locate binary: molecule. Please ensure you are running the script from within correctly set-up Python virtual environment."
+ exit $ERROR_MISSING_BINARY
+fi
+
+# Assume success.
+test_result=0
+
+# Assemble list of roles to run against.
+roles=()
+roles_to_test=()
+roles_to_skip=()
+
+# Assemble list of roles to test.
+if [[ $1 == "all" ]]; then
+ debug "Testing of all roles was requested, assembling role list."
+
+ # Traverse directory.
+ for role_dir in roles/*; do
+ if [[ -d $role_dir ]]; then
+ debug "Located candiate role in directory $role_dir"
+ roles+=("${role_dir#roles/}")
+ else
+ debug "Ignoring non-directory $role_dir"
+ fi
+ done
+else
+ while [[ -n $1 ]]; do
+ roles+=($1)
+ shift 1
+ done
+fi
+
+# Determine which roles have available tests.
+for role in "${roles[@]}"; do
+ role_dir="roles/$role"
+ if [[ ! -d "roles/$role" ]]; then
+ warning "Could not locate role $role in directory $role_dir"
+ roles_to_skip+=("$role")
+ elif [[ -f "$role_dir/molecule.yml" ]]; then
+ debug "Role $role contains Molecule configuration."
+ roles_to_test+=("$role")
+ else
+ warning "Role $role cannot be tested - missing Molecule configuration."
+ roles_to_skip+=("$role")
+ fi
+done
+
+# Output some helpful info, and ensure we can actually run tests against
+# something.
+info "Testing requested for roles: ${roles[@]}"
+[[ ${#roles_to_skip[@]} != 0 ]] && info "The following roles will not be tested: ${roles_to_skip[@]}"
+if [[ ${#roles_to_test[@]} == 0 ]]; then
+ error "No roles can be tested."
+ exit $ERROR_NO_ROLES
+fi
+info "The following roles will be tested: ${roles_to_test[@]}"
+
+# Prepare directory for storing reports.
+if [[ $report == 1 ]]; then
+ report_directory="$(pwd)/test_report-$(date +%Y_%m_%d-%H_%M_%S)"
+ report_summary="${report_directory}/summary.txt"
+ if ! mkdir "$report_directory"; then
+ error "Failed to create report directory $report_directory"
+ exit $ERROR_REPORT
+ fi
+ touch "$report_summary"
+ debug "Created report directory $report_directory."
+
+ # Output skipped roles to summary immediatelly.
+ for role in "${roles_to_skip[@]}"; do
+ echo "[SKIP] $role" >> "$report_summary"
+ done
+else
+ debug "No report directory will be created."
+fi
+
+# Test the roles.
+passed_roles=()
+failed_roles=()
+work_dir="$(pwd)"
+for role in "${roles_to_test[@]}"; do
+
+ # Calculate directories between which we need to move.
+ role_dir="$work_dir/roles/$role"
+
+ # Run tests.
+ cd "$role_dir"
+ for platform in $(molecule status --platforms --porcelain | cut -f 1 -d ' '); do
+
+ if [[ $report == 1 ]]; then
+ report_file="$report_directory/role-${role}-${platform}.txt"
+ else
+ report_file="/dev/null"
+ fi
+
+ info "Running tests for: $role ($platform)"
+ molecule test --destroy always --platform "$platform" 2>&1 | tee "$report_file"
+
+ # Determine result.
+ if [[ "${PIPESTATUS[0]}" == 0 ]]; then
+ passed_roles+=("$role ($platform)")
+
+ # Log failure in summary if requested.
+ [[ $report == 1 ]] && echo "[PASS] $role ($platform)" >> "$report_summary"
+ else
+ test_result=$ERROR_FAILED_ROLES
+ failed_roles+=("$role ($platform)")
+
+ # Log failure in summary if requested.
+ [[ $report == 1 ]] && echo "[FAIL] $role ($platform)" >> "$report_summary"
+ fi
+
+ # Make sure the instances have been cleaned-up to avoid any error.
+ molecule destroy
+
+ done
+done
+
+for role in "${roles_to_skip[@]}"; do
+ warning "SKIP: $role"
+done
+
+for role in "${passed_roles[@]}"; do
+ success "PASS: $role"
+done
+
+for role in "${failed_roles[@]}"; do
+ error "FAIL: $role"
+done
+
+exit $test_result