From d885a66830da7fd873befd842a10c653511eebcd 2017-07-19 15:18:21 From: Branko Majic Date: 2017-07-19 15:18:21 Subject: [PATCH] MAR-110: Implemented script for running tests for all roles: - Updated gitignore file to ignore test reports. - Updated instructions for running tests. - Added Bash script that wraps around Molecule and allows output of reports into a directory. --- diff --git a/.gitignore b/.gitignore index 785a5dfa7358d3e79b18c56de52156f738080686..c316ba7632240ad7ac6c495a8f53bb385f01aa51 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,7 @@ testsite/backup_keyring/ # Ignore Molecule artefacts. .molecule .vagrant -.cache \ No newline at end of file +.cache + +# Ignore test report artefacts +test_report* \ No newline at end of file diff --git a/docs/development.rst b/docs/development.rst index 4d24a8d48252bdfa9a7a79dc9b50b4cccb70b91b..c8e79719effaebf56d163994f6664303c2947b04 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -57,8 +57,8 @@ performing the following steps: pip-sync -Running role tests ------------------- +Running role tests directly +--------------------------- Role tests are implemented using `Molecule `_, `Testinfra `_, `VirtualBox @@ -68,6 +68,10 @@ of Pyhton virtual environment, while *VirtualBox* and *Vagrant* need to be installed distribution-wide, following instructions outlined on their corresponding websites. +Tests can be run directly for a single role, or for one or more roles using a +dedicated shell script (see below). The shell script can also be used for +generating reports in an automated environment. + In order to run tests for a specific role, perform the following steps: 1. Switch to Python virtual environment:: @@ -91,6 +95,36 @@ In order to run tests for a specific role, perform the following steps: molecule test --platform debian-stretch64 +Running role tests via shell script +----------------------------------- + +In order to make it easier to run tests for all roles, and eventually produce +reports of such runs, a dedicated shell script is provided for running the +tests. + +In order to run tests, perform the following steps: + +1. Switch to Python virtual environment:: + + workon majic-ansible-roles + +2. Make sure you are within the root directory of Git repository. + +3. Run tests for all roles and generate report:: + + ./scripts/run_tests.sh -r all + + .. note:: + Optionally you can run tests for a specific set of roles, or without + generating the report, for example ``./scripts/run_tests.sh web_server + common`` + +4. Check results either from script output, or within directory + ``test_report-YYYY_MM_DD-hh_mm_ss``. For overview of what roles have failed, + have a look at ``summary.txt``. For details have a look at each role's + individual report. + + .. _testsite: Test Site diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh new file mode 100755 index 0000000000000000000000000000000000000000..edaa2a27a81fb918ebbc5b806721c1a15a5eb95a --- /dev/null +++ b/scripts/run_tests.sh @@ -0,0 +1,300 @@ +#!/bin/bash +# +# run_tests.sh +# +# Copyright (C) 2017, Branko Majic +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +program="run_tests.sh" + +function usage() { + cat <. +EOF +} + +function version() { + cat < | +| | +| This program is free software: you can redistribute it and/or modify | +| it under the terms of the GNU General Public License as published by | +| the Free Software Foundation, either version 3 of the License, or | +| (at your option) any later version. | +| | +| This program is distributed in the hope that it will be useful, | +| but WITHOUT ANY WARRANTY; without even the implied warranty of | +| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | +| GNU General Public License for more details. | +| | +| You should have received a copy of the GNU General Public License | +| along with this program. If not, see . | ++-----------------------------------------------------------------------+ + +EOF +} + +# Set-up colours for message printing if we're not piping and terminal is +# capable of outputting the colors. +_color_terminal=$(tput colors 2>&1) +if [[ -t 1 ]] && (( ${_color_terminal} > 0 )); then + _text_bold=$(tput bold) + _text_white=$(tput setaf 7) + _text_blue=$(tput setaf 6) + _text_green=$(tput setaf 2) + _text_yellow=$(tput setaf 3) + _text_red=$(tput setaf 1) + _text_reset=$(tput sgr0) +else + _text_bold="" + _text_white="" + _text_blue="" + _text_green="" + _text_yellow="" + _text_red="" + _text_reset="" +fi + +# Set-up functions for printing coloured messages. +function debug() { + if [[ $debug != 0 ]]; then + echo "${_text_bold}${_text_blue}[DEBUG]${_text_reset}" "$@" + fi +} + +function info() { + echo "${_text_bold}${_text_white}[INFO] ${_text_reset}" "$@" +} + +function success() { + echo "${_text_bold}${_text_green}[OK] ${_text_reset}" "$@" +} + +function warning() { + echo "${_text_bold}${_text_yellow}[WARN] ${_text_reset}" "$@" +} + +function error() { + echo "${_text_bold}${_text_red}[ERROR]${_text_reset}" "$@" >&2 +} + +# Define error codes. +SUCCESS=0 +ERROR_ARGUMENTS=1 +ERROR_NO_ROLES=2 +ERROR_MISSING_BINARY=3 +ERROR_FAILED_ROLES=4 +ERROR_REPORT=5 + +# Disable debug and quiet modes by default. +debug=0 +quiet=0 + +# Default values. +report=0 + +# If no arguments were given, just show usage help. +if [[ -z $1 ]]; then + usage + exit "$SUCCESS" +fi + +# Parse the arguments +while getopts "rqdvh" opt; do + case "$opt" in + r) report=1;; + q) quiet=1;; + d) debug=1;; + v) version + exit "$SUCCESS";; + h) usage + exit "$SUCCESS";; + *) usage + exit "$ERROR_ARGUMENTS";; + esac +done +i=$OPTIND +shift $(($i-1)) + +# Test if the necessary binaries are available. +if ! type molecule > /dev/null 2>&1; then + error "Could not locate binary: molecule. Please ensure you are running the script from within correctly set-up Python virtual environment." + exit $ERROR_MISSING_BINARY +fi + +# Assume success. +test_result=0 + +# Assemble list of roles to run against. +roles=() +roles_to_test=() +roles_to_skip=() + +# Assemble list of roles to test. +if [[ $1 == "all" ]]; then + debug "Testing of all roles was requested, assembling role list." + + # Traverse directory. + for role_dir in roles/*; do + if [[ -d $role_dir ]]; then + debug "Located candiate role in directory $role_dir" + roles+=("${role_dir#roles/}") + else + debug "Ignoring non-directory $role_dir" + fi + done +else + while [[ -n $1 ]]; do + roles+=($1) + shift 1 + done +fi + +# Determine which roles have available tests. +for role in "${roles[@]}"; do + role_dir="roles/$role" + if [[ ! -d "roles/$role" ]]; then + warning "Could not locate role $role in directory $role_dir" + roles_to_skip+=("$role") + elif [[ -f "$role_dir/molecule.yml" ]]; then + debug "Role $role contains Molecule configuration." + roles_to_test+=("$role") + else + warning "Role $role cannot be tested - missing Molecule configuration." + roles_to_skip+=("$role") + fi +done + +# Output some helpful info, and ensure we can actually run tests against +# something. +info "Testing requested for roles: ${roles[@]}" +[[ ${#roles_to_skip[@]} != 0 ]] && info "The following roles will not be tested: ${roles_to_skip[@]}" +if [[ ${#roles_to_test[@]} == 0 ]]; then + error "No roles can be tested." + exit $ERROR_NO_ROLES +fi +info "The following roles will be tested: ${roles_to_test[@]}" + +# Prepare directory for storing reports. +if [[ $report == 1 ]]; then + report_directory="$(pwd)/test_report-$(date +%Y_%m_%d-%H_%M_%S)" + report_summary="${report_directory}/summary.txt" + if ! mkdir "$report_directory"; then + error "Failed to create report directory $report_directory" + exit $ERROR_REPORT + fi + touch "$report_summary" + debug "Created report directory $report_directory." + + # Output skipped roles to summary immediatelly. + for role in "${roles_to_skip[@]}"; do + echo "[SKIP] $role" >> "$report_summary" + done +else + debug "No report directory will be created." +fi + +# Test the roles. +passed_roles=() +failed_roles=() +work_dir="$(pwd)" +for role in "${roles_to_test[@]}"; do + + # Calculate directories between which we need to move. + role_dir="$work_dir/roles/$role" + + # Run tests. + cd "$role_dir" + for platform in $(molecule status --platforms --porcelain | cut -f 1 -d ' '); do + + if [[ $report == 1 ]]; then + report_file="$report_directory/role-${role}-${platform}.txt" + else + report_file="/dev/null" + fi + + info "Running tests for: $role ($platform)" + molecule test --destroy always --platform "$platform" 2>&1 | tee "$report_file" + + # Determine result. + if [[ "${PIPESTATUS[0]}" == 0 ]]; then + passed_roles+=("$role ($platform)") + + # Log failure in summary if requested. + [[ $report == 1 ]] && echo "[PASS] $role ($platform)" >> "$report_summary" + else + test_result=$ERROR_FAILED_ROLES + failed_roles+=("$role ($platform)") + + # Log failure in summary if requested. + [[ $report == 1 ]] && echo "[FAIL] $role ($platform)" >> "$report_summary" + fi + + # Make sure the instances have been cleaned-up to avoid any error. + molecule destroy + + done +done + +for role in "${roles_to_skip[@]}"; do + warning "SKIP: $role" +done + +for role in "${passed_roles[@]}"; do + success "PASS: $role" +done + +for role in "${failed_roles[@]}"; do + error "FAIL: $role" +done + +exit $test_result