Changeset - da53a3aa5208
[Not reviewed]
0 1 0
Branko Majic (branko) - 2 months ago 2024-09-09 23:01:48
branko@majic.rs
MAR-218: Fix the test run logic in test runner script:

- Linter tests should _always_ be run.
- Test report files should be appended to when using tee.
- Get rid of excess molecule destroy (probably not required any
longer). This will reduce the output and speed up things a little
bit.
1 file changed with 24 insertions and 21 deletions:
0 comments (0 inline, 0 general)
scripts/run_tests.sh
Show inline comments
 
#!/bin/bash
 
#
 
# run_tests.sh
 
#
 
# Copyright (C) 2017, Branko Majic <branko@majic.rs>
 
#
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
#
 

	
 
program="run_tests.sh"
 

	
 
function usage() {
 
    cat <<EOF
 
$program, a non-interactive utility for running role tests for Majic Ansible
 
Roles
 

	
 
Usage: $program [OPTIONS] (all|role1 ... roleN)
 

	
 
$program is a non-interactive utility for running role tests for Majic Ansible
 
Roles.
 

	
 
The script iterates over all roles within sub-directory "roles" (relative to
 
working directory) that have a valid Molecule configuration, runs the tests, and
 
optionally writes report for all tested roles (see option -r).
 

	
 
Only the default scenario is tested at the moment.
 

	
 
$program accepts the following options:
 

	
 
    -r
 
        Generate report for all roles under sub-directory of current
 
        directory "test_report-YYYY_MM_DD-hh_mm". Within the
 
        sub-directory each role will have its own file named
 
        "role-ROLENAME.txt". In addition, a summary file (summary.txt)
 
        is produced with overview of which roles have passed, failed,
 
        and which roles were skipped.
 

	
 
    -l all|lint
 
        Limit what type of tests should be run. Currently supported
 
        values are:
 

	
 
        all
 
            Runs all the available tests. This is the default.
 

	
 
        lint
 
            Runs only the linting tests. This is useful for performing
 
            a quick check on syntax prior to running more extensive
 
            testing.
 

	
 
    -d
 
        Enable debug mode.
 
    -v
 
        Show script licensing information.
 
    -h
 
        Show usage help.
 

	
 

	
 
Please report bugs and send feature requests to <branko@majic.rs>.
 
EOF
 
}
 

	
 
function version() {
 
    cat <<EOF
 
$program
 

	
 
+-----------------------------------------------------------------------+
 
| Copyright (C) 2017, Branko Majic <branko@majic.rs>                    |
 
|                                                                       |
 
| This program is free software: you can redistribute it and/or modify  |
 
| it under the terms of the GNU General Public License as published by  |
 
| the Free Software Foundation, either version 3 of the License, or     |
 
| (at your option) any later version.                                   |
 
|                                                                       |
 
| This program is distributed in the hope that it will be useful,       |
 
| but WITHOUT ANY WARRANTY; without even the implied warranty of        |
 
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         |
 
| GNU General Public License for more details.                          |
 
|                                                                       |
 
| You should have received a copy of the GNU General Public License     |
 
| along with this program.  If not, see <http://www.gnu.org/licenses/>. |
 
+-----------------------------------------------------------------------+
 

	
 
EOF
 
}
 

	
 
# Set-up colours for message printing if we're not piping and terminal is
 
# capable of outputting the colors.
 
_color_terminal=$(tput colors 2>&1)
 
if [[ -t 1 ]] && (( _color_terminal > 0 )); then
 
    _text_bold=$(tput bold)
 
    _text_white=$(tput setaf 7)
 
    _text_blue=$(tput setaf 6)
 
    _text_green=$(tput setaf 2)
 
    _text_yellow=$(tput setaf 3)
 
    _text_red=$(tput setaf 1)
 
    _text_reset=$(tput sgr0)
 
else
 
    _text_bold=""
 
    _text_white=""
 
    _text_blue=""
 
    _text_green=""
 
    _text_yellow=""
 
    _text_red=""
 
    _text_reset=""
 
fi
 

	
 
# Set-up functions for printing coloured messages.
 
function debug() {
 
    if [[ $debug != 0 ]]; then
 
        echo "${_text_bold}${_text_blue}[DEBUG]${_text_reset}" "$@"
 
    fi
 
}
 

	
 
function info() {
 
    echo "${_text_bold}${_text_white}[INFO] ${_text_reset}" "$@"
 
}
 

	
 
function success() {
 
    echo "${_text_bold}${_text_green}[OK]   ${_text_reset}" "$@"
 
}
 

	
 
function warning() {
 
    echo "${_text_bold}${_text_yellow}[WARN] ${_text_reset}" "$@"
 
}
 

	
 
function error() {
 
    echo "${_text_bold}${_text_red}[ERROR]${_text_reset}" "$@" >&2
 
}
 

	
 
# Define error codes.
 
SUCCESS=0
 
ERROR_ARGUMENTS=1
 
ERROR_NO_ROLES=2
 
ERROR_MISSING_BINARY=3
 
ERROR_FAILED_ROLES=4
 
ERROR_REPORT=5
 

	
 
# Disable debug mode by default.
 
debug=0
 

	
 
# Default values.
 
report=0
 
tests=all
 

	
 
# If no arguments were given, just show usage help.
 
if [[ -z $1 ]]; then
 
    usage
 
    exit "$SUCCESS"
 
fi
 

	
 
# Parse the arguments
 
while getopts "rl:qdvh" opt; do
 
    case "$opt" in
 
        r) report=1;;
 
        l) tests="$OPTARG";;
 
	d) debug=1;;
 
        v) version
 
           exit "$SUCCESS";;
 
        h) usage
 
           exit "$SUCCESS";;
 
        *) usage
 
           exit "$ERROR_ARGUMENTS";;
 
    esac
 
done
 
i=$OPTIND
 
shift $(( i-1 ))
 

	
 
# Test if the necessary binaries are available.
 
if ! type molecule > /dev/null 2>&1; then
 
    error "Could not locate binary: molecule. Please ensure you are running the script from within correctly set-up Python virtual environment."
 
    exit $ERROR_MISSING_BINARY
 
fi
 

	
 
# Assume success.
 
test_result=0
 

	
 
# Verify arguments.
 
if [[ $tests != "all" && $tests != "lint" ]]; then
 
    error "Unsupported type of tests requested: $tests"
 
    exit $ERROR_ARGUMENTS
 
fi
 

	
 
# Assemble list of roles to run against.
 
roles=()
 
roles_to_test=()
 
roles_to_skip=()
 

	
 
# Assemble list of roles to test.
 
if [[ $1 == "all" ]]; then
 
    debug "Testing of all roles was requested, assembling role list."
 

	
 
    # Traverse directory.
 
    for role_dir in roles/*; do
 
        if [[ -d $role_dir ]]; then
 
            debug "Located candiate role in directory $role_dir"
 
            roles+=("${role_dir#roles/}")
 
        else
 
            debug "Ignoring non-directory $role_dir"
 
        fi
 
    done
 
else
 
    while [[ -n $1 ]]; do
 
        roles+=("$1")
 
        shift 1
 
    done
 
fi
 

	
 
if [[ ${#roles[@]} == 0 ]]; then
 
    error "No role has been specified for testing."
 
    exit $ERROR_ARGUMENTS
 
fi
 

	
 
# Determine which roles have available tests.
 
for role in "${roles[@]}"; do
 
    role_dir="roles/$role"
 
    if [[ ! -d "roles/$role" ]]; then
 
        warning "Could not locate role $role in directory $role_dir"
 
        roles_to_skip+=("$role")
 
    elif [[ -f "$role_dir/molecule/default/molecule.yml" ]]; then
 
        debug "Role $role contains Molecule configuration."
 
         roles_to_test+=("$role")
 
    else
 
        warning "Role $role cannot be tested - missing Molecule configuration."
 
        roles_to_skip+=("$role")
 
    fi
 
done
 

	
 
# Output some helpful info, and ensure we can actually run tests against
 
# something.
 
info "Testing requested for roles: ${roles[*]}"
 
[[ ${#roles_to_skip[@]} != 0 ]] && info "The following roles will not be tested: ${roles_to_skip[*]}"
 
if [[ ${#roles_to_test[@]} == 0 ]]; then
 
    error "No roles can be tested."
 
    exit $ERROR_NO_ROLES
 
fi
 
info "The following roles will be tested: ${roles_to_test[*]}"
 

	
 
# Prepare directory for storing reports.
 
if [[ $report == 1 ]]; then
 
    report_directory="$(pwd)/test_report-$(date +%Y_%m_%d-%H_%M_%S)"
 
    report_summary="${report_directory}/summary.txt"
 
    if ! mkdir "$report_directory"; then
 
        error "Failed to create report directory $report_directory"
 
        exit $ERROR_REPORT
 
    fi
 
    touch "$report_summary"
 
    debug "Created report directory $report_directory."
 

	
 
    # Output skipped roles to summary immediatelly.
 
    for role in "${roles_to_skip[@]}"; do
 
        echo "[SKIP] $role" >> "$report_summary"
 
    done
 
else
 
    debug "No report directory will be created."
 
fi
 

	
 
# Test the roles.
 
passed_roles=()
 
failed_roles=()
 
work_dir="$(pwd)"
 
for role in "${roles_to_test[@]}"; do
 

	
 
    # Calculate directories between which we need to move.
 
    role_dir="$work_dir/roles/$role"
 

	
 
    # Run tests.
 
    cd "$role_dir"
 

	
 
    if [[ $report == 1 ]]; then
 
        report_file="$report_directory/role-${role}.txt"
 
    else
 
        report_file="/dev/null"
 
    fi
 

	
 
    info "Running tests for: $role"
 
    molecule_result=0
 
    if [[ $tests == all ]]; then
 

	
 
        molecule test --destroy always 2>&1 | tee "$report_file"
 
    role_test_result=0
 

	
 
    # Switch directory, make sure it is successful.
 
    # shellcheck disable=SC2164  # failed directory switch is dealt with additional checks below.
 
    cd "$role_dir" 2>&1 | tee -a "$report_file"
 
    last_pipe_status="${PIPESTATUS[0]}"
 
        [[ $last_pipe_status == 0 ]] || molecule_result="$last_pipe_status"
 
    [[ $last_pipe_status == 0 ]] || role_test_result="$last_pipe_status"
 

	
 
    elif [[ $tests == lint ]]; then
 
    # Run the linters.
 
    if [[ $role_test_result == 0 ]]; then
 

	
 
        flake8 . 2>&1 | tee "$report_file"
 
        flake8 . 2>&1 | tee -a "$report_file"
 
        last_pipe_status="${PIPESTATUS[0]}"
 
        [[ $last_pipe_status == 0 ]] || molecule_result="$last_pipe_status"
 
        [[ $last_pipe_status == 0 ]] || role_test_result="$last_pipe_status"
 

	
 
        yamllint . 2>&1 | tee "$report_file"
 
        yamllint . 2>&1 | tee -a "$report_file"
 
        last_pipe_status="${PIPESTATUS[0]}"
 
        [[ $last_pipe_status == 0 ]] || molecule_result="$last_pipe_status"
 
        [[ $last_pipe_status == 0 ]] || role_test_result="$last_pipe_status"
 

	
 
        ansible-lint --quiet . 2>&1 | tee "$report_file"
 
        ansible-lint -q . 2>&1 | tee -a "$report_file"
 
        last_pipe_status="${PIPESTATUS[0]}"
 
        [[ $last_pipe_status == 0 ]] || molecule_result="$last_pipe_status"
 
        [[ $last_pipe_status == 0 ]] || role_test_result="$last_pipe_status"
 
    fi
 

	
 
    # Run the unit/functional tests (if requested).
 
    if [[ $role_test_result == 0 && $tests == all ]]; then
 

	
 
        molecule test --destroy always 2>&1 | tee -a "$report_file"
 
        last_pipe_status="${PIPESTATUS[0]}"
 
        [[ $last_pipe_status == 0 ]] || role_test_result="$last_pipe_status"
 

	
 
    fi
 

	
 
    # Determine result.
 
    if [[ $molecule_result == 0 ]]; then
 
    if [[ $role_test_result == 0 ]]; then
 
        passed_roles+=("$role")
 

	
 
        # Log failure in summary if requested.
 
        # Log success in summary if requested.
 
        [[ $report == 1 ]] && echo "[PASS] $role" >> "$report_summary"
 
    else
 
        test_result=$ERROR_FAILED_ROLES
 
        failed_roles+=("$role")
 

	
 
        # Log failure in summary if requested.
 
        [[ $report == 1 ]] && echo "[FAIL] $role" >> "$report_summary"
 
    fi
 

	
 
    # Make sure the instances have been cleaned-up to avoid errors.
 
    if [[ $tests == all ]]; then
 
        molecule destroy
 
    fi
 

	
 
done
 

	
 
for role in "${roles_to_skip[@]}"; do
 
    warning "SKIP: $role"
 
done
 

	
 
for role in "${passed_roles[@]}"; do
 
    success "PASS: $role"
 
done
 

	
 
for role in "${failed_roles[@]}"; do
 
    error "FAIL: $role"
 
done
 

	
 
exit $test_result
0 comments (0 inline, 0 general)