Changeset - da53a3aa5208
[Not reviewed]
0 1 0
Branko Majic (branko) - 18 days ago 2024-09-09 23:01:48
branko@majic.rs
MAR-218: Fix the test run logic in test runner script:

- Linter tests should _always_ be run.
- Test report files should be appended to when using tee.
- Get rid of excess molecule destroy (probably not required any
longer). This will reduce the output and speed up things a little
bit.
1 file changed with 24 insertions and 21 deletions:
0 comments (0 inline, 0 general)
scripts/run_tests.sh
Show inline comments
 
@@ -181,163 +181,166 @@ if ! type molecule > /dev/null 2>&1; then
 
    error "Could not locate binary: molecule. Please ensure you are running the script from within correctly set-up Python virtual environment."
 
    exit $ERROR_MISSING_BINARY
 
fi
 

	
 
# Assume success.
 
test_result=0
 

	
 
# Verify arguments.
 
if [[ $tests != "all" && $tests != "lint" ]]; then
 
    error "Unsupported type of tests requested: $tests"
 
    exit $ERROR_ARGUMENTS
 
fi
 

	
 
# Assemble list of roles to run against.
 
roles=()
 
roles_to_test=()
 
roles_to_skip=()
 

	
 
# Assemble list of roles to test.
 
if [[ $1 == "all" ]]; then
 
    debug "Testing of all roles was requested, assembling role list."
 

	
 
    # Traverse directory.
 
    for role_dir in roles/*; do
 
        if [[ -d $role_dir ]]; then
 
            debug "Located candiate role in directory $role_dir"
 
            roles+=("${role_dir#roles/}")
 
        else
 
            debug "Ignoring non-directory $role_dir"
 
        fi
 
    done
 
else
 
    while [[ -n $1 ]]; do
 
        roles+=("$1")
 
        shift 1
 
    done
 
fi
 

	
 
if [[ ${#roles[@]} == 0 ]]; then
 
    error "No role has been specified for testing."
 
    exit $ERROR_ARGUMENTS
 
fi
 

	
 
# Determine which roles have available tests.
 
for role in "${roles[@]}"; do
 
    role_dir="roles/$role"
 
    if [[ ! -d "roles/$role" ]]; then
 
        warning "Could not locate role $role in directory $role_dir"
 
        roles_to_skip+=("$role")
 
    elif [[ -f "$role_dir/molecule/default/molecule.yml" ]]; then
 
        debug "Role $role contains Molecule configuration."
 
         roles_to_test+=("$role")
 
    else
 
        warning "Role $role cannot be tested - missing Molecule configuration."
 
        roles_to_skip+=("$role")
 
    fi
 
done
 

	
 
# Output some helpful info, and ensure we can actually run tests against
 
# something.
 
info "Testing requested for roles: ${roles[*]}"
 
[[ ${#roles_to_skip[@]} != 0 ]] && info "The following roles will not be tested: ${roles_to_skip[*]}"
 
if [[ ${#roles_to_test[@]} == 0 ]]; then
 
    error "No roles can be tested."
 
    exit $ERROR_NO_ROLES
 
fi
 
info "The following roles will be tested: ${roles_to_test[*]}"
 

	
 
# Prepare directory for storing reports.
 
if [[ $report == 1 ]]; then
 
    report_directory="$(pwd)/test_report-$(date +%Y_%m_%d-%H_%M_%S)"
 
    report_summary="${report_directory}/summary.txt"
 
    if ! mkdir "$report_directory"; then
 
        error "Failed to create report directory $report_directory"
 
        exit $ERROR_REPORT
 
    fi
 
    touch "$report_summary"
 
    debug "Created report directory $report_directory."
 

	
 
    # Output skipped roles to summary immediatelly.
 
    for role in "${roles_to_skip[@]}"; do
 
        echo "[SKIP] $role" >> "$report_summary"
 
    done
 
else
 
    debug "No report directory will be created."
 
fi
 

	
 
# Test the roles.
 
passed_roles=()
 
failed_roles=()
 
work_dir="$(pwd)"
 
for role in "${roles_to_test[@]}"; do
 

	
 
    # Calculate directories between which we need to move.
 
    role_dir="$work_dir/roles/$role"
 

	
 
    # Run tests.
 
    cd "$role_dir"
 

	
 
    if [[ $report == 1 ]]; then
 
        report_file="$report_directory/role-${role}.txt"
 
    else
 
        report_file="/dev/null"
 
    fi
 

	
 
    info "Running tests for: $role"
 
    molecule_result=0
 
    if [[ $tests == all ]]; then
 

	
 
        molecule test --destroy always 2>&1 | tee "$report_file"
 
    role_test_result=0
 

	
 
    # Switch directory, make sure it is successful.
 
    # shellcheck disable=SC2164  # failed directory switch is dealt with additional checks below.
 
    cd "$role_dir" 2>&1 | tee -a "$report_file"
 
    last_pipe_status="${PIPESTATUS[0]}"
 
        [[ $last_pipe_status == 0 ]] || molecule_result="$last_pipe_status"
 
    [[ $last_pipe_status == 0 ]] || role_test_result="$last_pipe_status"
 

	
 
    elif [[ $tests == lint ]]; then
 
    # Run the linters.
 
    if [[ $role_test_result == 0 ]]; then
 

	
 
        flake8 . 2>&1 | tee "$report_file"
 
        flake8 . 2>&1 | tee -a "$report_file"
 
        last_pipe_status="${PIPESTATUS[0]}"
 
        [[ $last_pipe_status == 0 ]] || molecule_result="$last_pipe_status"
 
        [[ $last_pipe_status == 0 ]] || role_test_result="$last_pipe_status"
 

	
 
        yamllint . 2>&1 | tee "$report_file"
 
        yamllint . 2>&1 | tee -a "$report_file"
 
        last_pipe_status="${PIPESTATUS[0]}"
 
        [[ $last_pipe_status == 0 ]] || molecule_result="$last_pipe_status"
 
        [[ $last_pipe_status == 0 ]] || role_test_result="$last_pipe_status"
 

	
 
        ansible-lint --quiet . 2>&1 | tee "$report_file"
 
        ansible-lint -q . 2>&1 | tee -a "$report_file"
 
        last_pipe_status="${PIPESTATUS[0]}"
 
        [[ $last_pipe_status == 0 ]] || molecule_result="$last_pipe_status"
 
        [[ $last_pipe_status == 0 ]] || role_test_result="$last_pipe_status"
 
    fi
 

	
 
    # Run the unit/functional tests (if requested).
 
    if [[ $role_test_result == 0 && $tests == all ]]; then
 

	
 
        molecule test --destroy always 2>&1 | tee -a "$report_file"
 
        last_pipe_status="${PIPESTATUS[0]}"
 
        [[ $last_pipe_status == 0 ]] || role_test_result="$last_pipe_status"
 

	
 
    fi
 

	
 
    # Determine result.
 
    if [[ $molecule_result == 0 ]]; then
 
    if [[ $role_test_result == 0 ]]; then
 
        passed_roles+=("$role")
 

	
 
        # Log failure in summary if requested.
 
        # Log success in summary if requested.
 
        [[ $report == 1 ]] && echo "[PASS] $role" >> "$report_summary"
 
    else
 
        test_result=$ERROR_FAILED_ROLES
 
        failed_roles+=("$role")
 

	
 
        # Log failure in summary if requested.
 
        [[ $report == 1 ]] && echo "[FAIL] $role" >> "$report_summary"
 
    fi
 

	
 
    # Make sure the instances have been cleaned-up to avoid errors.
 
    if [[ $tests == all ]]; then
 
        molecule destroy
 
    fi
 

	
 
done
 

	
 
for role in "${roles_to_skip[@]}"; do
 
    warning "SKIP: $role"
 
done
 

	
 
for role in "${passed_roles[@]}"; do
 
    success "PASS: $role"
 
done
 

	
 
for role in "${failed_roles[@]}"; do
 
    error "FAIL: $role"
 
done
 

	
 
exit $test_result
0 comments (0 inline, 0 general)