diff options
| -rw-r--r-- | .codecov.yml | 2 | ||||
| -rw-r--r-- | .travis.yml | 33 | ||||
| -rwxr-xr-x | share/spack/qa/run-unit-tests | 30 | ||||
| -rwxr-xr-x | share/spack/qa/setup-env-test.sh | 289 | ||||
| -rwxr-xr-x | share/spack/qa/setup.sh | 18 |
5 files changed, 343 insertions, 29 deletions
diff --git a/.codecov.yml b/.codecov.yml index 042cc86e9c..c06cb57f88 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -11,5 +11,7 @@ ignore: - lib/spack/spack/test/.* - lib/spack/docs/.* - lib/spack/external/.* + - share/spack/qa/.* + - share/spack/spack-completion.bash comment: off diff --git a/.travis.yml b/.travis.yml index 91a95486e9..465232093a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,24 +50,6 @@ jobs: os: linux language: python env: [ TEST_SUITE=unit, COVERAGE=true ] - addons: - apt: - packages: - - cmake - - gfortran - - graphviz - - gnupg2 - - kcov - - mercurial - - ninja-build - - perl - - perl-base - - realpath - - patchelf - - r-base - - r-base-core - - r-base-dev - - python: '3.7' sudo: required os: linux @@ -159,6 +141,7 @@ addons: - r-base - r-base-core - r-base-dev + - zsh # for Mac builds, we use Homebrew homebrew: packages: @@ -166,6 +149,8 @@ addons: - gcc - gnupg2 - ccache + - dash + - kcov update: true # ~/.ccache needs to be cached directly as Travis is not taking care of it @@ -223,11 +208,13 @@ script: after_success: - ccache -s - - if [[ "$TEST_SUITE" == "unit" || "$TEST_SUITE" == "build" ]]; then - codecov --env PYTHON_VERSION - --required - --flags "${TEST_SUITE}${TRAVIS_OS_NAME}"; - fi + - case "$TEST_SUITE" in + unit) + codecov --env PYTHON_VERSION + --required + --flags "${TEST_SUITE}${TRAVIS_OS_NAME}"; + ;; + esac #============================================================================= # Notifications diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests index 49dbda0d32..4403d53a76 100755 --- a/share/spack/qa/run-unit-tests +++ b/share/spack/qa/run-unit-tests @@ -16,6 +16,12 @@ # Optionally add one or more unit tests # to only run these tests. # + +#----------------------------------------------------------- +# Run a few initial commands and set up test environment +#----------------------------------------------------------- +ORIGINAL_PATH="$PATH" + . "$(dirname $0)/setup.sh" check_dependencies ${coverage} git hg svn @@ -33,9 +39,33 @@ bin/spack help -a # Profile and print top 20 lines for a simple call to spack spec bin/spack -p --lines 20 spec mpileaks%gcc ^elfutils@0.170 +#----------------------------------------------------------- # Run unit tests with code coverage +#----------------------------------------------------------- extra_args="" if [[ -n "$@" ]]; then extra_args="-k $@" fi ${coverage_run} bin/spack test --verbose "$extra_args" + +#----------------------------------------------------------- +# Run tests for setup-env.sh +#----------------------------------------------------------- +# Clean the environment by removing Spack from the path and getting rid of +# the spack shell function +export PATH="$ORIGINAL_PATH" +unset spack + +# start in the spack root directory +cd $SPACK_ROOT + +# Run bash tests with coverage enabled, but pipe output to /dev/null +# because it seems that kcov seems to undo the script's redirection +if [ "$BASH_COVERAGE" = true ]; then + ${QA_DIR}/bashcov ${QA_DIR}/setup-env-test.sh &> /dev/null +fi + +# run the test scripts for their output (these will print nicely) +bash ${QA_DIR}/setup-env-test.sh +zsh ${QA_DIR}/setup-env-test.sh +dash ${QA_DIR}/setup-env-test.sh diff --git a/share/spack/qa/setup-env-test.sh b/share/spack/qa/setup-env-test.sh new file mode 100755 index 0000000000..b75d006e75 --- /dev/null +++ b/share/spack/qa/setup-env-test.sh @@ -0,0 +1,289 @@ +#!/bin/sh +# +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# +# This script tests that Spack's setup-env.sh init script works. +# +# The tests are portable to bash, zsh, and bourne shell, and can be run +# in any of these shells. +# + +# ------------------------------------------------------------------------ +# Functions for color output. +# ------------------------------------------------------------------------ + +# Colors for output +red='\033[1;31m' +cyan='\033[1;36m' +green='\033[1;32m' +reset='\033[0m' + +echo_red() { + printf "${red}$*${reset}\n" +} + +echo_green() { + printf "${green}$*${reset}\n" +} + +echo_msg() { + printf "${cyan}$*${reset}\n" +} + +# ------------------------------------------------------------------------ +# Generic functions for testing shell code. +# ------------------------------------------------------------------------ + +# counts of test successes and failures. +success=0 +errors=0 + +# Print out a header for a group of tests. +title() { + echo + echo_msg "$@" + echo_msg "---------------------------------" +} + +# echo FAIL in red text; increment failures +fail() { + echo_red FAIL + errors=$((errors+1)) +} + +# +# Echo SUCCESS in green; increment successes +# +pass() { + echo_green SUCCESS + success=$((success+1)) +} + +# +# Run a command and suppress output unless it fails. +# On failure, echo the exit code and output. +# +succeeds() { + printf "'%s' succeeds ... " "$*" + output=$("$@" 2>&1) + err="$?" + + if [ "$err" != 0 ]; then + fail + echo_red "Command failed with error $err." + if [ -n "$output" ]; then + echo_msg "Output:" + echo "$output" + else + echo_msg "No output." + fi + else + pass + fi +} + +# +# Run a command and suppress output unless it succeeds. +# If the command succeeds, echo the output. +# +fails() { + printf "'%s' fails ... " "$*" + output=$("$@" 2>&1) + err="$?" + + if [ "$err" = 0 ]; then + fail + echo_red "Command failed with error $err." + if [ -n "$output" ]; then + echo_msg "Output:" + echo "$output" + else + echo_msg "No output." + fi + else + pass + fi +} + +# +# Ensure that a string is in the output of a command. +# Suppresses output on success. +# On failure, echo the exit code and output. +# +contains() { + string="$1" + shift + + printf "'%s' output contains '$string' ... " "$*" + output=$("$@" 2>&1) + err="$?" + + if [ "${output#*$string}" = "${output}" ]; then + fail + echo_red "Command exited with error $err." + echo_red "'$string' was not in output." + if [ -n "$output" ]; then + echo_msg "Output:" + echo "$output" + else + echo_msg "No output." + fi + else + pass + fi +} + +# ----------------------------------------------------------------------- +# Instead of invoking the module/use/dotkit commands, we print the +# arguments that Spack invokes the command with, so we can check that +# Spack passes the expected arguments in the tests below. +# +# We make that happen by defining the sh functions below. +# ----------------------------------------------------------------------- +module() { + echo module "$@" +} + +use() { + echo use "$@" +} + +unuse() { + echo unuse "$@" +} + +# ----------------------------------------------------------------------- +# Setup test environment and do some preliminary checks +# ----------------------------------------------------------------------- + +# Make sure no environment is active +unset SPACK_ENV + +# Source setup-env.sh before tests +. share/spack/setup-env.sh + +title "Testing setup-env.sh with $_sp_shell" + +# spack command is now avaialble +succeeds which spack + +# mock cd command (intentionally define only AFTER setup-env.sh) +cd() { + echo cd "$@" +} + +# create a fake mock package install and store its location for later +title "Setup" +echo "Creating a mock package installation" +spack -m install --fake a +a_install=$(spack location -i a) +a_module=$(spack -m module tcl find a) +a_dotkit=$(spack -m module dotkit find a) + +b_install=$(spack location -i b) +b_module=$(spack -m module tcl find b) +b_dotkit=$(spack -m module dotkit find b) + +# ensure that we uninstall b on exit +cleanup() { + title "Cleanup" + echo "Removing test package before exiting test script." + spack -m uninstall -yf b + spack -m uninstall -yf a + + echo + echo "$success tests succeeded." + echo "$errors tests failed." + if [ "$errors" = 0 ]; then + pass + exit 0 + else + fail + exit 1 + fi +} +trap cleanup EXIT + +# ----------------------------------------------------------------------- +# Test all spack commands with special env support +# ----------------------------------------------------------------------- +title 'Testing `spack`' +contains 'usage: spack ' spack +contains "usage: spack " spack -h +contains "usage: spack " spack help +contains "usage: spack " spack -H +contains "usage: spack " spack help --all + +title 'Testing `spack cd`' +contains "usage: spack cd " spack cd -h +contains "usage: spack cd " spack cd --help +contains "cd $b_install" spack cd -i b + +title 'Testing `spack module`' +contains "usage: spack module " spack -m module -h +contains "usage: spack module " spack -m module --help +contains "usage: spack module " spack -m module + +title 'Testing `spack load`' +contains "module load $b_module" spack -m load b +fails spack -m load -l +contains "module load -l --arg $b_module" spack -m load -l --arg b +contains "module load $b_module $a_module" spack -m load -r a +contains "module load $b_module $a_module" spack -m load --dependencies a +fails spack -m load d +contains "usage: spack load " spack -m load -h +contains "usage: spack load " spack -m load -h d +contains "usage: spack load " spack -m load --help + +title 'Testing `spack unload`' +contains "module unload $b_module" spack -m unload b +fails spack -m unload -l +contains "module unload -l --arg $b_module" spack -m unload -l --arg b +fails spack -m unload d +contains "usage: spack unload " spack -m unload -h +contains "usage: spack unload " spack -m unload -h d +contains "usage: spack unload " spack -m unload --help + +title 'Testing `spack use`' +contains "use $b_dotkit" spack -m use b +fails spack -m use -l +contains "use -l --arg $b_dotkit" spack -m use -l --arg b +contains "use $b_dotkit $a_dotkit" spack -m use -r a +contains "use $b_dotkit $a_dotkit" spack -m use --dependencies a +fails spack -m use d +contains "usage: spack use " spack -m use -h +contains "usage: spack use " spack -m use -h d +contains "usage: spack use " spack -m use --help + +title 'Testing `spack unuse`' +contains "unuse $b_dotkit" spack -m unuse b +fails spack -m unuse -l +contains "unuse -l --arg $b_dotkit" spack -m unuse -l --arg b +fails spack -m unuse d +contains "usage: spack unuse " spack -m unuse -h +contains "usage: spack unuse " spack -m unuse -h d +contains "usage: spack unuse " spack -m unuse --help + +title 'Testing `spack env`' +contains "usage: spack env " spack env -h +contains "usage: spack env " spack env --help + +title 'Testing `spack env activate`' +contains "No such environment:" spack env activate no_such_environment +contains "usage: spack env activate " spack env activate +contains "usage: spack env activate " spack env activate -h +contains "usage: spack env activate " spack env activate --help + +title 'Testing `spack env deactivate`' +contains "Error: No environment is currently active" spack env deactivate +contains "usage: spack env deactivate " spack env deactivate no_such_environment +contains "usage: spack env deactivate " spack env deactivate -h +contains "usage: spack env deactivate " spack env deactivate --help + +title 'Testing `spack env list`' +contains " spack env list " spack env list -h +contains " spack env list " spack env list --help diff --git a/share/spack/qa/setup.sh b/share/spack/qa/setup.sh index 6256269211..3ff226d057 100755 --- a/share/spack/qa/setup.sh +++ b/share/spack/qa/setup.sh @@ -20,18 +20,24 @@ export SPACK_ROOT=$(realpath "$QA_DIR/../../..") coverage="" coverage_run="" +# bash coverage depends on some other factors -- there are issues with +# kcov for Python 2.6, unit tests, and build tests. +if [[ $TEST_SUITE == unit && # kcov segfaults for the MPICH build test + $TRAVIS_OS_NAME == linux && + $TRAVIS_PYTHON_VERSION != 2.6 ]]; +then + BASH_COVERAGE="true" +else + BASH_COVERAGE="false" +fi + # Set up some variables for running coverage tests. if [[ "$COVERAGE" == "true" ]]; then # these set up coverage for Python coverage=coverage coverage_run="coverage run" - # make a coverage directory for kcov, and patch cc to use our bashcov - # script instead of plain bash - if [[ $TEST_SUITE == unit && # kcov segfaults for the MPICH build test - $TRAVIS_OS_NAME == linux && - $TRAVIS_PYTHON_VERSION != 2.6 ]]; - then + if [ "$BASH_COVERAGE" = true ]; then mkdir -p coverage cc_script="$SPACK_ROOT/lib/spack/env/cc" bashcov=$(realpath ${QA_DIR}/bashcov) |