tests: add tests for setup-env.sh

- tests use a shell-script harness and test all Spack commands that
  require special shell support.

- tests work in bash, zsh, and dash

- run setup-env.sh tests on macos and linux builds.
  - we run them on macos and linux

5 files changed, 343 insertions, 29 deletions

diff --git a/.codecov.yml b/.codecov.yml
index 042cc86e9c..c06cb57f88 100644
--- a/.codecov.yml
+++ b/.codecov.yml
@@ -11,5 +11,7 @@ ignore:
   - lib/spack/spack/test/.*
   - lib/spack/docs/.*
   - lib/spack/external/.*
+  - share/spack/qa/.*
+  - share/spack/spack-completion.bash
 
 comment: off
diff --git a/.travis.yml b/.travis.yml
index 91a95486e9..465232093a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -50,24 +50,6 @@ jobs:
       os: linux
       language: python
       env: [ TEST_SUITE=unit, COVERAGE=true ]
-      addons:
-        apt:
-          packages:
-          - cmake
-          - gfortran
-          - graphviz
-          - gnupg2
-          - kcov
-          - mercurial
-          - ninja-build
-          - perl
-          - perl-base
-          - realpath
-          - patchelf
-          - r-base
-          - r-base-core
-          - r-base-dev
-
     - python: '3.7'
       sudo: required
       os: linux
@@ -159,6 +141,7 @@ addons:
       - r-base
       - r-base-core
       - r-base-dev
+      - zsh
   # for Mac builds, we use Homebrew
   homebrew:
     packages:
@@ -166,6 +149,8 @@ addons:
       - gcc
       - gnupg2
       - ccache
+      - dash
+      - kcov
     update: true
 
 # ~/.ccache needs to be cached directly as Travis is not taking care of it
@@ -223,11 +208,13 @@ script:
 
 after_success:
   - ccache -s
-  - if [[ "$TEST_SUITE" == "unit" || "$TEST_SUITE" == "build" ]]; then
-        codecov --env PYTHON_VERSION
-                --required
-                --flags "${TEST_SUITE}${TRAVIS_OS_NAME}";
-    fi
+  - case "$TEST_SUITE" in
+        unit)
+            codecov --env PYTHON_VERSION
+                    --required
+                    --flags "${TEST_SUITE}${TRAVIS_OS_NAME}";
+            ;;
+    esac
 
 #=============================================================================
 # Notifications
diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests
index 49dbda0d32..4403d53a76 100755
--- a/share/spack/qa/run-unit-tests
+++ b/share/spack/qa/run-unit-tests
@@ -16,6 +16,12 @@
 #     Optionally add one or more unit tests
 #     to only run these tests.
 #
+
+#-----------------------------------------------------------
+# Run a few initial commands and set up test environment
+#-----------------------------------------------------------
+ORIGINAL_PATH="$PATH"
+
 . "$(dirname $0)/setup.sh"
 check_dependencies ${coverage} git hg svn
 
@@ -33,9 +39,33 @@ bin/spack help -a
 # Profile and print top 20 lines for a simple call to spack spec
 bin/spack -p --lines 20 spec mpileaks%gcc ^elfutils@0.170
 
+#-----------------------------------------------------------
 # Run unit tests with code coverage
+#-----------------------------------------------------------
 extra_args=""
 if [[ -n "$@" ]]; then
     extra_args="-k $@"
 fi
 ${coverage_run} bin/spack test --verbose "$extra_args"
+
+#-----------------------------------------------------------
+# Run tests for setup-env.sh
+#-----------------------------------------------------------
+# Clean the environment by removing Spack from the path and getting rid of
+# the spack shell function
+export PATH="$ORIGINAL_PATH"
+unset spack
+
+# start in the spack root directory
+cd $SPACK_ROOT
+
+# Run bash tests with coverage enabled, but pipe output to /dev/null
+# because it seems that kcov seems to undo the script's redirection
+if [ "$BASH_COVERAGE" = true ]; then
+    ${QA_DIR}/bashcov ${QA_DIR}/setup-env-test.sh &> /dev/null
+fi
+
+# run the test scripts for their output (these will print nicely)
+bash ${QA_DIR}/setup-env-test.sh
+zsh  ${QA_DIR}/setup-env-test.sh
+dash ${QA_DIR}/setup-env-test.sh
diff --git a/share/spack/qa/setup-env-test.sh b/share/spack/qa/setup-env-test.sh
new file mode 100755
index 0000000000..b75d006e75
--- /dev/null
+++ b/share/spack/qa/setup-env-test.sh
@@ -0,0 +1,289 @@
+#!/bin/sh
+#
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+#
+# This script tests that Spack's setup-env.sh init script works.
+#
+# The tests are portable to bash, zsh, and bourne shell, and can be run
+# in any of these shells.
+#
+
+# ------------------------------------------------------------------------
+# Functions for color output.
+# ------------------------------------------------------------------------
+
+# Colors for output
+red='\033[1;31m'
+cyan='\033[1;36m'
+green='\033[1;32m'
+reset='\033[0m'
+
+echo_red() {
+    printf "${red}$*${reset}\n"
+}
+
+echo_green() {
+    printf "${green}$*${reset}\n"
+}
+
+echo_msg() {
+    printf "${cyan}$*${reset}\n"
+}
+
+# ------------------------------------------------------------------------
+# Generic functions for testing shell code.
+# ------------------------------------------------------------------------
+
+# counts of test successes and failures.
+success=0
+errors=0
+
+# Print out a header for a group of tests.
+title() {
+    echo
+    echo_msg "$@"
+    echo_msg "---------------------------------"
+}
+
+# echo FAIL in red text; increment failures
+fail() {
+    echo_red FAIL
+    errors=$((errors+1))
+}
+
+#
+# Echo SUCCESS in green; increment successes
+#
+pass() {
+    echo_green SUCCESS
+    success=$((success+1))
+}
+
+#
+# Run a command and suppress output unless it fails.
+# On failure, echo the exit code and output.
+#
+succeeds() {
+    printf "'%s' succeeds ... " "$*"
+    output=$("$@" 2>&1)
+    err="$?"
+
+    if [ "$err" != 0 ]; then
+        fail
+        echo_red "Command failed with error $err."
+        if [ -n "$output" ]; then
+            echo_msg "Output:"
+            echo "$output"
+        else
+            echo_msg "No output."
+        fi
+    else
+        pass
+    fi
+}
+
+#
+# Run a command and suppress output unless it succeeds.
+# If the command succeeds, echo the output.
+#
+fails() {
+    printf "'%s' fails ... " "$*"
+    output=$("$@" 2>&1)
+    err="$?"
+
+    if [ "$err" = 0 ]; then
+        fail
+        echo_red "Command failed with error $err."
+        if [ -n "$output" ]; then
+            echo_msg "Output:"
+            echo "$output"
+        else
+            echo_msg "No output."
+        fi
+    else
+        pass
+    fi
+}
+
+#
+# Ensure that a string is in the output of a command.
+# Suppresses output on success.
+# On failure, echo the exit code and output.
+#
+contains() {
+    string="$1"
+    shift
+
+    printf "'%s' output contains '$string' ... " "$*"
+    output=$("$@" 2>&1)
+    err="$?"
+
+    if [ "${output#*$string}" = "${output}" ]; then
+        fail
+        echo_red "Command exited with error $err."
+        echo_red "'$string' was not in output."
+        if [ -n "$output" ]; then
+            echo_msg "Output:"
+            echo "$output"
+        else
+            echo_msg "No output."
+        fi
+    else
+        pass
+    fi
+}
+
+# -----------------------------------------------------------------------
+# Instead of invoking the module/use/dotkit commands, we print the
+# arguments that Spack invokes the command with, so we can check that
+# Spack passes the expected arguments in the tests below.
+#
+# We make that happen by defining the sh functions below.
+# -----------------------------------------------------------------------
+module() {
+    echo module "$@"
+}
+
+use() {
+    echo use "$@"
+}
+
+unuse() {
+    echo unuse "$@"
+}
+
+# -----------------------------------------------------------------------
+# Setup test environment and do some preliminary checks
+# -----------------------------------------------------------------------
+
+# Make sure no environment is active
+unset SPACK_ENV
+
+# Source setup-env.sh before tests
+.  share/spack/setup-env.sh
+
+title "Testing setup-env.sh with $_sp_shell"
+
+# spack command is now avaialble
+succeeds which spack
+
+# mock cd command (intentionally define only AFTER setup-env.sh)
+cd() {
+    echo cd "$@"
+}
+
+# create a fake mock package install and store its location for later
+title "Setup"
+echo "Creating a mock package installation"
+spack -m install --fake a
+a_install=$(spack location -i a)
+a_module=$(spack -m module tcl find a)
+a_dotkit=$(spack -m module dotkit find a)
+
+b_install=$(spack location -i b)
+b_module=$(spack -m module tcl find b)
+b_dotkit=$(spack -m module dotkit find b)
+
+# ensure that we uninstall b on exit
+cleanup() {
+    title "Cleanup"
+    echo "Removing test package before exiting test script."
+    spack -m uninstall -yf b
+    spack -m uninstall -yf a
+
+    echo
+    echo "$success tests succeeded."
+    echo "$errors tests failed."
+    if [ "$errors" = 0 ]; then
+        pass
+        exit 0
+    else
+        fail
+        exit 1
+    fi
+}
+trap cleanup EXIT
+
+# -----------------------------------------------------------------------
+# Test all spack commands with special env support
+# -----------------------------------------------------------------------
+title 'Testing `spack`'
+contains 'usage: spack ' spack
+contains "usage: spack " spack -h
+contains "usage: spack " spack help
+contains "usage: spack " spack -H
+contains "usage: spack " spack help --all
+
+title 'Testing `spack cd`'
+contains "usage: spack cd " spack cd -h
+contains "usage: spack cd " spack cd --help
+contains "cd $b_install" spack cd -i b
+
+title 'Testing `spack module`'
+contains "usage: spack module " spack -m module -h
+contains "usage: spack module " spack -m module --help
+contains "usage: spack module " spack -m module
+
+title 'Testing `spack load`'
+contains "module load $b_module" spack -m load b
+fails spack -m load -l
+contains "module load -l --arg $b_module" spack -m load -l --arg b
+contains "module load $b_module $a_module" spack -m load -r a
+contains "module load $b_module $a_module" spack -m load --dependencies a
+fails spack -m load d
+contains "usage: spack load " spack -m load -h
+contains "usage: spack load " spack -m load -h d
+contains "usage: spack load " spack -m load --help
+
+title 'Testing `spack unload`'
+contains "module unload $b_module" spack -m unload b
+fails spack -m unload -l
+contains "module unload -l --arg $b_module" spack -m unload -l --arg b
+fails spack -m unload d
+contains "usage: spack unload " spack -m unload -h
+contains "usage: spack unload " spack -m unload -h d
+contains "usage: spack unload " spack -m unload --help
+
+title 'Testing `spack use`'
+contains "use $b_dotkit" spack -m use b
+fails spack -m use -l
+contains "use -l --arg $b_dotkit" spack -m use -l --arg b
+contains "use $b_dotkit $a_dotkit" spack -m use -r a
+contains "use $b_dotkit $a_dotkit" spack -m use --dependencies a
+fails spack -m use d
+contains "usage: spack use " spack -m use -h
+contains "usage: spack use " spack -m use -h d
+contains "usage: spack use " spack -m use --help
+
+title 'Testing `spack unuse`'
+contains "unuse $b_dotkit" spack -m unuse b
+fails spack -m unuse -l
+contains "unuse -l --arg $b_dotkit" spack -m unuse -l --arg b
+fails spack -m unuse d
+contains "usage: spack unuse "  spack -m unuse -h
+contains "usage: spack unuse "  spack -m unuse -h d
+contains "usage: spack unuse "  spack -m unuse --help
+
+title 'Testing `spack env`'
+contains "usage: spack env " spack env -h
+contains "usage: spack env " spack env --help
+
+title 'Testing `spack env activate`'
+contains "No such environment:" spack env activate no_such_environment
+contains "usage: spack env activate " spack env activate
+contains "usage: spack env activate " spack env activate -h
+contains "usage: spack env activate " spack env activate --help
+
+title 'Testing `spack env deactivate`'
+contains "Error: No environment is currently active" spack env deactivate
+contains "usage: spack env deactivate " spack env deactivate no_such_environment
+contains "usage: spack env deactivate " spack env deactivate -h
+contains "usage: spack env deactivate " spack env deactivate --help
+
+title 'Testing `spack env list`'
+contains " spack env list " spack env list -h
+contains " spack env list " spack env list --help
diff --git a/share/spack/qa/setup.sh b/share/spack/qa/setup.sh
index 6256269211..3ff226d057 100755
--- a/share/spack/qa/setup.sh
+++ b/share/spack/qa/setup.sh
@@ -20,18 +20,24 @@ export SPACK_ROOT=$(realpath "$QA_DIR/../../..")
 coverage=""
 coverage_run=""
 
+# bash coverage depends on some other factors -- there are issues with
+# kcov for Python 2.6, unit tests, and build tests.
+if [[ $TEST_SUITE == unit &&   # kcov segfaults for the MPICH build test
+      $TRAVIS_OS_NAME == linux &&
+      $TRAVIS_PYTHON_VERSION != 2.6 ]];
+then
+    BASH_COVERAGE="true"
+else
+    BASH_COVERAGE="false"
+fi
+
 # Set up some variables for running coverage tests.
 if [[ "$COVERAGE" == "true" ]]; then
     # these set up coverage for Python
     coverage=coverage
     coverage_run="coverage run"
 
-    # make a coverage directory for kcov, and patch cc to use our bashcov
-    # script instead of plain bash
-    if [[ $TEST_SUITE == unit &&   # kcov segfaults for the MPICH build test
-          $TRAVIS_OS_NAME == linux &&
-          $TRAVIS_PYTHON_VERSION != 2.6 ]];
-    then
+    if [ "$BASH_COVERAGE" = true ]; then
         mkdir -p coverage
         cc_script="$SPACK_ROOT/lib/spack/env/cc"
         bashcov=$(realpath ${QA_DIR}/bashcov)