From 77b2e578ec47f7713cae965fede1ab6e60aa69c4 Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Wed, 18 Nov 2020 02:39:02 -0800 Subject: spack test (#15702) Users can add test() methods to their packages to run smoke tests on installations with the new `spack test` command (the old `spack test` is now `spack unit-test`). spack test is environment-aware, so you can `spack install` an environment and then run `spack test run` to run smoke tests on all of its packages. Historical test logs can be perused with `spack test results`. Generic smoke tests for MPI implementations, C, C++, and Fortran compilers as well as specific smoke tests for 18 packages. Inside the test method, individual tests can be run separately (and continue to run best-effort after a test failure) using the `run_test` method. The `run_test` method encapsulates finding test executables, running and checking return codes, checking output, and error handling. This handles the following trickier aspects of testing with direct support in Spack's package API: - [x] Caching source or intermediate build files at build time for use at test time. - [x] Test dependencies, - [x] packages that require a compiler for testing (such as library only packages). See the packaging guide for more details on using Spack testing support. Included is support for package.py files for virtual packages. This does not change the Spack interface, but is a major change in internals. Co-authored-by: Tamara Dahlgren Co-authored-by: wspear Co-authored-by: Adam J. Stewart --- share/spack/qa/completion-test.sh | 2 +- share/spack/qa/run-unit-tests | 2 +- share/spack/spack-completion.bash | 77 ++++++++++++++++++++++++++-- share/spack/templates/reports/cdash/Test.xml | 27 ++++++++++ 4 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 share/spack/templates/reports/cdash/Test.xml (limited to 'share') diff --git a/share/spack/qa/completion-test.sh b/share/spack/qa/completion-test.sh index 5b326b4a6d..59a5181b98 100755 --- a/share/spack/qa/completion-test.sh +++ b/share/spack/qa/completion-test.sh @@ -56,7 +56,7 @@ contains 'hdf5' _spack_completions spack -d install --jobs 8 '' contains 'hdf5' _spack_completions spack install -v '' # XFAIL: Fails for Python 2.6 because pkg_resources not found? -#contains 'compilers.py' _spack_completions spack test '' +#contains 'compilers.py' _spack_completions spack unit-test '' title 'Testing debugging functions' diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests index c529f8297e..ec8aaf76b9 100755 --- a/share/spack/qa/run-unit-tests +++ b/share/spack/qa/run-unit-tests @@ -42,4 +42,4 @@ spack -p --lines 20 spec mpileaks%gcc ^elfutils@0.170 #----------------------------------------------------------- # Run unit tests with code coverage #----------------------------------------------------------- -$coverage_run $(which spack) test -x --verbose +$coverage_run $(which spack) unit-test -x --verbose diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 7d54414397..969a0898fe 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -320,7 +320,7 @@ _spack() { then SPACK_COMPREPLY="-h --help -H --all-help --color -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars" else - SPACK_COMPREPLY="activate add arch blame build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config containerize create deactivate debug dependencies dependents deprecate dev-build develop docs edit env extensions external fetch find flake8 gc gpg graph help info install license list load location log-parse maintainers mirror module patch pkg providers pydoc python reindex remove rm repo resource restage setup solve spec stage test tutorial undevelop uninstall unload url verify versions view" + SPACK_COMPREPLY="activate add arch blame build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config containerize create deactivate debug dependencies dependents deprecate dev-build develop docs edit env extensions external fetch find flake8 gc gpg graph help info install license list load location log-parse maintainers mirror module patch pkg providers pydoc python reindex remove rm repo resource restage setup solve spec stage test test-env tutorial undevelop uninstall unit-test unload url verify versions view" fi } @@ -1020,7 +1020,7 @@ _spack_info() { _spack_install() { if $list_options then - SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --no-check-signature --require-full-hash-match --show-log-on-error --source -n --no-checksum -v --verbose --fake --only-concrete -f --file --clean --dirty --test --run-tests --log-format --log-file --help-cdash -y --yes-to-all --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp" + SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --no-check-signature --require-full-hash-match --show-log-on-error --source -n --no-checksum -v --verbose --fake --only-concrete -f --file --clean --dirty --test --run-tests --log-format --log-file --help-cdash --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp -y --yes-to-all" else _all_packages fi @@ -1046,7 +1046,7 @@ _spack_license_verify() { _spack_list() { if $list_options then - SPACK_COMPREPLY="-h --help -d --search-description --format --update -t --tags" + SPACK_COMPREPLY="-h --help -d --search-description --format --update -v --virtuals -t --tags" else _all_packages fi @@ -1494,9 +1494,67 @@ _spack_stage() { _spack_test() { if $list_options then - SPACK_COMPREPLY="-h --help -H --pytest-help -l --list -L --list-long -N --list-names --extension -s -k --showlocals" + SPACK_COMPREPLY="-h --help" else - _tests + SPACK_COMPREPLY="run list find status results remove" + fi +} + +_spack_test_run() { + if $list_options + then + SPACK_COMPREPLY="-h --help --alias --fail-fast --fail-first --keep-stage --log-format --log-file --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp --help-cdash --clean --dirty" + else + _installed_packages + fi +} + +_spack_test_list() { + SPACK_COMPREPLY="-h --help" +} + +_spack_test_find() { + if $list_options + then + SPACK_COMPREPLY="-h --help" + else + _all_packages + fi +} + +_spack_test_status() { + if $list_options + then + SPACK_COMPREPLY="-h --help" + else + SPACK_COMPREPLY="" + fi +} + +_spack_test_results() { + if $list_options + then + SPACK_COMPREPLY="-h --help -l --logs -f --failed" + else + SPACK_COMPREPLY="" + fi +} + +_spack_test_remove() { + if $list_options + then + SPACK_COMPREPLY="-h --help -y --yes-to-all" + else + SPACK_COMPREPLY="" + fi +} + +_spack_test_env() { + if $list_options + then + SPACK_COMPREPLY="-h --help --clean --dirty --dump --pickle" + else + _all_packages fi } @@ -1522,6 +1580,15 @@ _spack_uninstall() { fi } +_spack_unit_test() { + if $list_options + then + SPACK_COMPREPLY="-h --help -H --pytest-help -l --list -L --list-long -N --list-names --extension -s -k --showlocals" + else + _tests + fi +} + _spack_unload() { if $list_options then diff --git a/share/spack/templates/reports/cdash/Test.xml b/share/spack/templates/reports/cdash/Test.xml new file mode 100644 index 0000000000..6aeed4e263 --- /dev/null +++ b/share/spack/templates/reports/cdash/Test.xml @@ -0,0 +1,27 @@ + + {{ test.starttime }} + {{ install_command }} +{% for warning in test.warnings %} + + {{ warning.line_no }} + {{ warning.text }} + {{ warning.source_file }} + {{ warning.source_line_no }} + {{ warning.pre_context }} + {{ warning.post_context }} + +{% endfor %} +{% for error in test.errors %} + + {{ error.line_no }} + {{ error.text }} + {{ error.source_file }} + {{ error.source_line_no }} + {{ error.pre_context }} + {{ error.post_context }} + +{% endfor %} + {{ test.endtime }} + 0 + + -- cgit v1.2.3-70-g09d2