spack ci: add support for running stand-alone tests (#27877)

This support requires adding the '--tests' option to 'spack ci rebuild'.
Packages whose stand-alone tests are broken (in the CI environment) can
be configured in gitlab-ci to be skipped by adding them to
broken-tests-packages.

Highlights include:
- Restructured 'spack ci' help to provide better subcommand summaries;
- Ensured only one InstallError (i.e., installer's) rather than allowing
  build_environment to have its own; and
- Refactored CI and CDash reporting to keep CDash-related properties and
  behavior in a separate class.

This allows stand-alone tests from `spack ci` to run when the `--tests`
option is used.  With `--tests`, stand-alone tests are run **after** a
**successful** (re)build of the package.  Test results are collected
and report(able) using CDash.

This PR adds the following features:
- Adds `-t` and `--tests` to `spack ci rebuild` to run stand-alone tests;
- Adds `--fail-fast` to stop stand-alone tests after the first failure;
- Ensures a *single* `InstallError` across packages
    (i.e., removes second class from build environment);
- Captures skipping tests for externals and uninstalled packages
    (for CDash reporting);
- Copies test logs and outputs to the CI artifacts directory to facilitate
    debugging;
- Parses stand-alone test results to report outputs from each `run_test` as
    separate test parts (CDash reporting);
- Logs a test completion message to allow capture of timing of the last
    `run_test` part;
- Adds the runner description to the CDash site to better distinguish entries
    in CDash tables;
- Adds `gitlab-ci` `broken-tests-packages` to CI configuration to skip
    stand-alone testing for packages with known issues;
- Changes `spack ci --help` so description of each subcommand is a single line;
- Changes `spack ci <subcommand> --help` to provide the full description of
    each command (versus no description); and
- Ensures `junit` test log file ends in an `.xml` extension (versus default where
    it does not).

Tasks:

- [x] Include the equivalent of the architecture information, or at least the host target, in the CDash output
- [x] Upload stand-alone test results files as  `test` artifacts
- [x] Confirm tests are run in GitLab
- [x] Ensure CDash results are uploaded as artifacts
- [x] Resolve issues with CDash build-and test results appearing on same row of the table 
- [x] Add unit tests  as needed
- [x] Investigate why some (dependency) packages don't have test results (e.g., related from other pipelines)
- [x] Ensure proper parsing and reporting of skipped tests (as `not run`) .. post- #28701 merge
- [x] Restore the proper CDash URLand or mirror ONCE out-of-band testing completed

4 files changed, 53 insertions, 3 deletions

diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
index 31d408db6e..7b06fd2dda 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
@@ -248,10 +248,13 @@ spack:
       - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
       - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
       - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
-      - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+      - spack -d ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
 
     image: ecpe4s/ubuntu22.04-runner-x86_64:2022-07-01
     
+    broken-tests-packages:
+      - gptune
+
     mappings:
       - match:
           - hipblas
diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash
index 936593d98a..4cfde14e7a 100755
--- a/share/spack/spack-completion.bash
+++ b/share/spack/spack-completion.bash
@@ -612,7 +612,7 @@ _spack_ci_rebuild_index() {
 }
 
 _spack_ci_rebuild() {
-    SPACK_COMPREPLY="-h --help"
+    SPACK_COMPREPLY="-h --help -t --tests --fail-fast"
 }
 
 _spack_ci_reproduce_build() {
diff --git a/share/spack/templates/reports/cdash/Site.xml b/share/spack/templates/reports/cdash/Site.xml
index f0a150b6e5..e8a6c0609b 100644
--- a/share/spack/templates/reports/cdash/Site.xml
+++ b/share/spack/templates/reports/cdash/Site.xml
@@ -2,6 +2,9 @@
 <Site BuildName="{{ buildname }}"
       BuildStamp="{{ buildstamp }}"
       Name="{{ site }}"
+      Generator="{{ generator }}"
+      Hostname="{{ hostname }}"
       OSName="{{ osname }}"
+      OSRelease="{{ osrelease }}"
+      VendorString="{{ target }}"
 >
-
diff --git a/share/spack/templates/reports/cdash/Testing.xml b/share/spack/templates/reports/cdash/Testing.xml
new file mode 100644
index 0000000000..a5eb58c35a
--- /dev/null
+++ b/share/spack/templates/reports/cdash/Testing.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    This file has been modeled after the examples at this url:
+
+    https://www.paraview.org/Wiki/CDash:XML
+-->
+<Site BuildName="{{ buildname }}"
+      BuildStamp="{{ buildstamp }}"
+      Name="{{ site }}"
+      Generator="{{ generator }}"
+      Hostname="{{ hostname }}"
+      OSName="{{ osname }}"
+      OSRelease="{{ osrelease }}"
+      VendorString="{{ target }}"
+>
+  <Testing>
+    <StartTestTime>{{ testing.starttime }}</StartTestTime>
+{% for part in testing.parts %}
+    <Test Status="{{ part.status }}">
+      <Name>{{ part.name }}</Name>
+      <FullCommandLine>{{ part.command }}</FullCommandLine>
+      <Results>
+        <NamedMeasurement type="numeric/double" name="Execution Time">
+          <Value>{{ part.elapsed }}</Value>
+        </NamedMeasurement>
+{% if part.desc %}
+        <NamedMeasurement type="text/string" name="Description">
+          <Value>{{ part.desc }}</Value>
+        </NamedMeasurement>
+{% endif %}
+        <NamedMeasurement type="text/string" name="Completion Status">
+          <Value>{{ part.completed }}</Value>
+        </NamedMeasurement>
+{% if part.output %}
+        <Measurement>
+          <Value>{{ part.output }}</Value>
+        </Measurement>
+{% endif %}
+      </Results>
+    </Test>
+{% endfor %}
+    <EndTestTime>{{ testing.endtime }}</EndTestTime>
+  </Testing>
+</Site>