summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTamara Dahlgren <35777542+tldahlgren@users.noreply.github.com>2022-08-23 00:52:48 -0700
committerGitHub <noreply@github.com>2022-08-23 00:52:48 -0700
commit3c3b18d8588a82b6ace4ca5e852497f115e88479 (patch)
tree9ad8e683acb4556aa7eb0433615b6b07ab2dc331
parent8b49790784456f997433c3ea245018d750d5ee2e (diff)
downloadspack-3c3b18d8588a82b6ace4ca5e852497f115e88479.tar.gz
spack-3c3b18d8588a82b6ace4ca5e852497f115e88479.tar.bz2
spack-3c3b18d8588a82b6ace4ca5e852497f115e88479.tar.xz
spack-3c3b18d8588a82b6ace4ca5e852497f115e88479.zip
spack ci: add support for running stand-alone tests (#27877)
This support requires adding the '--tests' option to 'spack ci rebuild'. Packages whose stand-alone tests are broken (in the CI environment) can be configured in gitlab-ci to be skipped by adding them to broken-tests-packages. Highlights include: - Restructured 'spack ci' help to provide better subcommand summaries; - Ensured only one InstallError (i.e., installer's) rather than allowing build_environment to have its own; and - Refactored CI and CDash reporting to keep CDash-related properties and behavior in a separate class. This allows stand-alone tests from `spack ci` to run when the `--tests` option is used. With `--tests`, stand-alone tests are run **after** a **successful** (re)build of the package. Test results are collected and report(able) using CDash. This PR adds the following features: - Adds `-t` and `--tests` to `spack ci rebuild` to run stand-alone tests; - Adds `--fail-fast` to stop stand-alone tests after the first failure; - Ensures a *single* `InstallError` across packages (i.e., removes second class from build environment); - Captures skipping tests for externals and uninstalled packages (for CDash reporting); - Copies test logs and outputs to the CI artifacts directory to facilitate debugging; - Parses stand-alone test results to report outputs from each `run_test` as separate test parts (CDash reporting); - Logs a test completion message to allow capture of timing of the last `run_test` part; - Adds the runner description to the CDash site to better distinguish entries in CDash tables; - Adds `gitlab-ci` `broken-tests-packages` to CI configuration to skip stand-alone testing for packages with known issues; - Changes `spack ci --help` so description of each subcommand is a single line; - Changes `spack ci <subcommand> --help` to provide the full description of each command (versus no description); and - Ensures `junit` test log file ends in an `.xml` extension (versus default where it does not). Tasks: - [x] Include the equivalent of the architecture information, or at least the host target, in the CDash output - [x] Upload stand-alone test results files as `test` artifacts - [x] Confirm tests are run in GitLab - [x] Ensure CDash results are uploaded as artifacts - [x] Resolve issues with CDash build-and test results appearing on same row of the table - [x] Add unit tests as needed - [x] Investigate why some (dependency) packages don't have test results (e.g., related from other pipelines) - [x] Ensure proper parsing and reporting of skipped tests (as `not run`) .. post- #28701 merge - [x] Restore the proper CDash URLand or mirror ONCE out-of-band testing completed
-rw-r--r--lib/spack/spack/build_environment.py10
-rw-r--r--lib/spack/spack/ci.py519
-rw-r--r--lib/spack/spack/cmd/__init__.py5
-rw-r--r--lib/spack/spack/cmd/ci.py309
-rw-r--r--lib/spack/spack/cmd/install.py1
-rw-r--r--lib/spack/spack/cmd/test.py39
-rw-r--r--lib/spack/spack/environment/environment.py2
-rw-r--r--lib/spack/spack/install_test.py5
-rw-r--r--lib/spack/spack/installer.py52
-rw-r--r--lib/spack/spack/package_base.py7
-rw-r--r--lib/spack/spack/report.py5
-rw-r--r--lib/spack/spack/reporters/cdash.py137
-rw-r--r--lib/spack/spack/reporters/extract.py212
-rw-r--r--lib/spack/spack/reporters/junit.py10
-rw-r--r--lib/spack/spack/schema/gitlab_ci.py6
-rw-r--r--lib/spack/spack/test/ci.py178
-rw-r--r--lib/spack/spack/test/cmd/ci.py336
-rw-r--r--lib/spack/spack/test/cmd/config.py2
-rw-r--r--lib/spack/spack/test/cmd/install.py5
-rw-r--r--lib/spack/spack/test/cmd/test.py4
-rw-r--r--lib/spack/spack/test/reporters.py175
-rw-r--r--lib/spack/spack/test/test_suite.py95
-rw-r--r--share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml5
-rwxr-xr-xshare/spack/spack-completion.bash2
-rw-r--r--share/spack/templates/reports/cdash/Site.xml5
-rw-r--r--share/spack/templates/reports/cdash/Testing.xml44
26 files changed, 1722 insertions, 448 deletions
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index 5a780e4bd5..c292fb89bb 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -65,6 +65,7 @@ import spack.subprocess_context
import spack.user_environment
import spack.util.path
from spack.error import NoHeadersError, NoLibrariesError
+from spack.installer import InstallError
from spack.util.cpus import cpus_available
from spack.util.environment import (
EnvironmentModifications,
@@ -1279,15 +1280,6 @@ def get_package_context(traceback, context=3):
return lines
-class InstallError(spack.error.SpackError):
- """Raised by packages when a package fails to install.
-
- Any subclass of InstallError will be annotated by Spack with a
- ``pkg`` attribute on failure, which the caller can use to get the
- package for which the exception was raised.
- """
-
-
class ChildError(InstallError):
"""Special exception class for wrapping exceptions from child processes
in Spack's build environment.
diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py
index 06dc4741ba..b1c777bb8b 100644
--- a/lib/spack/spack/ci.py
+++ b/lib/spack/spack/ci.py
@@ -10,7 +10,9 @@ import os
import re
import shutil
import stat
+import subprocess
import tempfile
+import time
import zipfile
from six import iteritems
@@ -20,6 +22,7 @@ from six.moves.urllib.request import HTTPHandler, Request, build_opener
import llnl.util.filesystem as fs
import llnl.util.tty as tty
+from llnl.util.lang import memoized
import spack
import spack.binary_distribution as bindist
@@ -35,7 +38,10 @@ import spack.util.gpg as gpg_util
import spack.util.spack_yaml as syaml
import spack.util.web as web_util
from spack.error import SpackError
+from spack.reporters.cdash import CDash
+from spack.reporters.cdash import build_stamp as cdash_build_stamp
from spack.spec import Spec
+from spack.util.pattern import Bunch
JOB_RETRY_CONDITIONS = [
"always",
@@ -60,69 +66,6 @@ class TemporaryDirectory(object):
return False
-def _create_buildgroup(opener, headers, url, project, group_name, group_type):
- data = {"newbuildgroup": group_name, "project": project, "type": group_type}
-
- enc_data = json.dumps(data).encode("utf-8")
-
- request = Request(url, data=enc_data, headers=headers)
-
- response = opener.open(request)
- response_code = response.getcode()
-
- if response_code != 200 and response_code != 201:
- msg = "Creating buildgroup failed (response code = {0}".format(response_code)
- tty.warn(msg)
- return None
-
- response_text = response.read()
- response_json = json.loads(response_text)
- build_group_id = response_json["id"]
-
- return build_group_id
-
-
-def _populate_buildgroup(job_names, group_name, project, site, credentials, cdash_url):
- url = "{0}/api/v1/buildgroup.php".format(cdash_url)
-
- headers = {
- "Authorization": "Bearer {0}".format(credentials),
- "Content-Type": "application/json",
- }
-
- opener = build_opener(HTTPHandler)
-
- parent_group_id = _create_buildgroup(opener, headers, url, project, group_name, "Daily")
- group_id = _create_buildgroup(
- opener, headers, url, project, "Latest {0}".format(group_name), "Latest"
- )
-
- if not parent_group_id or not group_id:
- msg = "Failed to create or retrieve buildgroups for {0}".format(group_name)
- tty.warn(msg)
- return
-
- data = {
- "project": project,
- "buildgroupid": group_id,
- "dynamiclist": [
- {"match": name, "parentgroupid": parent_group_id, "site": site} for name in job_names
- ],
- }
-
- enc_data = json.dumps(data).encode("utf-8")
-
- request = Request(url, data=enc_data, headers=headers)
- request.get_method = lambda: "PUT"
-
- response = opener.open(request)
- response_code = response.getcode()
-
- if response_code != 200:
- msg = "Error response code ({0}) in _populate_buildgroup".format(response_code)
- tty.warn(msg)
-
-
def _is_main_phase(phase_name):
return True if phase_name == "specs" else False
@@ -180,12 +123,6 @@ def get_job_name(phase, strip_compiler, spec, osarch, build_group):
return format_str.format(*format_args)
-def _get_cdash_build_name(spec, build_group):
- return "{0}@{1}%{2} arch={3} ({4})".format(
- spec.name, spec.version, spec.compiler, spec.architecture, build_group
- )
-
-
def _remove_reserved_tags(tags):
"""Convenience function to strip reserved tags from jobs"""
return [tag for tag in tags if tag not in SPACK_RESERVED_TAGS]
@@ -672,21 +609,8 @@ def generate_gitlab_ci_yaml(
gitlab_ci = yaml_root["gitlab-ci"]
- build_group = None
- enable_cdash_reporting = False
- cdash_auth_token = None
-
- if "cdash" in yaml_root:
- enable_cdash_reporting = True
- ci_cdash = yaml_root["cdash"]
- build_group = ci_cdash["build-group"]
- cdash_url = ci_cdash["url"]
- cdash_project = ci_cdash["project"]
- cdash_site = ci_cdash["site"]
-
- if "SPACK_CDASH_AUTH_TOKEN" in os.environ:
- tty.verbose("Using CDash auth token from environment")
- cdash_auth_token = os.environ.get("SPACK_CDASH_AUTH_TOKEN")
+ cdash_handler = CDashHandler(yaml_root.get("cdash")) if "cdash" in yaml_root else None
+ build_group = cdash_handler.build_group if cdash_handler else None
prune_untouched_packages = os.environ.get("SPACK_PRUNE_UNTOUCHED", None)
if prune_untouched_packages:
@@ -820,6 +744,7 @@ def generate_gitlab_ci_yaml(
job_log_dir = os.path.join(pipeline_artifacts_dir, "logs")
job_repro_dir = os.path.join(pipeline_artifacts_dir, "reproduction")
+ job_test_dir = os.path.join(pipeline_artifacts_dir, "tests")
local_mirror_dir = os.path.join(pipeline_artifacts_dir, "mirror")
user_artifacts_dir = os.path.join(pipeline_artifacts_dir, "user_data")
@@ -833,7 +758,8 @@ def generate_gitlab_ci_yaml(
rel_concrete_env_dir = os.path.relpath(concrete_env_dir, ci_project_dir)
rel_job_log_dir = os.path.relpath(job_log_dir, ci_project_dir)
rel_job_repro_dir = os.path.relpath(job_repro_dir, ci_project_dir)
- rel_local_mirror_dir = os.path.relpath(local_mirror_dir, ci_project_dir)
+ rel_job_test_dir = os.path.relpath(job_test_dir, ci_project_dir)
+ rel_local_mirror_dir = os.path.join(local_mirror_dir, ci_project_dir)
rel_user_artifacts_dir = os.path.relpath(user_artifacts_dir, ci_project_dir)
# Speed up staging by first fetching binary indices from all mirrors
@@ -1101,14 +1027,23 @@ def generate_gitlab_ci_yaml(
job_vars["SPACK_SPEC_NEEDS_REBUILD"] = str(rebuild_spec)
- if enable_cdash_reporting:
- cdash_build_name = _get_cdash_build_name(release_spec, build_group)
- all_job_names.append(cdash_build_name)
- job_vars["SPACK_CDASH_BUILD_NAME"] = cdash_build_name
+ if cdash_handler:
+ cdash_handler.current_spec = release_spec
+ build_name = cdash_handler.build_name
+ all_job_names.append(build_name)
+ job_vars["SPACK_CDASH_BUILD_NAME"] = build_name
+
+ build_stamp = cdash_handler.build_stamp
+ job_vars["SPACK_CDASH_BUILD_STAMP"] = build_stamp
variables.update(job_vars)
- artifact_paths = [rel_job_log_dir, rel_job_repro_dir, rel_user_artifacts_dir]
+ artifact_paths = [
+ rel_job_log_dir,
+ rel_job_repro_dir,
+ rel_job_test_dir,
+ rel_user_artifacts_dir,
+ ]
if enable_artifacts_buildcache:
bc_root = os.path.join(local_mirror_dir, "build_cache")
@@ -1176,11 +1111,9 @@ def generate_gitlab_ci_yaml(
)
# Use "all_job_names" to populate the build group for this set
- if enable_cdash_reporting and cdash_auth_token:
+ if cdash_handler and cdash_handler.auth_token:
try:
- _populate_buildgroup(
- all_job_names, build_group, cdash_project, cdash_site, cdash_auth_token, cdash_url
- )
+ cdash_handler.populate_buildgroup(all_job_names)
except (SpackError, HTTPError, URLError) as err:
tty.warn("Problem populating buildgroup: {0}".format(err))
else:
@@ -1341,6 +1274,7 @@ def generate_gitlab_ci_yaml(
"SPACK_REMOTE_MIRROR_URL": remote_mirror_url,
"SPACK_JOB_LOG_DIR": rel_job_log_dir,
"SPACK_JOB_REPRO_DIR": rel_job_repro_dir,
+ "SPACK_JOB_TEST_DIR": rel_job_test_dir,
"SPACK_LOCAL_MIRROR_DIR": rel_local_mirror_dir,
"SPACK_PIPELINE_TYPE": str(spack_pipeline_type),
}
@@ -1609,33 +1543,70 @@ def push_mirror_contents(env, specfile_path, mirror_url, sign_binaries):
raise inst
+def copy_files_to_artifacts(src, artifacts_dir):
+ """
+ Copy file(s) to the given artifacts directory
+
+ Parameters:
+ src (str): the glob-friendly path expression for the file(s) to copy
+ artifacts_dir (str): the destination directory
+ """
+ try:
+ fs.copy(src, artifacts_dir)
+ except Exception as err:
+ msg = ("Unable to copy files ({0}) to artifacts {1} due to " "exception: {2}").format(
+ src, artifacts_dir, str(err)
+ )
+ tty.error(msg)
+
+
def copy_stage_logs_to_artifacts(job_spec, job_log_dir):
- """Looks for spack-build-out.txt in the stage directory of the given
- job_spec, and attempts to copy the file into the directory given
- by job_log_dir.
+ """Copy selected build stage file(s) to the given artifacts directory
- Arguments:
+ Looks for spack-build-out.txt in the stage directory of the given
+ job_spec, and attempts to copy the file into the directory given
+ by job_log_dir.
- job_spec (spack.spec.Spec): Spec associated with spack install log
- job_log_dir (str): Path into which build log should be copied
+ Parameters:
+ job_spec (spack.spec.Spec): spec associated with spack install log
+ job_log_dir (str): path into which build log should be copied
"""
+ tty.debug("job spec: {0}".format(job_spec))
+ if not job_spec:
+ msg = "Cannot copy stage logs: job spec ({0}) is required"
+ tty.error(msg.format(job_spec))
+ return
+
try:
pkg_cls = spack.repo.path.get_pkg_class(job_spec.name)
job_pkg = pkg_cls(job_spec)
- tty.debug("job package: {0.fullname}".format(job_pkg))
- stage_dir = job_pkg.stage.path
- tty.debug("stage dir: {0}".format(stage_dir))
- build_out_src = os.path.join(stage_dir, "spack-build-out.txt")
- build_out_dst = os.path.join(job_log_dir, "spack-build-out.txt")
- tty.debug(
- "Copying build log ({0}) to artifacts ({1})".format(build_out_src, build_out_dst)
- )
- shutil.copyfile(build_out_src, build_out_dst)
- except Exception as inst:
- msg = (
- "Unable to copy build logs from stage to artifacts " "due to exception: {0}"
- ).format(inst)
- tty.error(msg)
+ tty.debug("job package: {0}".format(job_pkg))
+ except AssertionError:
+ msg = "Cannot copy stage logs: job spec ({0}) must be concrete"
+ tty.error(msg.format(job_spec))
+ return
+
+ stage_dir = job_pkg.stage.path
+ tty.debug("stage dir: {0}".format(stage_dir))
+ build_out_src = os.path.join(stage_dir, "spack-build-out.txt")
+ copy_files_to_artifacts(build_out_src, job_log_dir)
+
+
+def copy_test_logs_to_artifacts(test_stage, job_test_dir):
+ """
+ Copy test log file(s) to the given artifacts directory
+
+ Parameters:
+ test_stage (str): test stage path
+ job_test_dir (str): the destination artifacts test directory
+ """
+ tty.debug("test stage: {0}".format(test_stage))
+ if not os.path.exists(test_stage):
+ msg = "Cannot copy test logs: job test stage ({0}) does not exist"
+ tty.error(msg.format(test_stage))
+ return
+
+ copy_files_to_artifacts(os.path.join(test_stage, "*", "*.txt"), job_test_dir)
def download_and_extract_artifacts(url, work_dir):
@@ -1985,3 +1956,323 @@ def reproduce_ci_job(url, work_dir):
)
print("".join(inst_list))
+
+
+def process_command(cmd, cmd_args, repro_dir):
+ """
+ Create a script for and run the command. Copy the script to the
+ reproducibility directory.
+
+ Arguments:
+ cmd (str): name of the command being processed
+ cmd_args (list): string arguments to pass to the command
+ repro_dir (str): Job reproducibility directory
+
+ Returns: the exit code from processing the command
+ """
+ tty.debug("spack {0} arguments: {1}".format(cmd, cmd_args))
+
+ # Write the command to a shell script
+ script = "{0}.sh".format(cmd)
+ with open(script, "w") as fd:
+ fd.write("#!/bin/bash\n\n")
+ fd.write("\n# spack {0} command\n".format(cmd))
+ fd.write(" ".join(['"{0}"'.format(i) for i in cmd_args]))
+ fd.write("\n")
+
+ st = os.stat(script)
+ os.chmod(script, st.st_mode | stat.S_IEXEC)
+
+ copy_path = os.path.join(repro_dir, script)
+ shutil.copyfile(script, copy_path)
+
+ # Run the generated install.sh shell script as if it were being run in
+ # a login shell.
+ try:
+ cmd_process = subprocess.Popen(["bash", "./{0}".format(script)])
+ cmd_process.wait()
+ exit_code = cmd_process.returncode
+ except (ValueError, subprocess.CalledProcessError, OSError) as err:
+ tty.error("Encountered error running {0} script".format(cmd))
+ tty.error(err)
+ exit_code = 1
+
+ tty.debug("spack {0} exited {1}".format(cmd, exit_code))
+ return exit_code
+
+
+def create_buildcache(**kwargs):
+ """Create the buildcache at the provided mirror(s).
+
+ Arguments:
+ kwargs (dict): dictionary of arguments used to create the buildcache
+
+ List of recognized keys:
+
+ * "env" (spack.environment.Environment): the active environment
+ * "buildcache_mirror_url" (str or None): URL for the buildcache mirror
+ * "pipeline_mirror_url" (str or None): URL for the pipeline mirror
+ * "pr_pipeline" (bool): True if the CI job is for a PR
+ * "json_path" (str): path the the spec's JSON file
+ """
+ env = kwargs.get("env")
+ buildcache_mirror_url = kwargs.get("buildcache_mirror_url")
+ pipeline_mirror_url = kwargs.get("pipeline_mirror_url")
+ pr_pipeline = kwargs.get("pr_pipeline")
+ json_path = kwargs.get("json_path")
+
+ sign_binaries = pr_pipeline is False and can_sign_binaries()
+
+ # Create buildcache in either the main remote mirror, or in the
+ # per-PR mirror, if this is a PR pipeline
+ if buildcache_mirror_url:
+ push_mirror_contents(env, json_path, buildcache_mirror_url, sign_binaries)
+
+ # Create another copy of that buildcache in the per-pipeline
+ # temporary storage mirror (this is only done if either
+ # artifacts buildcache is enabled or a temporary storage url
+ # prefix is set)
+ if pipeline_mirror_url:
+ push_mirror_contents(env, json_path, pipeline_mirror_url, sign_binaries)
+
+
+def run_standalone_tests(**kwargs):
+ """Run stand-alone tests on the current spec.
+
+ Arguments:
+ kwargs (dict): dictionary of arguments used to run the tests
+
+ List of recognized keys:
+
+ * "cdash" (CDashHandler): (optional) cdash handler instance
+ * "fail_fast" (bool): (optional) terminate tests after the first failure
+ * "log_file" (str): (optional) test log file name if NOT CDash reporting
+ * "job_spec" (Spec): spec that was built
+ * "repro_dir" (str): reproduction directory
+ """
+ cdash = kwargs.get("cdash")
+ fail_fast = kwargs.get("fail_fast")
+ log_file = kwargs.get("log_file")
+
+ if cdash and log_file:
+ tty.msg("The test log file {0} option is ignored with CDash reporting".format(log_file))
+ log_file = None
+
+ # Error out but do NOT terminate if there are missing required arguments.
+ job_spec = kwargs.get("job_spec")
+ if not job_spec:
+ tty.error("Job spec is required to run stand-alone tests")
+ return
+
+ repro_dir = kwargs.get("repro_dir")
+ if not repro_dir:
+ tty.error("Reproduction directory is required for stand-alone tests")
+ return
+
+ test_args = [
+ "spack",
+ "-d",
+ "-v",
+ "test",
+ "run",
+ ]
+ if fail_fast:
+ test_args.append("--fail-fast")
+
+ if cdash:
+ test_args.extend(cdash.args())
+ else:
+ test_args.extend(["--log-format", "junit"])
+ if log_file:
+ test_args.extend(["--log-file", log_file])
+ test_args.append(job_spec.name)
+
+ tty.debug("Running {0} stand-alone tests".format(job_spec.name))
+ exit_code = process_command("test", test_args, repro_dir)
+
+ tty.debug("spack test exited {0}".format(exit_code))
+
+
+class CDashHandler(object):
+ """
+ Class for managing CDash data and processing.
+ """
+
+ def __init__(self, ci_cdash):
+ # start with the gitlab ci configuration
+ self.url = ci_cdash.get("url")
+ self.build_group = ci_cdash.get("build-group")
+ self.project = ci_cdash.get("project")
+ self.site = ci_cdash.get("site")
+
+ # grab the authorization token when available
+ self.auth_token = os.environ.get("SPACK_CDASH_AUTH_TOKEN")
+ if self.auth_token:
+ tty.verbose("Using CDash auth token from environment")
+
+ # append runner description to the site if available
+ runner = os.environ.get("CI_RUNNER_DESCRIPTION")
+ if runner:
+ self.site += " ({0})".format(runner)
+
+ # track current spec, if any
+ self.current_spec = None
+
+ def args(self):
+ return [
+ "--cdash-upload-url",
+ self.upload_url,
+ "--cdash-build",
+ self.build_name,
+ "--cdash-site",
+ self.site,
+ "--cdash-buildstamp",
+ self.build_stamp,
+ ]
+
+ @property # type: ignore
+ def build_name(self):
+ """Returns the CDash build name.
+
+ A name will be generated if the `current_spec` property is set;
+ otherwise, the value will be retrieved from the environment
+ through the `SPACK_CDASH_BUILD_NAME` variable.
+
+ Returns: (str) current spec's CDash build name."""
+ spec = self.current_spec
+ if spec:
+ build_name = "{0}@{1}%{2} hash={3} arch={4} ({5})".format(
+ spec.name,
+ spec.version,
+ spec.compiler,
+ spec.dag_hash(),
+ spec.architecture,
+ self.build_group,
+ )
+ tty.verbose(
+ "Generated CDash build name ({0}) from the {1}".format(build_name, spec.name)
+ )
+ return build_name
+
+ build_name = os.environ.get("SPACK_CDASH_BUILD_NAME")
+ tty.verbose("Using CDash build name ({0}) from the environment".format(build_name))
+ return build_name
+
+ @property # type: ignore
+ def build_stamp(self):
+ """Returns the CDash build stamp.
+
+ The one defined by SPACK_CDASH_BUILD_STAMP environment variable
+ is preferred due to the representation of timestamps; otherwise,
+ one will be built.
+
+ Returns: (str) current CDash build stamp"""
+ build_stamp = os.environ.get("SPACK_CDASH_BUILD_STAMP")
+ if build_stamp:
+ tty.verbose("Using build stamp ({0}) from the environment".format(build_stamp))
+ return build_stamp
+
+ build_stamp = cdash_build_stamp(self.build_group, time.time())
+ tty.verbose("Generated new build stamp ({0})".format(build_stamp))
+ return build_stamp
+
+ @property # type: ignore
+ @memoized
+ def project_enc(self):
+ tty.debug("Encoding project ({0}): {1})".format(type(self.project), self.project))
+ encode = urlencode({"project": self.project})
+ index = encode.find("=") + 1
+ return encode[index:]
+
+ @property
+ def upload_url(self):
+ url_format = "{0}/submit.php?project={1}"
+ return url_format.format(self.url, self.project_enc)
+
+ def copy_test_results(self, source, dest):
+ """Copy test results to artifacts directory."""
+ reports = fs.join_path(source, "*_Test*.xml")
+ copy_files_to_artifacts(reports, dest)
+
+ def create_buildgroup(self, opener, headers, url, group_name, group_type):
+ data = {"newbuildgroup": group_name, "project": self.project, "type": group_type}
+
+ enc_data = json.dumps(data).encode("utf-8")
+
+ request = Request(url, data=enc_data, headers=headers)
+
+ response = opener.open(request)
+ response_code = response.getcode()
+
+ if response_code not in [200, 201]:
+ msg = "Creating buildgroup failed (response code = {0})".format(response_code)
+ tty.warn(msg)
+ return None
+
+ response_text = response.read()
+ response_json = json.loads(response_text)
+ build_group_id = response_json["id"]
+
+ return build_group_id
+
+ def populate_buildgroup(self, job_names):
+ url = "{0}/api/v1/buildgroup.php".format(self.url)
+
+ headers = {
+ "Authorization": "Bearer {0}".format(self.auth_token),
+ "Content-Type": "application/json",
+ }
+
+ opener = build_opener(HTTPHandler)
+
+ parent_group_id = self.create_buildgroup(
+ opener,
+ headers,
+ url,
+ self.build_group,
+ "Daily",
+ )
+ group_id = self.create_buildgroup(
+ opener,
+ headers,
+ url,
+ "Latest {0}".format(self.build_group),
+ "Latest",
+ )
+
+ if not parent_group_id or not group_id:
+ msg = "Failed to create or retrieve buildgroups for {0}".format(self.build_group)
+ tty.warn(msg)
+ return
+
+ data = {
+ "dynamiclist": [
+ {
+ "match": name,
+ "parentgroupid": parent_group_id,
+ "site": self.site,
+ }
+ for name in job_names
+ ],
+ }
+
+ enc_data = json.dumps(data).encode("utf-8")
+
+ request = Request(url, data=enc_data, headers=headers)
+ request.get_method = lambda: "PUT"
+
+ response = opener.open(request)
+ response_code = response.getcode()
+
+ if response_code != 200:
+ msg = "Error response code ({0}) in populate_buildgroup".format(response_code)
+ tty.warn(msg)
+
+ def report_skipped(self, spec, directory_name, reason):
+ cli_args = self.args()
+ cli_args.extend(["package", [spec.name]])
+ it = iter(cli_args)
+ kv = {x.replace("--", "").replace("-", "_"): next(it) for x in it}
+
+ reporter = CDash(Bunch(**kv))
+ reporter.test_skipped_report(directory_name, spec, reason)
diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py
index 49e5c70019..bf7f1a5959 100644
--- a/lib/spack/spack/cmd/__init__.py
+++ b/lib/spack/spack/cmd/__init__.py
@@ -640,3 +640,8 @@ def find_environment(args):
return ev.Environment(env)
raise ev.SpackEnvironmentError("no environment in %s" % env)
+
+
+def first_line(docstring):
+ """Return the first line of the docstring."""
+ return docstring.split("\n")[0]
diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py
index 3087a7881e..7bb0497c81 100644
--- a/lib/spack/spack/cmd/ci.py
+++ b/lib/spack/spack/cmd/ci.py
@@ -6,13 +6,10 @@
import json
import os
import shutil
-import stat
-import subprocess
import sys
import tempfile
-from six.moves.urllib.parse import urlencode
-
+import llnl.util.filesystem as fs
import llnl.util.tty as tty
import spack.binary_distribution as bindist
@@ -34,6 +31,10 @@ CI_REBUILD_INSTALL_BASE_ARGS = ["spack", "-d", "-v"]
INSTALL_FAIL_CODE = 1
+def deindent(desc):
+ return desc.replace(" ", "")
+
+
def get_env_var(variable_name):
if variable_name in os.environ:
return os.environ.get(variable_name)
@@ -45,27 +46,35 @@ def setup_parser(subparser):
subparsers = subparser.add_subparsers(help="CI sub-commands")
# Dynamic generation of the jobs yaml from a spack environment
- generate = subparsers.add_parser("generate", help=ci_generate.__doc__)
+ generate = subparsers.add_parser(
+ "generate",
+ description=deindent(ci_generate.__doc__),
+ help=spack.cmd.first_line(ci_generate.__doc__),
+ )
generate.add_argument(
"--output-file",
default=None,
- help="Path to file where generated jobs file should be "
- + "written. The default is .gitlab-ci.yml in the root of the "
- + "repository.",
+ help="""pathname for the generated gitlab ci yaml file
+ Path to the file where generated jobs file should
+be written. Default is .gitlab-ci.yml in the root of
+the repository.""",
)
generate.add_argument(
"--copy-to",
default=None,
- help="Absolute path of additional location where generated jobs "
- + "yaml file should be copied. Default is not to copy.",
+ help="""path to additional directory for job files
+ This option provides an absolute path to a directory
+where the generated jobs yaml file should be copied.
+Default is not to copy.""",
)
generate.add_argument(
"--optimize",
action="store_true",
default=False,
- help="(Experimental) run the generated document through a series of "
- "optimization passes designed to reduce the size of the "
- "generated file.",
+ help="""(Experimental) optimize the gitlab yaml file for size
+ Run the generated document through a series of
+optimization passes designed to reduce the size
+of the generated file.""",
)
generate.add_argument(
"--dependencies",
@@ -86,53 +95,84 @@ def setup_parser(subparser):
action="store_true",
dest="prune_dag",
default=True,
- help="""Do not generate jobs for specs already up to
-date on the mirror""",
+ help="""skip up-to-date specs
+ Do not generate jobs for specs that are up-to-date
+on the mirror.""",
)
prune_group.add_argument(
"--no-prune-dag",
action="store_false",
dest="prune_dag",
default=True,
- help="""Generate jobs for specs already up to date
-on the mirror""",
+ help="""process up-to-date specs
+ Generate jobs for specs even when they are up-to-date
+on the mirror.""",
)
generate.add_argument(
"--check-index-only",
action="store_true",
dest="index_only",
default=False,
- help="""Spack always check specs against configured
-binary mirrors when generating the pipeline, regardless of whether or not
-DAG pruning is enabled. This flag controls whether it might attempt to
-fetch remote spec files directly (ensuring no spec is rebuilt if it
-is present on the mirror), or whether it should reduce pipeline generation time
-by assuming all remote buildcache indices are up to date and only use those
-to determine whether a given spec is up to date on mirrors. In the latter
-case, specs might be needlessly rebuilt if remote buildcache indices are out
-of date.""",
+ help="""only check spec state from buildcache indices
+ Spack always checks specs against configured binary
+mirrors, regardless of the DAG pruning option.
+ If enabled, Spack will assume all remote buildcache
+indices are up-to-date when assessing whether the spec
+on the mirror, if present, is up-to-date. This has the
+benefit of reducing pipeline generation time but at the
+potential cost of needlessly rebuilding specs when the
+indices are outdated.
+ If not enabled, Spack will fetch remote spec files
+directly to assess whether the spec on the mirror is
+up-to-date.""",
)
generate.add_argument(
"--artifacts-root",
default=None,
- help="""Path to root of artifacts directory. If provided, concrete
-environment files (spack.yaml, spack.lock) will be generated under this
-path and their location sent to generated child jobs via the custom job
-variable SPACK_CONCRETE_ENVIRONMENT_PATH.""",
+ help="""path to the root of the artifacts directory
+ If provided, concrete environment files (spack.yaml,
+spack.lock) will be generated under this directory.
+Their location will be passed to generated child jobs
+through the SPACK_CONCRETE_ENVIRONMENT_PATH variable.""",
)
generate.set_defaults(func=ci_generate)
# Rebuild the buildcache index associated with the mirror in the
# active, gitlab-enabled environment.
- index = subparsers.add_parser("rebuild-index", help=ci_reindex.__doc__)
+ index = subparsers.add_parser(
+ "rebuild-index",
+ description=deindent(ci_reindex.__doc__),
+ help=spack.cmd.first_line(ci_reindex.__doc__),
+ )
index.set_defaults(func=ci_reindex)
# Handle steps of a ci build/rebuild
- rebuild = subparsers.add_parser("rebuild", help=ci_rebuild.__doc__)
+ rebuild = subparsers.add_parser(
+ "rebuild",
+ description=deindent(ci_rebuild.__doc__),
+ help=spack.cmd.first_line(ci_rebuild.__doc__),
+ )
+ rebuild.add_argument(
+ "-t",
+ "--tests",
+ action="store_true",
+ default=False,
+ help="""run stand-alone tests after the build""",
+ )
+ rebuild.add_argument(
+ "--fail-fast",
+ action="store_true",
+ default=False,
+ help="""stop stand-alone tests after the first failure""",
+ )
rebuild.set_defaults(func=ci_rebuild)
# Facilitate reproduction of a failed CI build job
- reproduce = subparsers.add_parser("reproduce-build", help=ci_reproduce.__doc__)
+ reproduce = subparsers.add_parser(
+ "reproduce-build",
+ description=deindent(ci_reproduce.__doc__),
+ help=spack.cmd.first_line(ci_reproduce.__doc__),
+ )
reproduce.add_argument("job_url", help="Url of job artifacts bundle")
reproduce.add_argument(
"--working-dir",
@@ -144,12 +184,12 @@ variable SPACK_CONCRETE_ENVIRONMENT_PATH.""",
def ci_generate(args):
- """Generate jobs file from a spack environment file containing CI info.
- Before invoking this command, you can set the environment variable
- SPACK_CDASH_AUTH_TOKEN to contain the CDash authorization token
- for creating a build group for the generated workload and registering
- all generated jobs under that build group. If this environment
- variable is not set, no build group will be created on CDash."""
+ """Generate jobs file from a CI-aware spack file.
+
+ If you want to report the results on CDash, you will need to set
+ the SPACK_CDASH_AUTH_TOKEN before invoking this command. The
+ value must be the CDash authorization token needed to create a
+ build group and register all generated jobs under it."""
env = spack.cmd.require_active_env(cmd_name="ci generate")
output_file = args.output_file
@@ -190,8 +230,10 @@ def ci_generate(args):
def ci_reindex(args):
- """Rebuild the buildcache index associated with the mirror in the
- active, gitlab-enabled environment."""
+ """Rebuild the buildcache index for the remote mirror.
+
+ Use the active, gitlab-enabled environment to rebuild the buildcache
+ index for the associated mirror."""
env = spack.cmd.require_active_env(cmd_name="ci rebuild-index")
yaml_root = ev.config_dict(env.yaml)
@@ -206,17 +248,16 @@ def ci_reindex(args):
def ci_rebuild(args):
- """Check a single spec against the remote mirror, and rebuild it from
+ """Rebuild a spec if it is not on the remote mirror.
+
+ Check a single spec against the remote mirror, and rebuild it from
source if the mirror does not contain the hash."""
env = spack.cmd.require_active_env(cmd_name="ci rebuild")
# Make sure the environment is "gitlab-enabled", or else there's nothing
# to do.
yaml_root = ev.config_dict(env.yaml)
- gitlab_ci = None
- if "gitlab-ci" in yaml_root:
- gitlab_ci = yaml_root["gitlab-ci"]
-
+ gitlab_ci = yaml_root["gitlab-ci"] if "gitlab-ci" in yaml_root else None
if not gitlab_ci:
tty.die("spack ci rebuild requires an env containing gitlab-ci cfg")
@@ -231,6 +272,7 @@ def ci_rebuild(args):
# out as variables, or else provided by GitLab itself.
pipeline_artifacts_dir = get_env_var("SPACK_ARTIFACTS_ROOT")
job_log_dir = get_env_var("SPACK_JOB_LOG_DIR")
+ job_test_dir = get_env_var("SPACK_JOB_TEST_DIR")
repro_dir = get_env_var("SPACK_JOB_REPRO_DIR")
local_mirror_dir = get_env_var("SPACK_LOCAL_MIRROR_DIR")
concrete_env_dir = get_env_var("SPACK_CONCRETE_ENV_DIR")
@@ -240,7 +282,6 @@ def ci_rebuild(args):
root_spec = get_env_var("SPACK_ROOT_SPEC")
job_spec_pkg_name = get_env_var("SPACK_JOB_SPEC_PKG_NAME")
compiler_action = get_env_var("SPACK_COMPILER_ACTION")
- cdash_build_name = get_env_var("SPACK_CDASH_BUILD_NAME")
spack_pipeline_type = get_env_var("SPACK_PIPELINE_TYPE")
remote_mirror_override = get_env_var("SPACK_REMOTE_MIRROR_OVERRIDE")
remote_mirror_url = get_env_var("SPACK_REMOTE_MIRROR_URL")
@@ -249,6 +290,7 @@ def ci_rebuild(args):
ci_project_dir = get_env_var("CI_PROJECT_DIR")
pipeline_artifacts_dir = os.path.join(ci_project_dir, pipeline_artifacts_dir)
job_log_dir = os.path.join(ci_project_dir, job_log_dir)
+ job_test_dir = os.path.join(ci_project_dir, job_test_dir)
repro_dir = os.path.join(ci_project_dir, repro_dir)
local_mirror_dir = os.path.join(ci_project_dir, local_mirror_dir)
concrete_env_dir = os.path.join(ci_project_dir, concrete_env_dir)
@@ -263,23 +305,15 @@ def ci_rebuild(args):
# Query the environment manifest to find out whether we're reporting to a
# CDash instance, and if so, gather some information from the manifest to
# support that task.
- enable_cdash = False
- if "cdash" in yaml_root:
- enable_cdash = True
- ci_cdash = yaml_root["cdash"]
- job_spec_buildgroup = ci_cdash["build-group"]
- cdash_base_url = ci_cdash["url"]
- cdash_project = ci_cdash["project"]
- proj_enc = urlencode({"project": cdash_project})
- eq_idx = proj_enc.find("=") + 1
- cdash_project_enc = proj_enc[eq_idx:]
- cdash_site = ci_cdash["site"]
- tty.debug("cdash_base_url = {0}".format(cdash_base_url))
- tty.debug("cdash_project = {0}".format(cdash_project))
- tty.debug("cdash_project_enc = {0}".format(cdash_project_enc))
- tty.debug("cdash_build_name = {0}".format(cdash_build_name))
- tty.debug("cdash_site = {0}".format(cdash_site))
- tty.debug("job_spec_buildgroup = {0}".format(job_spec_buildgroup))
+ cdash_handler = spack_ci.CDashHandler(yaml_root.get("cdash")) if "cdash" in yaml_root else None
+ if cdash_handler:
+ tty.debug("cdash url = {0}".format(cdash_handler.url))
+ tty.debug("cdash project = {0}".format(cdash_handler.project))
+ tty.debug("cdash project_enc = {0}".format(cdash_handler.project_enc))
+ tty.debug("cdash build_name = {0}".format(cdash_handler.build_name))
+ tty.debug("cdash build_stamp = {0}".format(cdash_handler.build_stamp))
+ tty.debug("cdash site = {0}".format(cdash_handler.site))
+ tty.debug("cdash build_group = {0}".format(cdash_handler.build_group))
# Is this a pipeline run on a spack PR or a merge to develop? It might
# be neither, e.g. a pipeline run on some environment repository.
@@ -344,6 +378,9 @@ def ci_rebuild(args):
if os.path.exists(job_log_dir):
shutil.rmtree(job_log_dir)
+ if os.path.exists(job_test_dir):
+ shutil.rmtree(job_test_dir)
+
if os.path.exists(repro_dir):
shutil.rmtree(repro_dir)
@@ -351,6 +388,7 @@ def ci_rebuild(args):
# need for storing artifacts. The cdash_report directory will be
# created internally if needed.
os.makedirs(job_log_dir)
+ os.makedirs(job_test_dir)
os.makedirs(repro_dir)
# Copy the concrete environment files to the repro directory so we can
@@ -468,6 +506,7 @@ def ci_rebuild(args):
install_args.extend(
[
"install",
+ "--show-log-on-error", # Print full log on fails
"--keep-stage",
]
)
@@ -477,22 +516,9 @@ def ci_rebuild(args):
if not verify_binaries:
install_args.append("--no-check-signature")
- if enable_cdash:
+ if cdash_handler:
# Add additional arguments to `spack install` for CDash reporting.
- cdash_upload_url = "{0}/submit.php?project={1}".format(cdash_base_url, cdash_project_enc)
-
- install_args.extend(
- [
- "--cdash-upload-url",
- cdash_upload_url,
- "--cdash-build",
- cdash_build_name,
- "--cdash-site",
- cdash_site,
- "--cdash-track",
- job_spec_buildgroup,
- ]
- )
+ install_args.extend(cdash_handler.args())
# A compiler action of 'FIND_ANY' means we are building a bootstrap
# compiler or one of its deps.
@@ -506,29 +532,7 @@ def ci_rebuild(args):
install_args.extend(["-f", job_spec_json_path])
tty.debug("Installing {0} from source".format(job_spec.name))
- tty.debug("spack install arguments: {0}".format(install_args))
-
- # Write the install command to a shell script
- with open("install.sh", "w") as fd:
- fd.write("#!/bin/bash\n\n")
- fd.write("\n# spack install command\n")
- fd.write(" ".join(['"{0}"'.format(i) for i in install_args]))
- fd.write("\n")
-
- st = os.stat("install.sh")
- os.chmod("install.sh", st.st_mode | stat.S_IEXEC)
-
- install_copy_path = os.path.join(repro_dir, "install.sh")
- shutil.copyfile("install.sh", install_copy_path)
-
- # Run the generated install.sh shell script
- try:
- install_process = subprocess.Popen(["bash", "./install.sh"])
- install_process.wait()
- install_exit_code = install_process.returncode
- except (ValueError, subprocess.CalledProcessError, OSError) as inst:
- tty.error("Encountered error running install script")
- tty.error(inst)
+ install_exit_code = spack_ci.process_command("install", install_args, repro_dir)
# Now do the post-install tasks
tty.debug("spack install exited {0}".format(install_exit_code))
@@ -564,7 +568,7 @@ def ci_rebuild(args):
extra_args={"ContentType": "text/plain"},
)
except Exception as err:
- # If we got some kind of S3 (access denied or other connection
+ # If there is an S3 error (e.g., access denied or connection
# error), the first non boto-specific class in the exception
# hierarchy is Exception. Just print a warning and return
msg = "Error writing to broken specs list {0}: {1}".format(broken_spec_path, err)
@@ -576,28 +580,79 @@ def ci_rebuild(args):
# any logs from the staging directory to artifacts now
spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir)
+ # If the installation succeeded and we're running stand-alone tests for
+ # the package, run them and copy the output. Failures of any kind should
+ # *not* terminate the build process or preclude creating the build cache.
+ broken_tests = (
+ "broken-tests-packages" in gitlab_ci
+ and job_spec.name in gitlab_ci["broken-tests-packages"]
+ )
+ reports_dir = fs.join_path(os.getcwd(), "cdash_report")
+ if args.tests and broken_tests:
+ tty.warn(
+ "Unable to run stand-alone tests since listed in "
+ "gitlab-ci's 'broken-tests-packages'"
+ )
+ if cdash_handler:
+ msg = "Package is listed in gitlab-ci's broken-tests-packages"
+ cdash_handler.report_skipped(job_spec, reports_dir, reason=msg)
+ cdash_handler.copy_test_results(reports_dir, job_test_dir)
+ elif args.tests:
+ if install_exit_code == 0:
+ try:
+ # First ensure we will use a reasonable test stage directory
+ stage_root = os.path.dirname(str(job_spec.package.stage.path))
+ test_stage = fs.join_path(stage_root, "spack-standalone-tests")
+ tty.debug("Configuring test_stage to {0}".format(test_stage))
+ config_test_path = "config:test_stage:{0}".format(test_stage)
+ cfg.add(config_test_path, scope=cfg.default_modify_scope())
+
+ # Run the tests, resorting to junit results if not using cdash
+ log_file = (
+ None if cdash_handler else fs.join_path(test_stage, "ci-test-results.xml")
+ )
+ spack_ci.run_standalone_tests(
+ cdash=cdash_handler,
+ job_spec=job_spec,
+ fail_fast=args.fail_fast,
+ log_file=log_file,
+ repro_dir=repro_dir,
+ )
+
+ except Exception as err:
+ # If there is any error, just print a warning.
+ msg = "Error processing stand-alone tests: {0}".format(str(err))
+ tty.warn(msg)
+
+ finally:
+ # Copy the test log/results files
+ spack_ci.copy_test_logs_to_artifacts(test_stage, job_test_dir)
+ if cdash_handler:
+ cdash_handler.copy_test_results(reports_dir, job_test_dir)
+ elif log_file:
+ spack_ci.copy_files_to_artifacts(log_file, job_test_dir)
+ else:
+ tty.warn("No recognized test results reporting option")
+
+ else:
+ tty.warn("Unable to run stand-alone tests due to unsuccessful " "installation")
+ if cdash_handler:
+ msg = "Failed to install the package"
+ cdash_handler.report_skipped(job_spec, reports_dir, reason=msg)
+ cdash_handler.copy_test_results(reports_dir, job_test_dir)
+
# If the install succeeded, create a buildcache entry for this job spec
# and push it to one or more mirrors. If the install did not succeed,
# print out some instructions on how to reproduce this build failure
# outside of the pipeline environment.
if install_exit_code == 0:
- can_sign = spack_ci.can_sign_binaries()
- sign_binaries = can_sign and spack_is_pr_pipeline is False
-
- # Create buildcache in either the main remote mirror, or in the
- # per-PR mirror, if this is a PR pipeline
- if buildcache_mirror_url:
- spack_ci.push_mirror_contents(
- env, job_spec_json_path, buildcache_mirror_url, sign_binaries
- )
-
- # Create another copy of that buildcache in the per-pipeline
- # temporary storage mirror (this is only done if either
- # artifacts buildcache is enabled or a temporary storage url
- # prefix is set)
- if pipeline_mirror_url:
- spack_ci.push_mirror_contents(
- env, job_spec_json_path, pipeline_mirror_url, sign_binaries
+ if buildcache_mirror_url or pipeline_mirror_url:
+ spack_ci.create_buildcache(
+ env=env,
+ buildcache_mirror_url=buildcache_mirror_url,
+ pipeline_mirror_url=pipeline_mirror_url,
+ pr_pipeline=spack_is_pr_pipeline,
+ json_path=job_spec_json_path,
)
# If this is a develop pipeline, check if the spec that we just built is
@@ -611,13 +666,11 @@ def ci_rebuild(args):
try:
web_util.remove_url(broken_spec_path)
except Exception as err:
- # If we got some kind of S3 (access denied or other connection
+ # If there is an S3 error (e.g., access denied or connection
# error), the first non boto-specific class in the exception
- # hierarchy is Exception. Just print a warning and return
- msg = "Error removing {0} from broken specs list: {1}".format(
- broken_spec_path, err
- )
- tty.warn(msg)
+ # hierarchy is Exception. Just print a warning and return.
+ msg = "Error removing {0} from broken specs list: {1}"
+ tty.warn(msg.format(broken_spec_path, err))
else:
tty.debug("spack install exited non-zero, will not create buildcache")
@@ -654,6 +707,10 @@ If this project does not have public pipelines, you will need to first:
def ci_reproduce(args):
+ """Generate instructions for reproducing the spec rebuild job.
+
+ Artifacts of the provided gitlab pipeline rebuild job's URL will be
+ used to derive instructions for reproducing the build locally."""
job_url = args.job_url
work_dir = args.working_dir
diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py
index f74b164b31..dd3d5b8c9c 100644
--- a/lib/spack/spack/cmd/install.py
+++ b/lib/spack/spack/cmd/install.py
@@ -236,6 +236,7 @@ def install_specs(specs, install_kwargs, cli_args):
except spack.build_environment.InstallError as e:
if cli_args.show_log_on_error:
e.print_context()
+ assert e.pkg, "Expected InstallError to include the associated package"
if not os.path.exists(e.pkg.build_log_path):
tty.error("'spack install' created no log.")
else:
diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py
index ec062f259a..fcd72a123e 100644
--- a/lib/spack/spack/cmd/test.py
+++ b/lib/spack/spack/cmd/test.py
@@ -29,17 +29,14 @@ section = "admin"
level = "long"
-def first_line(docstring):
- """Return the first line of the docstring."""
- return docstring.split("\n")[0]
-
-
def setup_parser(subparser):
sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="test_command")
# Run
run_parser = sp.add_parser(
- "run", description=test_run.__doc__, help=first_line(test_run.__doc__)
+ "run",
+ description=test_run.__doc__,
+ help=spack.cmd.first_line(test_run.__doc__),
)
alias_help_msg = "Provide an alias for this test-suite"
@@ -83,7 +80,9 @@ def setup_parser(subparser):
# List
list_parser = sp.add_parser(
- "list", description=test_list.__doc__, help=first_line(test_list.__doc__)
+ "list",
+ description=test_list.__doc__,
+ help=spack.cmd.first_line(test_list.__doc__),
)
list_parser.add_argument(
"-a",
@@ -97,7 +96,9 @@ def setup_parser(subparser):
# Find
find_parser = sp.add_parser(
- "find", description=test_find.__doc__, help=first_line(test_find.__doc__)
+ "find",
+ description=test_find.__doc__,
+ help=spack.cmd.first_line(test_find.__doc__),
)
find_parser.add_argument(
"filter",
@@ -107,7 +108,9 @@ def setup_parser(subparser):
# Status
status_parser = sp.add_parser(
- "status", description=test_status.__doc__, help=first_line(test_status.__doc__)
+ "status",
+ description=test_status.__doc__,
+ help=spack.cmd.first_line(test_status.__doc__),
)
status_parser.add_argument(
"names", nargs=argparse.REMAINDER, help="Test suites for which to print status"
@@ -115,7 +118,9 @@ def setup_parser(subparser):
# Results
results_parser = sp.add_parser(
- "results", description=test_results.__doc__, help=first_line(test_results.__doc__)
+ "results",
+ description=test_results.__doc__,
+ help=spack.cmd.first_line(test_results.__doc__),
)
results_parser.add_argument(
"-l", "--logs", action="store_true", help="print the test log for each matching package"
@@ -142,7 +147,9 @@ def setup_parser(subparser):
# Remove
remove_parser = sp.add_parser(
- "remove", description=test_remove.__doc__, help=first_line(test_remove.__doc__)
+ "remove",
+ description=test_remove.__doc__,
+ help=spack.cmd.first_line(test_remove.__doc__),
)
arguments.add_common_arguments(remove_parser, ["yes_to_all"])
remove_parser.add_argument(
@@ -191,6 +198,16 @@ environment variables:
matching = spack.store.db.query_local(spec, hashes=hashes)
if spec and not matching:
tty.warn("No installed packages match spec %s" % spec)
+ """
+ TODO: Need to write out a log message and/or CDASH Testing
+ output that package not installed IF continue to process
+ these issues here.
+
+ if args.log_format:
+ # Proceed with the spec assuming the test process
+ # to ensure report package as skipped (e.g., for CI)
+ specs_to_test.append(spec)
+ """
specs_to_test.extend(matching)
# test_stage_dir
diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py
index 105b60642f..a0869cf533 100644
--- a/lib/spack/spack/environment/environment.py
+++ b/lib/spack/spack/environment/environment.py
@@ -1694,7 +1694,7 @@ class Environment(object):
spec for already concretized but not yet installed specs.
"""
# use a transaction to avoid overhead of repeated calls
- # to `package.installed`
+ # to `package.spec.installed`
with spack.store.db.read_transaction():
concretized = dict(self.concretized_specs())
for spec in self.user_specs:
diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py
index 861db1a556..da2b73032e 100644
--- a/lib/spack/spack/install_test.py
+++ b/lib/spack/spack/install_test.py
@@ -12,6 +12,7 @@ import sys
import six
import llnl.util.filesystem as fs
+import llnl.util.tty as tty
import spack.error
import spack.paths
@@ -180,6 +181,9 @@ class TestSuite(object):
if spec.external and not externals:
status = "SKIPPED"
skipped += 1
+ elif not spec.installed:
+ status = "SKIPPED"
+ skipped += 1
else:
status = "NO-TESTS"
untested += 1
@@ -187,6 +191,7 @@ class TestSuite(object):
self.write_test_result(spec, status)
except BaseException as exc:
self.fails += 1
+ tty.debug("Test failure: {0}".format(str(exc)))
if isinstance(exc, (SyntaxError, TestSuiteSpecError)):
# Create the test log file and report the error.
self.ensure_stage()
diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py
index da5181c473..d97db35535 100644
--- a/lib/spack/spack/installer.py
+++ b/lib/spack/spack/installer.py
@@ -820,7 +820,7 @@ class PackageInstaller(object):
if spack.store.db.prefix_failed(dep):
action = "'spack install' the dependency"
msg = "{0} is marked as an install failure: {1}".format(dep_id, action)
- raise InstallError(err.format(request.pkg_id, msg))
+ raise InstallError(err.format(request.pkg_id, msg), pkg=dep_pkg)
# Attempt to get a read lock to ensure another process does not
# uninstall the dependency while the requested spec is being
@@ -828,7 +828,7 @@ class PackageInstaller(object):
ltype, lock = self._ensure_locked("read", dep_pkg)
if lock is None:
msg = "{0} is write locked by another process".format(dep_id)
- raise InstallError(err.format(request.pkg_id, msg))
+ raise InstallError(err.format(request.pkg_id, msg), pkg=request.pkg)
# Flag external and upstream packages as being installed
if dep_pkg.spec.external or dep_pkg.spec.installed_upstream:
@@ -883,6 +883,7 @@ class PackageInstaller(object):
"Install prefix collision for {0}".format(task.pkg_id),
long_msg="Prefix directory {0} already used by another "
"installed spec.".format(task.pkg.spec.prefix),
+ pkg=task.pkg,
)
# Make sure the installation directory is in the desired state
@@ -1571,7 +1572,8 @@ class PackageInstaller(object):
raise InstallError(
"Cannot proceed with {0}: {1} uninstalled {2}: {3}".format(
pkg_id, task.priority, dep_str, ",".join(task.uninstalled_deps)
- )
+ ),
+ pkg=pkg,
)
# Skip the installation if the spec is not being installed locally
@@ -1596,7 +1598,7 @@ class PackageInstaller(object):
spack.hooks.on_install_failure(task.request.pkg.spec)
if self.fail_fast:
- raise InstallError(fail_fast_err)
+ raise InstallError(fail_fast_err, pkg=pkg)
continue
@@ -1718,7 +1720,7 @@ class PackageInstaller(object):
)
# Terminate if requested to do so on the first failure.
if self.fail_fast:
- raise InstallError("{0}: {1}".format(fail_fast_err, str(exc)))
+ raise InstallError("{0}: {1}".format(fail_fast_err, str(exc)), pkg=pkg)
# Terminate at this point if the single explicit spec has
# failed to install.
@@ -1727,7 +1729,7 @@ class PackageInstaller(object):
# Track explicit spec id and error to summarize when done
if task.explicit:
- failed_explicits.append((pkg_id, str(exc)))
+ failed_explicits.append((pkg, pkg_id, str(exc)))
finally:
# Remove the install prefix if anything went wrong during
@@ -1750,19 +1752,38 @@ class PackageInstaller(object):
# Ensure we properly report if one or more explicit specs failed
# or were not installed when should have been.
missing = [
- request.pkg_id
+ (request.pkg, request.pkg_id)
for request in self.build_requests
if request.install_args.get("install_package") and request.pkg_id not in self.installed
]
+
if failed_explicits or missing:
- for pkg_id, err in failed_explicits:
+ for _, pkg_id, err in failed_explicits:
tty.error("{0}: {1}".format(pkg_id, err))
- for pkg_id in missing:
+ for _, pkg_id in missing:
tty.error("{0}: Package was not installed".format(pkg_id))
+ pkg = None
+ if len(failed_explicits) > 0:
+ pkg = failed_explicits[0][0]
+ ids = [pkg_id for _, pkg_id, _ in failed_explicits]
+ tty.debug(
+ "Associating installation failure with first failed "
+ "explicit package ({0}) from {1}".format(ids[0], ", ".join(ids))
+ )
+
+ if not pkg and len(missing) > 0:
+ pkg = missing[0][0]
+ ids = [pkg_id for _, pkg_id in missing]
+ tty.debug(
+ "Associating installation failure with first "
+ "missing package ({0}) from {1}".format(ids[0], ", ".join(ids))
+ )
+
raise InstallError(
- "Installation request failed. Refer to " "reported errors for failing package(s)."
+ "Installation request failed. Refer to reported errors for failing package(s).",
+ pkg=pkg,
)
@@ -2060,7 +2081,7 @@ class BuildTask(object):
# queue.
if status == STATUS_REMOVED:
msg = "Cannot create a build task for {0} with status '{1}'"
- raise InstallError(msg.format(self.pkg_id, status))
+ raise InstallError(msg.format(self.pkg_id, status), pkg=pkg)
self.status = status
@@ -2351,10 +2372,15 @@ class BuildRequest(object):
class InstallError(spack.error.SpackError):
- """Raised when something goes wrong during install or uninstall."""
+ """Raised when something goes wrong during install or uninstall.
- def __init__(self, message, long_msg=None):
+ The error can be annotated with a ``pkg`` attribute to allow the
+ caller to get the package for which the exception was raised.
+ """
+
+ def __init__(self, message, long_msg=None, pkg=None):
super(InstallError, self).__init__(message, long_msg)
+ self.pkg = pkg
class BadInstallPhase(InstallError):
diff --git a/lib/spack/spack/package_base.py b/lib/spack/spack/package_base.py
index 441563eeba..81801df9a5 100644
--- a/lib/spack/spack/package_base.py
+++ b/lib/spack/spack/package_base.py
@@ -2840,6 +2840,10 @@ def test_process(pkg, kwargs):
print_test_message(logger, "Skipped tests for external package", verbose)
return
+ if not pkg.spec.installed:
+ print_test_message(logger, "Skipped not installed package", verbose)
+ return
+
# run test methods from the package and all virtuals it
# provides virtuals have to be deduped by name
v_names = list(set([vspec.name for vspec in pkg.virtuals_provided]))
@@ -2910,6 +2914,9 @@ def test_process(pkg, kwargs):
# non-pass-only methods
if ran_actual_test_function:
fsys.touch(pkg.tested_file)
+ # log one more test message to provide a completion timestamp
+ # for CDash reporting
+ tty.msg("Completed testing")
else:
print_test_message(logger, "No tests to run", verbose)
diff --git a/lib/spack/spack/report.py b/lib/spack/spack/report.py
index ec81502887..8d4fb2b81d 100644
--- a/lib/spack/spack/report.py
+++ b/lib/spack/spack/report.py
@@ -245,6 +245,7 @@ class collect_info(object):
self.cls = cls
self.function = function
self.filename = None
+ self.ctest_parsing = getattr(args, "ctest_parsing", False)
if args.cdash_upload_url:
self.format_name = "cdash"
self.filename = "cdash_report"
@@ -271,10 +272,10 @@ class collect_info(object):
def __exit__(self, exc_type, exc_val, exc_tb):
if self.format_name:
- # Close the collector and restore the
- # original PackageInstaller._install_task
+ # Close the collector and restore the original function
self.collector.__exit__(exc_type, exc_val, exc_tb)
report_data = {"specs": self.collector.specs}
+ report_data["ctest-parsing"] = self.ctest_parsing
report_fn = getattr(self.report_writer, "%s_report" % self.type)
report_fn(self.filename, report_data)
diff --git a/lib/spack/spack/reporters/cdash.py b/lib/spack/spack/reporters/cdash.py
index 6107aaed50..2c57306412 100644
--- a/lib/spack/spack/reporters/cdash.py
+++ b/lib/spack/spack/reporters/cdash.py
@@ -23,8 +23,10 @@ from llnl.util.filesystem import working_dir
import spack.build_environment
import spack.fetch_strategy
import spack.package_base
+import spack.platforms
from spack.error import SpackError
from spack.reporter import Reporter
+from spack.reporters.extract import extract_test_parts
from spack.util.crypto import checksum
from spack.util.executable import which
from spack.util.log_parse import parse_log_events
@@ -46,6 +48,11 @@ cdash_phases = set(map_phases_to_cdash.values())
cdash_phases.add("update")
+def build_stamp(track, timestamp):
+ buildstamp_format = "%Y%m%d-%H%M-{0}".format(track)
+ return time.strftime(buildstamp_format, time.localtime(timestamp))
+
+
class CDash(Reporter):
"""Generate reports of spec installations for CDash.
@@ -80,6 +87,9 @@ class CDash(Reporter):
packages = args.spec
elif getattr(args, "specs", ""):
packages = args.specs
+ elif getattr(args, "package", ""):
+ # Ensure CI 'spack test run' can output CDash results
+ packages = args.package
else:
packages = []
for file in args.specfiles:
@@ -90,29 +100,36 @@ class CDash(Reporter):
self.base_buildname = args.cdash_build or self.install_command
self.site = args.cdash_site or socket.gethostname()
self.osname = platform.system()
+ self.osrelease = platform.release()
+ self.target = spack.platforms.host().target("default_target")
self.endtime = int(time.time())
- if args.cdash_buildstamp:
- self.buildstamp = args.cdash_buildstamp
- else:
- buildstamp_format = "%Y%m%d-%H%M-{0}".format(args.cdash_track)
- self.buildstamp = time.strftime(buildstamp_format, time.localtime(self.endtime))
+ self.buildstamp = (
+ args.cdash_buildstamp
+ if args.cdash_buildstamp
+ else build_stamp(args.cdash_track, self.endtime)
+ )
self.buildIds = collections.OrderedDict()
self.revision = ""
git = which("git")
with working_dir(spack.paths.spack_root):
self.revision = git("rev-parse", "HEAD", output=str).strip()
+ self.generator = "spack-{0}".format(spack.main.get_version())
self.multiple_packages = False
+ def report_build_name(self, pkg_name):
+ return (
+ "{0} - {1}".format(self.base_buildname, pkg_name)
+ if self.multiple_packages
+ else self.base_buildname
+ )
+
def build_report_for_package(self, directory_name, package, duration):
if "stdout" not in package:
# Skip reporting on packages that did not generate any output.
return
self.current_package_name = package["name"]
- if self.multiple_packages:
- self.buildname = "{0} - {1}".format(self.base_buildname, package["name"])
- else:
- self.buildname = self.base_buildname
+ self.buildname = self.report_build_name(self.current_package_name)
report_data = self.initialize_report(directory_name)
for phase in cdash_phases:
report_data[phase] = {}
@@ -228,6 +245,7 @@ class CDash(Reporter):
# Do an initial scan to determine if we are generating reports for more
# than one package. When we're only reporting on a single package we
# do not explicitly include the package's name in the CDash build name.
+ self.multipe_packages = False
num_packages = 0
for spec in input_data["specs"]:
# Do not generate reports for packages that were installed
@@ -255,27 +273,19 @@ class CDash(Reporter):
self.build_report_for_package(directory_name, package, duration)
self.finalize_report()
- def test_report_for_package(self, directory_name, package, duration):
- if "stdout" not in package:
- # Skip reporting on packages that did not generate any output.
- return
-
- self.current_package_name = package["name"]
- self.buildname = "{0} - {1}".format(self.base_buildname, package["name"])
-
- report_data = self.initialize_report(directory_name)
+ def extract_ctest_test_data(self, package, phases, report_data):
+ """Extract ctest test data for the package."""
+ # Track the phases we perform so we know what reports to create.
+ # We always report the update step because this is how we tell CDash
+ # what revision of Spack we are using.
+ assert "update" in phases
- for phase in ("test", "update"):
+ for phase in phases:
report_data[phase] = {}
report_data[phase]["loglines"] = []
report_data[phase]["status"] = 0
report_data[phase]["endtime"] = self.endtime
- # Track the phases we perform so we know what reports to create.
- # We always report the update step because this is how we tell CDash
- # what revision of Spack we are using.
- phases_encountered = ["test", "update"]
-
# Generate a report for this package.
# The first line just says "Testing package name-hash"
report_data["test"]["loglines"].append(
@@ -284,8 +294,7 @@ class CDash(Reporter):
for line in package["stdout"].splitlines()[1:]:
report_data["test"]["loglines"].append(xml.sax.saxutils.escape(line))
- self.starttime = self.endtime - duration
- for phase in phases_encountered:
+ for phase in phases:
report_data[phase]["starttime"] = self.starttime
report_data[phase]["log"] = "\n".join(report_data[phase]["loglines"])
errors, warnings = parse_log_events(report_data[phase]["loglines"])
@@ -326,6 +335,19 @@ class CDash(Reporter):
if phase == "update":
report_data[phase]["revision"] = self.revision
+ def extract_standalone_test_data(self, package, phases, report_data):
+ """Extract stand-alone test outputs for the package."""
+
+ testing = {}
+ report_data["testing"] = testing
+ testing["starttime"] = self.starttime
+ testing["endtime"] = self.starttime
+ testing["generator"] = self.generator
+ testing["parts"] = extract_test_parts(package["name"], package["stdout"].splitlines())
+
+ def report_test_data(self, directory_name, package, phases, report_data):
+ """Generate and upload the test report(s) for the package."""
+ for phase in phases:
# Write the report.
report_name = phase.capitalize() + ".xml"
report_file_name = package["name"] + "_" + report_name
@@ -333,7 +355,7 @@ class CDash(Reporter):
with codecs.open(phase_report, "w", "utf-8") as f:
env = spack.tengine.make_environment()
- if phase != "update":
+ if phase not in ["update", "testing"]:
# Update.xml stores site information differently
# than the rest of the CTest XML files.
site_template = posixpath.join(self.template_dir, "Site.xml")
@@ -343,18 +365,65 @@ class CDash(Reporter):
phase_template = posixpath.join(self.template_dir, report_name)
t = env.get_template(phase_template)
f.write(t.render(report_data))
+
+ tty.debug("Preparing to upload {0}".format(phase_report))
self.upload(phase_report)
+ def test_report_for_package(self, directory_name, package, duration, ctest_parsing=False):
+ if "stdout" not in package:
+ # Skip reporting on packages that did not generate any output.
+ tty.debug("Skipping report for {0}: No generated output".format(package["name"]))
+ return
+
+ self.current_package_name = package["name"]
+ if self.base_buildname == self.install_command:
+ # The package list is NOT all that helpful in this case
+ self.buildname = "{0}-{1}".format(self.current_package_name, package["id"])
+ else:
+ self.buildname = self.report_build_name(self.current_package_name)
+ self.starttime = self.endtime - duration
+
+ report_data = self.initialize_report(directory_name)
+ report_data["hostname"] = socket.gethostname()
+ if ctest_parsing:
+ phases = ["test", "update"]
+ self.extract_ctest_test_data(package, phases, report_data)
+ else:
+ phases = ["testing"]
+ self.extract_standalone_test_data(package, phases, report_data)
+
+ self.report_test_data(directory_name, package, phases, report_data)
+
def test_report(self, directory_name, input_data):
- # Generate reports for each package in each spec.
+ """Generate reports for each package in each spec."""
+ tty.debug("Processing test report")
for spec in input_data["specs"]:
duration = 0
if "time" in spec:
duration = int(spec["time"])
for package in spec["packages"]:
- self.test_report_for_package(directory_name, package, duration)
+ self.test_report_for_package(
+ directory_name,
+ package,
+ duration,
+ input_data["ctest-parsing"],
+ )
+
self.finalize_report()
+ def test_skipped_report(self, directory_name, spec, reason=None):
+ output = "Skipped {0} package".format(spec.name)
+ if reason:
+ output += "\n{0}".format(reason)
+
+ package = {
+ "name": spec.name,
+ "id": spec.dag_hash(),
+ "result": "skipped",
+ "stdout": output,
+ }
+ self.test_report_for_package(directory_name, package, duration=0.0, ctest_parsing=False)
+
def concretization_report(self, directory_name, msg):
self.buildname = self.base_buildname
report_data = self.initialize_report(directory_name)
@@ -384,12 +453,16 @@ class CDash(Reporter):
report_data["buildname"] = self.buildname
report_data["buildstamp"] = self.buildstamp
report_data["install_command"] = self.install_command
+ report_data["generator"] = self.generator
report_data["osname"] = self.osname
+ report_data["osrelease"] = self.osrelease
report_data["site"] = self.site
+ report_data["target"] = self.target
return report_data
def upload(self, filename):
if not self.cdash_upload_url:
+ print("Cannot upload {0} due to missing upload url".format(filename))
return
# Compute md5 checksum for the contents of this file.
@@ -412,7 +485,7 @@ class CDash(Reporter):
request.add_header("Authorization", "Bearer {0}".format(self.authtoken))
try:
# By default, urllib2 only support GET and POST.
- # CDash needs expects this file to be uploaded via PUT.
+ # CDash expects this file to be uploaded via PUT.
request.get_method = lambda: "PUT"
response = opener.open(request)
if self.current_package_name not in self.buildIds:
@@ -428,13 +501,13 @@ class CDash(Reporter):
def finalize_report(self):
if self.buildIds:
- print("View your build results here:")
+ tty.msg("View your build results here:")
for package_name, buildid in iteritems(self.buildIds):
# Construct and display a helpful link if CDash responded with
# a buildId.
build_url = self.cdash_upload_url
build_url = build_url[0 : build_url.find("submit.php")]
build_url += "buildSummary.php?buildid={0}".format(buildid)
- print("{0}: {1}".format(package_name, build_url))
+ tty.msg("{0}: {1}".format(package_name, build_url))
if not self.success:
raise SpackError("Errors encountered, see above for more details")
diff --git a/lib/spack/spack/reporters/extract.py b/lib/spack/spack/reporters/extract.py
new file mode 100644
index 0000000000..5814d791a5
--- /dev/null
+++ b/lib/spack/spack/reporters/extract.py
@@ -0,0 +1,212 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os
+import re
+import xml.sax.saxutils
+from datetime import datetime
+
+import llnl.util.tty as tty
+
+# The keys here represent the only recognized (ctest/cdash) status values
+completed = {
+ "failed": "Completed",
+ "passed": "Completed",
+ "notrun": "No tests to run",
+}
+
+log_regexp = re.compile(r"^==> \[([0-9:.\-]*)(?:, [0-9]*)?\] (.*)")
+returns_regexp = re.compile(r"\[([0-9 ,]*)\]")
+
+skip_msgs = ["Testing package", "Results for", "Detected the following"]
+skip_regexps = [re.compile(r"{0}".format(msg)) for msg in skip_msgs]
+
+status_values = ["FAILED", "PASSED", "NO-TESTS"]
+status_regexps = [re.compile(r"^({0})".format(stat)) for stat in status_values]
+
+
+def add_part_output(part, line):
+ if part:
+ part["loglines"].append(xml.sax.saxutils.escape(line))
+
+
+def elapsed(current, previous):
+ if not (current and previous):
+ return 0
+
+ diff = current - previous
+ tty.debug("elapsed = %s - %s = %s" % (current, previous, diff))
+ return diff.total_seconds()
+
+
+def expected_failure(line):
+ if not line:
+ return False
+
+ match = returns_regexp.search(line)
+ xfail = "0" not in match.group(0) if match else False
+ return xfail
+
+
+def new_part():
+ return {
+ "command": None,
+ "completed": "Unknown",
+ "desc": None,
+ "elapsed": None,
+ "name": None,
+ "loglines": [],
+ "output": None,
+ "status": "passed",
+ }
+
+
+def part_name(source):
+ # TODO: Should be passed the package prefix and only remove it
+ elements = []
+ for e in source.replace("'", "").split(" "):
+ elements.append(os.path.basename(e) if os.sep in e else e)
+ return "_".join(elements)
+
+
+def process_part_end(part, curr_time, last_time):
+ if part:
+ if not part["elapsed"]:
+ part["elapsed"] = elapsed(curr_time, last_time)
+
+ stat = part["status"]
+ if stat in completed:
+ if stat == "passed" and expected_failure(part["desc"]):
+ part["completed"] = "Expected to fail"
+ elif part["completed"] == "Unknown":
+ part["completed"] = completed[stat]
+ part["output"] = "\n".join(part["loglines"])
+
+
+def timestamp(time_string):
+ return datetime.strptime(time_string, "%Y-%m-%d-%H:%M:%S.%f")
+
+
+def skip(line):
+ for regex in skip_regexps:
+ match = regex.search(line)
+ if match:
+ return match
+
+
+def status(line):
+ for regex in status_regexps:
+ match = regex.search(line)
+ if match:
+ stat = match.group(0)
+ stat = "notrun" if stat == "NO-TESTS" else stat
+ return stat.lower()
+
+
+def extract_test_parts(default_name, outputs):
+ parts = []
+ part = {}
+ testdesc = ""
+ last_time = None
+ curr_time = None
+ for line in outputs:
+ line = line.strip()
+ if not line:
+ add_part_output(part, line)
+ continue
+
+ if skip(line):
+ continue
+
+ # Skipped tests start with "Skipped" and end with "package"
+ if line.startswith("Skipped") and line.endswith("package"):
+ part = new_part()
+ part["command"] = "Not Applicable"
+ part["completed"] = line
+ part["elapsed"] = 0.0
+ part["name"] = default_name
+ part["status"] = "notrun"
+ parts.append(part)
+ continue
+
+ # Process Spack log messages
+ if line.find("==>") != -1:
+ match = log_regexp.search(line)
+ if match:
+ curr_time = timestamp(match.group(1))
+ msg = match.group(2)
+
+ # Skip logged message for caching build-time data
+ if msg.startswith("Installing"):
+ continue
+
+ # New command means the start of a new test part
+ if msg.startswith("'") and msg.endswith("'"):
+ # Update the last part processed
+ process_part_end(part, curr_time, last_time)
+
+ part = new_part()
+ part["command"] = msg
+ part["name"] = part_name(msg)
+ parts.append(part)
+
+ # Save off the optional test description if it was
+ # tty.debuged *prior to* the command and reset
+ if testdesc:
+ part["desc"] = testdesc
+ testdesc = ""
+
+ else:
+ # Update the last part processed since a new log message
+ # means a non-test action
+ process_part_end(part, curr_time, last_time)
+
+ if testdesc:
+ # We had a test description but no command so treat
+ # as a new part (e.g., some import tests)
+ part = new_part()
+ part["name"] = "_".join(testdesc.split())
+ part["command"] = "unknown"
+ part["desc"] = testdesc
+ parts.append(part)
+ process_part_end(part, curr_time, curr_time)
+
+ # Assuming this is a description for the next test part
+ testdesc = msg
+
+ else:
+ tty.debug("Did not recognize test output '{0}'".format(line))
+
+ # Each log message potentially represents a new test part so
+ # save off the last timestamp
+ last_time = curr_time
+ continue
+
+ # Check for status values
+ stat = status(line)
+ if stat:
+ if part:
+ part["status"] = stat
+ add_part_output(part, line)
+ else:
+ tty.warn("No part to add status from '{0}'".format(line))
+ continue
+
+ add_part_output(part, line)
+
+ # Process the last lingering part IF it didn't generate status
+ process_part_end(part, curr_time, last_time)
+
+ # If no parts, create a skeleton to flag that the tests are not run
+ if not parts:
+ part = new_part()
+ stat = "notrun"
+ part["command"] = "Not Applicable"
+ part["completed"] = completed[stat]
+ part["elapsed"] = 0.0
+ part["name"] = default_name
+ part["status"] = stat
+ parts.append(part)
+
+ return parts
diff --git a/lib/spack/spack/reporters/junit.py b/lib/spack/spack/reporters/junit.py
index f845974d9d..ee080fc6ab 100644
--- a/lib/spack/spack/reporters/junit.py
+++ b/lib/spack/spack/reporters/junit.py
@@ -3,11 +3,10 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os.path
import posixpath
-import spack.build_environment
-import spack.fetch_strategy
-import spack.package_base
+import spack.tengine
from spack.reporter import Reporter
__all__ = ["JUnit"]
@@ -23,6 +22,11 @@ class JUnit(Reporter):
self.template_file = posixpath.join("reports", "junit.xml")
def build_report(self, filename, report_data):
+ if not (os.path.splitext(filename))[1]:
+ # Ensure the report name will end with the proper extension;
+ # otherwise, it currently defaults to the "directory" name.
+ filename = filename + ".xml"
+
# Write the report
with open(filename, "w") as f:
env = spack.tengine.make_environment()
diff --git a/lib/spack/spack/schema/gitlab_ci.py b/lib/spack/spack/schema/gitlab_ci.py
index eb8abc9682..d9da5c6ce7 100644
--- a/lib/spack/spack/schema/gitlab_ci.py
+++ b/lib/spack/spack/schema/gitlab_ci.py
@@ -101,6 +101,12 @@ core_shared_properties = union_dicts(
"signing-job-attributes": runner_selector_schema,
"rebuild-index": {"type": "boolean"},
"broken-specs-url": {"type": "string"},
+ "broken-tests-packages": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ },
+ },
},
)
diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py
index 0964a1ba1a..f42518f7fb 100644
--- a/lib/spack/spack/test/ci.py
+++ b/lib/spack/spack/test/ci.py
@@ -176,6 +176,33 @@ def test_download_and_extract_artifacts(tmpdir, monkeypatch, working_env):
ci.download_and_extract_artifacts(url, working_dir)
+def test_ci_copy_stage_logs_to_artifacts_fail(tmpdir, config, mock_packages, monkeypatch, capfd):
+ """The copy will fail because the spec is not concrete so does not have
+ a package."""
+ log_dir = tmpdir.join("log_dir")
+ s = spec.Spec("printing-package").concretized()
+
+ ci.copy_stage_logs_to_artifacts(s, log_dir)
+ _, err = capfd.readouterr()
+ assert "Unable to copy files" in err
+ assert "No such file or directory" in err
+
+
+def test_ci_copy_test_logs_to_artifacts_fail(tmpdir, capfd):
+ log_dir = tmpdir.join("log_dir")
+
+ ci.copy_test_logs_to_artifacts("no-such-dir", log_dir)
+ _, err = capfd.readouterr()
+ assert "Cannot copy test logs" in err
+
+ stage_dir = tmpdir.join("stage_dir").strpath
+ os.makedirs(stage_dir)
+ ci.copy_test_logs_to_artifacts(stage_dir, log_dir)
+ _, err = capfd.readouterr()
+ assert "Unable to copy files" in err
+ assert "No such file or directory" in err
+
+
def test_setup_spack_repro_version(tmpdir, capfd, last_two_git_commits, monkeypatch):
c1, c2 = last_two_git_commits
repro_dir = os.path.join(tmpdir.strpath, "repro")
@@ -467,3 +494,154 @@ def test_affected_specs_on_first_concretization(mutable_mock_env_path, config):
affected_specs = spack.ci.get_spec_filter_list(e, ["zlib"])
hdf5_specs = [s for s in affected_specs if s.name == "hdf5"]
assert len(hdf5_specs) == 2
+
+
+@pytest.mark.skipif(
+ sys.platform == "win32", reason="Reliance on bash script ot supported on Windows"
+)
+def test_ci_process_command(tmpdir):
+ repro_dir = tmpdir.join("repro_dir").strpath
+ os.makedirs(repro_dir)
+ result = ci.process_command("help", [], repro_dir)
+
+ assert os.path.exists(fs.join_path(repro_dir, "help.sh"))
+ assert not result
+
+
+@pytest.mark.skipif(
+ sys.platform == "win32", reason="Reliance on bash script ot supported on Windows"
+)
+def test_ci_process_command_fail(tmpdir, monkeypatch):
+ import subprocess
+
+ err = "subprocess wait exception"
+
+ def _fail(self, args):
+ raise RuntimeError(err)
+
+ monkeypatch.setattr(subprocess.Popen, "__init__", _fail)
+
+ repro_dir = tmpdir.join("repro_dir").strpath
+ os.makedirs(repro_dir)
+
+ with pytest.raises(RuntimeError, match=err):
+ ci.process_command("help", [], repro_dir)
+
+
+def test_ci_create_buildcache(tmpdir, working_env, config, mock_packages, monkeypatch):
+ # Monkeypatching ci method tested elsewhere to reduce number of methods
+ # that would need to be patched here.
+ monkeypatch.setattr(spack.ci, "push_mirror_contents", lambda a, b, c, d: None)
+
+ args = {
+ "env": None,
+ "buildcache_mirror_url": "file://fake-url",
+ "pipeline_mirror_url": "file://fake-url",
+ }
+ ci.create_buildcache(**args)
+
+
+def test_ci_run_standalone_tests_missing_requirements(
+ tmpdir, working_env, config, mock_packages, capfd
+):
+ """This test case checks for failing prerequisite checks."""
+ ci.run_standalone_tests()
+ err = capfd.readouterr()[1]
+ assert "Job spec is required" in err
+
+ args = {"job_spec": spec.Spec("printing-package").concretized()}
+ ci.run_standalone_tests(**args)
+ err = capfd.readouterr()[1]
+ assert "Reproduction directory is required" in err
+
+
+@pytest.mark.skipif(
+ sys.platform == "win32", reason="Reliance on bash script ot supported on Windows"
+)
+def test_ci_run_standalone_tests_not_installed_junit(
+ tmpdir, working_env, config, mock_packages, mock_test_stage, capfd
+):
+ log_file = tmpdir.join("junit.xml").strpath
+ args = {
+ "log_file": log_file,
+ "job_spec": spec.Spec("printing-package").concretized(),
+ "repro_dir": tmpdir.join("repro_dir").strpath,
+ "fail_fast": True,
+ }
+ os.makedirs(args["repro_dir"])
+
+ ci.run_standalone_tests(**args)
+ err = capfd.readouterr()[1]
+ assert "No installed packages" in err
+ assert os.path.getsize(log_file) > 0
+
+
+@pytest.mark.skipif(
+ sys.platform == "win32", reason="Reliance on bash script ot supported on Windows"
+)
+def test_ci_run_standalone_tests_not_installed_cdash(
+ tmpdir, working_env, config, mock_packages, mock_test_stage, capfd
+):
+ """Test run_standalone_tests with cdash and related options."""
+ log_file = tmpdir.join("junit.xml").strpath
+ args = {
+ "log_file": log_file,
+ "job_spec": spec.Spec("printing-package").concretized(),
+ "repro_dir": tmpdir.join("repro_dir").strpath,
+ }
+ os.makedirs(args["repro_dir"])
+
+ # Cover when CDash handler provided (with the log file as well)
+ ci_cdash = {
+ "url": "file://fake",
+ "build-group": "fake-group",
+ "project": "ci-unit-testing",
+ "site": "fake-site",
+ }
+ os.environ["SPACK_CDASH_BUILD_NAME"] = "ci-test-build"
+ os.environ["SPACK_CDASH_BUILD_STAMP"] = "ci-test-build-stamp"
+ os.environ["CI_RUNNER_DESCRIPTION"] = "test-runner"
+ handler = ci.CDashHandler(ci_cdash)
+ args["cdash"] = handler
+ ci.run_standalone_tests(**args)
+ out = capfd.readouterr()[0]
+ # CDash *and* log file output means log file ignored
+ assert "xml option is ignored" in out
+ assert "0 passed of 0" in out
+
+ # copy test results (though none)
+ artifacts_dir = tmpdir.join("artifacts")
+ fs.mkdirp(artifacts_dir.strpath)
+ handler.copy_test_results(tmpdir.strpath, artifacts_dir.strpath)
+ err = capfd.readouterr()[1]
+ assert "Unable to copy files" in err
+ assert "No such file or directory" in err
+
+
+def test_ci_skipped_report(tmpdir, mock_packages, config):
+ """Test explicit skipping of report as well as CI's 'package' arg."""
+ pkg = "trivial-smoke-test"
+ spec = spack.spec.Spec(pkg).concretized()
+ ci_cdash = {
+ "url": "file://fake",
+ "build-group": "fake-group",
+ "project": "ci-unit-testing",
+ "site": "fake-site",
+ }
+ os.environ["SPACK_CDASH_BUILD_NAME"] = "fake-test-build"
+ os.environ["SPACK_CDASH_BUILD_STAMP"] = "ci-test-build-stamp"
+ os.environ["CI_RUNNER_DESCRIPTION"] = "test-runner"
+ handler = ci.CDashHandler(ci_cdash)
+ reason = "Testing skip"
+ handler.report_skipped(spec, tmpdir.strpath, reason=reason)
+
+ report = fs.join_path(tmpdir, "{0}_Testing.xml".format(pkg))
+ expected = "Skipped {0} package".format(pkg)
+ with open(report, "r") as f:
+ have = [0, 0]
+ for line in f:
+ if expected in line:
+ have[0] += 1
+ elif reason in line:
+ have[1] += 1
+ assert all(count == 1 for count in have)
diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py
index 2435f8d92a..0ac352c1df 100644
--- a/lib/spack/spack/test/cmd/ci.py
+++ b/lib/spack/spack/test/cmd/ci.py
@@ -33,6 +33,7 @@ from spack.schema.gitlab_ci import schema as gitlab_ci_schema
from spack.spec import CompilerSpec, Spec
from spack.util.executable import which
from spack.util.mock_package import MockPackageMultiRepo
+from spack.util.pattern import Bunch
ci_cmd = spack.main.SpackCommand("ci")
env_cmd = spack.main.SpackCommand("env")
@@ -257,7 +258,12 @@ def _validate_needs_graph(yaml_contents, needs_graph, artifacts):
def test_ci_generate_bootstrap_gcc(
- tmpdir, mutable_mock_env_path, install_mockery, mock_packages, ci_base_environment
+ tmpdir,
+ working_env,
+ mutable_mock_env_path,
+ install_mockery,
+ mock_packages,
+ ci_base_environment,
):
"""Test that we can bootstrap a compiler and use it as the
compiler for a spec in the environment"""
@@ -320,7 +326,12 @@ spack:
def test_ci_generate_bootstrap_artifacts_buildcache(
- tmpdir, mutable_mock_env_path, install_mockery, mock_packages, ci_base_environment
+ tmpdir,
+ working_env,
+ mutable_mock_env_path,
+ install_mockery,
+ mock_packages,
+ ci_base_environment,
):
"""Test that we can bootstrap a compiler when artifacts buildcache
is turned on"""
@@ -387,6 +398,7 @@ spack:
def test_ci_generate_with_env_missing_section(
tmpdir,
+ working_env,
mutable_mock_env_path,
install_mockery,
mock_packages,
@@ -479,6 +491,7 @@ spack:
def test_ci_generate_with_custom_scripts(
tmpdir,
+ working_env,
mutable_mock_env_path,
install_mockery,
mock_packages,
@@ -575,7 +588,12 @@ spack:
def test_ci_generate_pkg_with_deps(
- tmpdir, mutable_mock_env_path, install_mockery, mock_packages, ci_base_environment
+ tmpdir,
+ working_env,
+ mutable_mock_env_path,
+ install_mockery,
+ mock_packages,
+ ci_base_environment,
):
"""Test pipeline generation for a package w/ dependencies"""
filename = str(tmpdir.join("spack.yaml"))
@@ -630,7 +648,13 @@ spack:
def test_ci_generate_for_pr_pipeline(
- tmpdir, mutable_mock_env_path, install_mockery, mock_packages, monkeypatch, ci_base_environment
+ tmpdir,
+ working_env,
+ mutable_mock_env_path,
+ install_mockery,
+ mock_packages,
+ monkeypatch,
+ ci_base_environment,
):
"""Test that PR pipelines do not include a final stage job for
rebuilding the mirror index, even if that job is specifically
@@ -690,7 +714,13 @@ spack:
def test_ci_generate_with_external_pkg(
- tmpdir, mutable_mock_env_path, install_mockery, mock_packages, monkeypatch, ci_base_environment
+ tmpdir,
+ working_env,
+ mutable_mock_env_path,
+ install_mockery,
+ mock_packages,
+ monkeypatch,
+ ci_base_environment,
):
"""Make sure we do not generate jobs for external pkgs"""
filename = str(tmpdir.join("spack.yaml"))
@@ -729,22 +759,40 @@ spack:
assert not any("externaltool" in key for key in yaml_contents)
-@pytest.mark.xfail(reason="fails intermittently and covered by gitlab ci")
-def test_ci_rebuild(
- tmpdir,
- mutable_mock_env_path,
- install_mockery,
- mock_packages,
- monkeypatch,
- mock_gnupghome,
- mock_fetch,
- ci_base_environment,
- mock_binary_index,
-):
+def test_ci_rebuild_missing_config(tmpdir, working_env, mutable_mock_env_path):
+ spack_yaml_contents = """
+spack:
+ specs:
+ - archive-files
+"""
+
+ filename = str(tmpdir.join("spack.yaml"))
+ with open(filename, "w") as f:
+ f.write(spack_yaml_contents)
+
+ with tmpdir.as_cwd():
+ env_cmd("create", "test", "./spack.yaml")
+ env_cmd("activate", "--without-view", "--sh", "test")
+ out = ci_cmd("rebuild", fail_on_error=False)
+ assert "env containing gitlab-ci" in out
+
+ env_cmd("deactivate")
+
+
+def _signing_key():
+ signing_key_dir = spack_paths.mock_gpg_keys_path
+ signing_key_path = os.path.join(signing_key_dir, "package-signing-key")
+ with open(signing_key_path) as fd:
+ key = fd.read()
+ return key
+
+
+def create_rebuild_env(tmpdir, pkg_name, broken_tests=False):
working_dir = tmpdir.join("working_dir")
log_dir = os.path.join(working_dir.strpath, "logs")
repro_dir = os.path.join(working_dir.strpath, "repro")
+ test_dir = os.path.join(working_dir.strpath, "test")
env_dir = working_dir.join("concrete_env")
mirror_dir = working_dir.join("mirror")
@@ -754,39 +802,37 @@ def test_ci_rebuild(
broken_specs_url = url_util.join("file://", broken_specs_path)
temp_storage_url = "file:///path/to/per/pipeline/storage"
+ broken_tests_packages = [pkg_name] if broken_tests else []
+
ci_job_url = "https://some.domain/group/project/-/jobs/42"
ci_pipeline_url = "https://some.domain/group/project/-/pipelines/7"
- signing_key_dir = spack_paths.mock_gpg_keys_path
- signing_key_path = os.path.join(signing_key_dir, "package-signing-key")
- with open(signing_key_path) as fd:
- signing_key = fd.read()
-
spack_yaml_contents = """
spack:
- definitions:
- - packages: [archive-files]
- specs:
- - $packages
- mirrors:
- test-mirror: {0}
- gitlab-ci:
- broken-specs-url: {1}
- temporary-storage-url-prefix: {2}
- mappings:
+ definitions:
+ - packages: [{0}]
+ specs:
+ - $packages
+ mirrors:
+ test-mirror: {1}
+ gitlab-ci:
+ broken-specs-url: {2}
+ broken-tests-packages: {3}
+ temporary-storage-url-prefix: {4}
+ mappings:
- match:
- - archive-files
+ - {0}
runner-attributes:
tags:
- donotcare
image: donotcare
- cdash:
- build-group: Not important
- url: https://my.fake.cdash
- project: Not used
- site: Nothing
+ cdash:
+ build-group: Not important
+ url: https://my.fake.cdash
+ project: Not used
+ site: Nothing
""".format(
- mirror_url, broken_specs_url, temp_storage_url
+ pkg_name, mirror_url, broken_specs_url, broken_tests_packages, temp_storage_url
)
filename = str(tmpdir.join("spack.yaml"))
@@ -809,43 +855,126 @@ spack:
root_spec_dag_hash = None
for h, s in env.specs_by_hash.items():
- if s.name == "archive-files":
+ if s.name == pkg_name:
root_spec_dag_hash = h
assert root_spec_dag_hash
- def fake_cdash_register(build_name, base_url, project, site, track):
- return ("fakebuildid", "fakestamp")
+ return Bunch(
+ broken_spec_file=os.path.join(broken_specs_path, root_spec_dag_hash),
+ ci_job_url=ci_job_url,
+ ci_pipeline_url=ci_pipeline_url,
+ env_dir=env_dir,
+ log_dir=log_dir,
+ mirror_dir=mirror_dir,
+ mirror_url=mirror_url,
+ repro_dir=repro_dir,
+ root_spec_dag_hash=root_spec_dag_hash,
+ test_dir=test_dir,
+ working_dir=working_dir,
+ )
+
+
+def activate_rebuild_env(tmpdir, pkg_name, rebuild_env):
+ env_cmd("activate", "--without-view", "--sh", "-d", ".")
+
+ # Create environment variables as gitlab would do it
+ os.environ.update(
+ {
+ "SPACK_ARTIFACTS_ROOT": rebuild_env.working_dir.strpath,
+ "SPACK_JOB_LOG_DIR": rebuild_env.log_dir,
+ "SPACK_JOB_REPRO_DIR": rebuild_env.repro_dir,
+ "SPACK_JOB_TEST_DIR": rebuild_env.test_dir,
+ "SPACK_LOCAL_MIRROR_DIR": rebuild_env.mirror_dir.strpath,
+ "SPACK_CONCRETE_ENV_DIR": rebuild_env.env_dir.strpath,
+ "CI_PIPELINE_ID": "7192",
+ "SPACK_SIGNING_KEY": _signing_key(),
+ "SPACK_ROOT_SPEC": rebuild_env.root_spec_dag_hash,
+ "SPACK_JOB_SPEC_DAG_HASH": rebuild_env.root_spec_dag_hash,
+ "SPACK_JOB_SPEC_PKG_NAME": pkg_name,
+ "SPACK_COMPILER_ACTION": "NONE",
+ "SPACK_CDASH_BUILD_NAME": "(specs) {0}".format(pkg_name),
+ "SPACK_REMOTE_MIRROR_URL": rebuild_env.mirror_url,
+ "SPACK_PIPELINE_TYPE": "spack_protected_branch",
+ "CI_JOB_URL": rebuild_env.ci_job_url,
+ "CI_PIPELINE_URL": rebuild_env.ci_pipeline_url,
+ "CI_PROJECT_DIR": tmpdir.join("ci-project").strpath,
+ }
+ )
+
+
+@pytest.mark.parametrize("broken_tests", [True, False])
+def test_ci_rebuild_mock_success(
+ tmpdir,
+ config,
+ working_env,
+ mutable_mock_env_path,
+ install_mockery,
+ mock_gnupghome,
+ mock_stage,
+ mock_fetch,
+ mock_binary_index,
+ monkeypatch,
+ broken_tests,
+):
+
+ pkg_name = "archive-files"
+ rebuild_env = create_rebuild_env(tmpdir, pkg_name, broken_tests)
+
+ monkeypatch.setattr(
+ spack.cmd.ci,
+ "CI_REBUILD_INSTALL_BASE_ARGS",
+ ["echo"],
+ )
+
+ with rebuild_env.env_dir.as_cwd():
+ activate_rebuild_env(tmpdir, pkg_name, rebuild_env)
+
+ out = ci_cmd("rebuild", "--tests", fail_on_error=False)
+
+ # We didn"t really run the build so build output file(s) are missing
+ assert "Unable to copy files" in out
+ assert "No such file or directory" in out
+
+ if broken_tests:
+ # We generate a skipped tests report in this case
+ assert "Unable to run stand-alone tests" in out
+ else:
+ # No installation means no package to test and no test log to copy
+ assert "Cannot copy test logs" in out
+
+
+@pytest.mark.xfail(reason="fails intermittently and covered by gitlab ci")
+def test_ci_rebuild(
+ tmpdir,
+ working_env,
+ mutable_mock_env_path,
+ install_mockery,
+ mock_packages,
+ monkeypatch,
+ mock_gnupghome,
+ mock_fetch,
+ ci_base_environment,
+ mock_binary_index,
+):
+ pkg_name = "archive-files"
+ rebuild_env = create_rebuild_env(tmpdir, pkg_name)
+
+ # Create job directories to be removed before processing (for coverage)
+ os.makedirs(rebuild_env.log_dir)
+ os.makedirs(rebuild_env.repro_dir)
+ os.makedirs(rebuild_env.test_dir)
+
+ with rebuild_env.env_dir.as_cwd():
+ activate_rebuild_env(tmpdir, pkg_name, rebuild_env)
+
+ ci_cmd("rebuild", "--tests", fail_on_error=False)
monkeypatch.setattr(spack.cmd.ci, "CI_REBUILD_INSTALL_BASE_ARGS", ["notcommand"])
monkeypatch.setattr(spack.cmd.ci, "INSTALL_FAIL_CODE", 127)
- with env_dir.as_cwd():
- env_cmd("activate", "--without-view", "--sh", "-d", ".")
-
- # Create environment variables as gitlab would do it
- os.environ.update(
- {
- "SPACK_ARTIFACTS_ROOT": working_dir.strpath,
- "SPACK_JOB_LOG_DIR": log_dir,
- "SPACK_JOB_REPRO_DIR": repro_dir,
- "SPACK_LOCAL_MIRROR_DIR": mirror_dir.strpath,
- "SPACK_CONCRETE_ENV_DIR": env_dir.strpath,
- "CI_PIPELINE_ID": "7192",
- "SPACK_SIGNING_KEY": signing_key,
- "SPACK_ROOT_SPEC": root_spec_dag_hash,
- "SPACK_JOB_SPEC_DAG_HASH": root_spec_dag_hash,
- "SPACK_JOB_SPEC_PKG_NAME": "archive-files",
- "SPACK_COMPILER_ACTION": "NONE",
- "SPACK_CDASH_BUILD_NAME": "(specs) archive-files",
- "SPACK_REMOTE_MIRROR_URL": mirror_url,
- "SPACK_PIPELINE_TYPE": "spack_protected_branch",
- "CI_JOB_URL": ci_job_url,
- "CI_PIPELINE_URL": ci_pipeline_url,
- }
- )
-
- ci_cmd("rebuild", fail_on_error=False)
+ with rebuild_env.env_dir.as_cwd():
+ activate_rebuild_env(tmpdir, pkg_name, rebuild_env)
expected_repro_files = [
"install.sh",
@@ -854,10 +983,10 @@ spack:
"spack.yaml",
"spack.lock",
]
- repro_files = os.listdir(repro_dir)
+ repro_files = os.listdir(rebuild_env.repro_dir)
assert all([f in repro_files for f in expected_repro_files])
- install_script_path = os.path.join(repro_dir, "install.sh")
+ install_script_path = os.path.join(rebuild_env.repro_dir, "install.sh")
install_line = None
with open(install_script_path) as fd:
for line in fd:
@@ -878,17 +1007,27 @@ spack:
flag_index = install_parts.index("-f")
assert "archive-files.json" in install_parts[flag_index + 1]
- broken_spec_file = os.path.join(broken_specs_path, root_spec_dag_hash)
- with open(broken_spec_file) as fd:
+ with open(rebuild_env.broken_spec_file) as fd:
broken_spec_content = fd.read()
- assert ci_job_url in broken_spec_content
- assert (ci_pipeline_url) in broken_spec_content
+ assert rebuild_env.ci_job_url in broken_spec_content
+ assert rebuild_env.ci_pipeline_url in broken_spec_content
+
+ # Ensure also produce CDash output for skipped (or notrun) tests
+ test_files = os.listdir(rebuild_env.test_dir)
+ with open(os.path.join(rebuild_env.test_dir, test_files[0]), "r") as f:
+ have = False
+ for line in f:
+ if "notrun" in line:
+ have = True
+ break
+ assert have
env_cmd("deactivate")
def test_ci_nothing_to_rebuild(
tmpdir,
+ working_env,
mutable_mock_env_path,
install_mockery,
mock_packages,
@@ -946,6 +1085,7 @@ spack:
"SPACK_ARTIFACTS_ROOT": working_dir.strpath,
"SPACK_JOB_LOG_DIR": "log_dir",
"SPACK_JOB_REPRO_DIR": "repro_dir",
+ "SPACK_JOB_TEST_DIR": "test_dir",
"SPACK_LOCAL_MIRROR_DIR": mirror_dir.strpath,
"SPACK_CONCRETE_ENV_DIR": tmpdir.strpath,
"SPACK_ROOT_SPEC": root_spec_dag_hash,
@@ -1073,12 +1213,7 @@ def test_push_mirror_contents(
mirror_dir = working_dir.join("mirror")
mirror_url = "file://{0}".format(mirror_dir.strpath)
- signing_key_dir = spack_paths.mock_gpg_keys_path
- signing_key_path = os.path.join(signing_key_dir, "package-signing-key")
- with open(signing_key_path) as fd:
- signing_key = fd.read()
-
- ci.import_signing_key(signing_key)
+ ci.import_signing_key(_signing_key())
spack_yaml_contents = """
spack:
@@ -1198,6 +1333,7 @@ spack:
# Also just make sure that if something goes wrong with the
# stage logs copy, no exception is thrown
+ ci.copy_stage_logs_to_artifacts(concrete_spec, None)
ci.copy_stage_logs_to_artifacts(None, logs_dir.strpath)
dl_dir = working_dir.join("download_dir")
@@ -1413,7 +1549,13 @@ spack:
@pytest.mark.disable_clean_stage_check
def test_ci_rebuild_index(
- tmpdir, mutable_mock_env_path, install_mockery, mock_packages, mock_fetch, mock_stage
+ tmpdir,
+ working_env,
+ mutable_mock_env_path,
+ install_mockery,
+ mock_packages,
+ mock_fetch,
+ mock_stage,
):
working_dir = tmpdir.join("working_dir")
@@ -2036,3 +2178,35 @@ spack:
expect_out = "docker run --rm -v {0}:{0} -ti {1}".format(working_dir.strpath, image_name)
assert expect_out in rep_out
+
+
+@pytest.mark.parametrize(
+ "subcmd",
+ [
+ (""),
+ ("generate"),
+ ("rebuild-index"),
+ ("rebuild"),
+ ("reproduce-build"),
+ ],
+)
+def test_ci_help(subcmd, capsys):
+ """Make sure `spack ci` --help describes the (sub)command help."""
+ with pytest.raises(SystemExit):
+ ci_cmd(subcmd, "--help")
+
+ out = str(capsys.readouterr())
+ usage = "usage: spack ci {0}{1}[".format(subcmd, " " if subcmd else "")
+ assert usage in out
+
+
+def test_cmd_first_line():
+ """Explicitly test first_line since not picked up in test_ci_help."""
+ first = "This is a test."
+ doc = """{0}
+
+ Is there more to be said?""".format(
+ first
+ )
+
+ assert spack.cmd.first_line(doc) == first
diff --git a/lib/spack/spack/test/cmd/config.py b/lib/spack/spack/test/cmd/config.py
index 42152ca3a5..f9c3fae2f9 100644
--- a/lib/spack/spack/test/cmd/config.py
+++ b/lib/spack/spack/test/cmd/config.py
@@ -78,7 +78,7 @@ repos:
)
-def test_config_edit():
+def test_config_edit(mutable_config, working_env):
"""Ensure `spack config edit` edits the right paths."""
dms = spack.config.default_modify_scope("compilers")
diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py
index 73e3cf15cf..91ef154001 100644
--- a/lib/spack/spack/test/cmd/install.py
+++ b/lib/spack/spack/test/cmd/install.py
@@ -49,11 +49,12 @@ def test_install_package_and_dependency(
tmpdir, mock_packages, mock_archive, mock_fetch, config, install_mockery
):
+ log = "test"
with tmpdir.as_cwd():
- install("--log-format=junit", "--log-file=test.xml", "libdwarf")
+ install("--log-format=junit", "--log-file={0}".format(log), "libdwarf")
files = tmpdir.listdir()
- filename = tmpdir.join("test.xml")
+ filename = tmpdir.join("{0}.xml".format(log))
assert filename in files
content = filename.open().read()
diff --git a/lib/spack/spack/test/cmd/test.py b/lib/spack/spack/test/cmd/test.py
index bc5724e04e..e842dbc465 100644
--- a/lib/spack/spack/test/cmd/test.py
+++ b/lib/spack/spack/test/cmd/test.py
@@ -186,7 +186,7 @@ def test_cdash_output_test_error(
report_dir = tmpdir.join("cdash_reports")
print(tmpdir.listdir())
assert report_dir in tmpdir.listdir()
- report_file = report_dir.join("test-error_Test.xml")
+ report_file = report_dir.join("test-error_Testing.xml")
assert report_file in report_dir.listdir()
content = report_file.open().read()
assert "FAILED: Command exited with status 1" in content
@@ -205,7 +205,7 @@ def test_cdash_upload_clean_test(
spack_test("run", "--log-file=cdash_reports", "--log-format=cdash", "printing-package")
report_dir = tmpdir.join("cdash_reports")
assert report_dir in tmpdir.listdir()
- report_file = report_dir.join("printing-package_Test.xml")
+ report_file = report_dir.join("printing-package_Testing.xml")
assert report_file in report_dir.listdir()
content = report_file.open().read()
assert "</Test>" in content
diff --git a/lib/spack/spack/test/reporters.py b/lib/spack/spack/test/reporters.py
new file mode 100644
index 0000000000..72aff79210
--- /dev/null
+++ b/lib/spack/spack/test/reporters.py
@@ -0,0 +1,175 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import pytest
+
+import llnl.util.filesystem as fs
+import llnl.util.tty as tty
+
+import spack.reporters.cdash
+import spack.reporters.extract
+import spack.spec
+from spack.util.pattern import Bunch
+
+# Use a path variable to appease Spack style line length checks
+fake_install_prefix = fs.join_path(
+ "usr",
+ "spack",
+ "spack",
+ "opt",
+ "spack",
+ "linux-rhel7-broadwell",
+ "intel-19.0.4.227",
+ "fake-1.0",
+)
+fake_install_test_root = fs.join_path(fake_install_prefix, ".spack", "test")
+fake_test_cache = fs.join_path(
+ "usr", "spack", ".spack", "test", "abcdefg", "fake-1.0-abcdefg", "cache", "fake"
+)
+
+
+def test_reporters_extract_no_parts(capfd):
+ # This test ticks three boxes:
+ # 1) has Installing, which is skipped;
+ # 2) does not define any test parts;
+ # 3) has a status value without a part so generates a warning
+ outputs = """
+==> Testing package fake-1.0-abcdefg
+==> [2022-02-11-17:14:38.875259] Installing {0} to {1}
+NO-TESTS
+""".format(
+ fake_install_test_root, fake_test_cache
+ ).splitlines()
+
+ parts = spack.reporters.extract.extract_test_parts("fake", outputs)
+ err = capfd.readouterr()[1]
+
+ assert len(parts) == 1
+ assert parts[0]["status"] == "notrun"
+ assert "No part to add status" in err
+
+
+def test_reporters_extract_no_command():
+ # This test ticks 2 boxes:
+ # 1) has a test description with no command or status
+ # 2) has a test description, command, and status
+ fake_bin = fs.join_path(fake_install_prefix, "bin", "fake")
+ outputs = """
+==> Testing package fake-1.0-abcdefg
+==> [2022-02-15-18:44:21.250165] command with no status
+==> [2022-02-15-18:44:21.250175] running test program
+==> [2022-02-15-18:44:21.250200] '{0}'
+PASSED
+""".format(
+ fake_bin
+ ).splitlines()
+
+ parts = spack.reporters.extract.extract_test_parts("fake", outputs)
+ assert len(parts) == 2
+ assert parts[0]["command"] == "unknown"
+ assert parts[1]["loglines"] == ["PASSED"]
+ assert parts[1]["elapsed"] == 0.0
+
+
+def test_reporters_extract_missing_desc():
+ fake_bin = fs.join_path(fake_install_prefix, "bin", "importer")
+ outputs = """
+==> Testing package fake-1.0-abcdefg
+==> [2022-02-15-18:44:21.250165] '{0}' '-c' 'import fake.bin'
+PASSED
+==> [2022-02-15-18:44:21.250200] '{0}' '-c' 'import fake.util'
+PASSED
+""".format(
+ fake_bin
+ ).splitlines()
+
+ parts = spack.reporters.extract.extract_test_parts("fake", outputs)
+
+ assert len(parts) == 2
+ assert parts[0]["desc"] is None
+ assert parts[1]["desc"] is None
+
+
+def test_reporters_extract_xfail():
+ fake_bin = fs.join_path(fake_install_prefix, "bin", "fake-app")
+ outputs = """
+==> Testing package fake-1.0-abcdefg
+==> [2022-02-15-18:44:21.250165] Expecting return code in [3]
+==> [2022-02-15-18:44:21.250200] '{0}'
+PASSED
+""".format(
+ fake_bin
+ ).splitlines()
+
+ parts = spack.reporters.extract.extract_test_parts("fake", outputs)
+
+ assert len(parts) == 1
+ parts[0]["completed"] == "Expected to fail"
+
+
+@pytest.mark.parametrize("state", [("not installed"), ("external")])
+def test_reporters_extract_skipped(state):
+ expected = "Skipped {0} package".format(state)
+ outputs = """
+==> Testing package fake-1.0-abcdefg
+{0}
+""".format(
+ expected
+ ).splitlines()
+
+ parts = spack.reporters.extract.extract_test_parts("fake", outputs)
+
+ assert len(parts) == 1
+ parts[0]["completed"] == expected
+
+
+def test_reporters_skip():
+ # This test ticks 3 boxes:
+ # 1) covers an as yet uncovered skip messages
+ # 2) covers debug timestamps
+ # 3) unrecognized output
+ fake_bin = fs.join_path(fake_install_prefix, "bin", "fake")
+ unknown_message = "missing timestamp"
+ outputs = """
+==> Testing package fake-1.0-abcdefg
+==> [2022-02-15-18:44:21.250165, 123456] Detected the following modules: fake1
+==> {0}
+==> [2022-02-15-18:44:21.250175, 123456] running fake program
+==> [2022-02-15-18:44:21.250200, 123456] '{1}'
+INVALID
+Results for test suite abcdefghijklmn
+""".format(
+ unknown_message, fake_bin
+ ).splitlines()
+
+ parts = spack.reporters.extract.extract_test_parts("fake", outputs)
+
+ assert len(parts) == 1
+ assert fake_bin in parts[0]["command"]
+ assert parts[0]["loglines"] == ["INVALID"]
+ assert parts[0]["elapsed"] == 0.0
+
+
+def test_reporters_report_for_package_no_stdout(tmpdir, monkeypatch, capfd):
+ class MockCDash(spack.reporters.cdash.CDash):
+ def upload(*args, **kwargs):
+ # Just return (Do NOT try to upload the report to the fake site)
+ return
+
+ args = Bunch(
+ cdash_upload_url="https://fake-upload",
+ package="fake-package",
+ cdash_build="fake-cdash-build",
+ cdash_site="fake-site",
+ cdash_buildstamp=None,
+ cdash_track="fake-track",
+ )
+ monkeypatch.setattr(tty, "_debug", 1)
+
+ reporter = MockCDash(args)
+ pkg_data = {"name": "fake-package"}
+ reporter.test_report_for_package(tmpdir.strpath, pkg_data, 0, False)
+ err = capfd.readouterr()[1]
+ assert "Skipping report for" in err
+ assert "No generated output" in err
diff --git a/lib/spack/spack/test/test_suite.py b/lib/spack/spack/test/test_suite.py
index 0f3fe97f90..3d8ebace39 100644
--- a/lib/spack/spack/test/test_suite.py
+++ b/lib/spack/spack/test/test_suite.py
@@ -7,7 +7,7 @@ import sys
import pytest
-import llnl.util.filesystem as fs
+import llnl.util.tty as tty
import spack.install_test
import spack.spec
@@ -15,6 +15,33 @@ import spack.spec
pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="Tests fail on Windows")
+def _true(*args, **kwargs):
+ """Generic monkeypatch function that always returns True."""
+ return True
+
+
+@pytest.fixture
+def ensure_debug(monkeypatch):
+ current_debug_level = tty.debug_level()
+ tty.set_debug(1)
+
+ yield
+
+ tty.set_debug(current_debug_level)
+
+
+def ensure_results(filename, expected):
+ assert os.path.exists(filename)
+ with open(filename, "r") as fd:
+ lines = fd.readlines()
+ have = False
+ for line in lines:
+ if expected in line:
+ have = True
+ break
+ assert have
+
+
def test_test_log_pathname(mock_packages, config):
"""Ensure test log path is reasonable."""
spec = spack.spec.Spec("libdwarf").concretized()
@@ -61,31 +88,15 @@ def test_write_test_result(mock_packages, mock_test_stage):
assert spec.name in msg
-def test_do_test(mock_packages, install_mockery, mock_test_stage):
- """Perform a stand-alone test with files to copy."""
+def test_test_uninstalled(mock_packages, install_mockery, mock_test_stage):
+ """Attempt to perform stand-alone test for uninstalled package."""
spec = spack.spec.Spec("trivial-smoke-test").concretized()
- test_name = "test_do_test"
- test_filename = "test_file.in"
-
- pkg = spec.package
- pkg.create_extra_test_source()
-
- test_suite = spack.install_test.TestSuite([spec], test_name)
- test_suite.current_test_spec = spec
- test_suite.current_base_spec = spec
- test_suite.ensure_stage()
-
- # Save off target paths for current spec since test suite processing
- # assumes testing multiple specs.
- cached_filename = fs.join_path(test_suite.current_test_cache_dir, pkg.test_source_filename)
- data_filename = fs.join_path(test_suite.current_test_data_dir, test_filename)
+ test_suite = spack.install_test.TestSuite([spec])
- # Run the test, making sure to retain the test stage directory
- # so we can ensure the files were copied.
- test_suite(remove_directory=False)
+ test_suite()
- assert os.path.exists(cached_filename)
- assert os.path.exists(data_filename)
+ ensure_results(test_suite.results_file, "SKIPPED")
+ ensure_results(test_suite.log_file_for_spec(spec), "Skipped not installed")
@pytest.mark.parametrize(
@@ -95,27 +106,21 @@ def test_do_test(mock_packages, install_mockery, mock_test_stage):
({"externals": True}, "NO-TESTS", "No tests"),
],
)
-def test_test_external(mock_packages, install_mockery, mock_test_stage, arguments, status, msg):
- def ensure_results(filename, expected):
- assert os.path.exists(filename)
- with open(filename, "r") as fd:
- lines = fd.readlines()
- have = False
- for line in lines:
- if expected in line:
- have = True
- break
- assert have
-
+def test_test_external(
+ mock_packages, install_mockery, mock_test_stage, monkeypatch, arguments, status, msg
+):
name = "trivial-smoke-test"
spec = spack.spec.Spec(name).concretized()
spec.external_path = "/path/to/external/{0}".format(name)
+ monkeypatch.setattr(spack.spec.Spec, "installed", _true)
+
test_suite = spack.install_test.TestSuite([spec])
test_suite(**arguments)
ensure_results(test_suite.results_file, status)
- ensure_results(test_suite.log_file_for_spec(spec), msg)
+ if arguments:
+ ensure_results(test_suite.log_file_for_spec(spec), msg)
def test_test_stage_caches(mock_packages, install_mockery, mock_test_stage):
@@ -152,21 +157,15 @@ def test_test_spec_run_once(mock_packages, install_mockery, mock_test_stage):
test_suite()
-def test_test_spec_verbose(mock_packages, install_mockery, mock_test_stage):
+def test_test_spec_passes(mock_packages, install_mockery, mock_test_stage, monkeypatch):
+
spec = spack.spec.Spec("simple-standalone-test").concretized()
+ monkeypatch.setattr(spack.spec.Spec, "installed", _true)
test_suite = spack.install_test.TestSuite([spec])
+ test_suite()
- test_suite(verbose=True)
- passed, msg = False, False
- with open(test_suite.log_file_for_spec(spec), "r") as fd:
- for line in fd:
- if "simple stand-alone test" in line:
- msg = True
- elif "PASSED" in line:
- passed = True
-
- assert msg
- assert passed
+ ensure_results(test_suite.results_file, "PASSED")
+ ensure_results(test_suite.log_file_for_spec(spec), "simple stand-alone")
def test_get_test_suite():
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
index 31d408db6e..7b06fd2dda 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
@@ -248,10 +248,13 @@ spack:
- mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
- if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
- if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
- - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+ - spack -d ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
image: ecpe4s/ubuntu22.04-runner-x86_64:2022-07-01
+ broken-tests-packages:
+ - gptune
+
mappings:
- match:
- hipblas
diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash
index 936593d98a..4cfde14e7a 100755
--- a/share/spack/spack-completion.bash
+++ b/share/spack/spack-completion.bash
@@ -612,7 +612,7 @@ _spack_ci_rebuild_index() {
}
_spack_ci_rebuild() {
- SPACK_COMPREPLY="-h --help"
+ SPACK_COMPREPLY="-h --help -t --tests --fail-fast"
}
_spack_ci_reproduce_build() {
diff --git a/share/spack/templates/reports/cdash/Site.xml b/share/spack/templates/reports/cdash/Site.xml
index f0a150b6e5..e8a6c0609b 100644
--- a/share/spack/templates/reports/cdash/Site.xml
+++ b/share/spack/templates/reports/cdash/Site.xml
@@ -2,6 +2,9 @@
<Site BuildName="{{ buildname }}"
BuildStamp="{{ buildstamp }}"
Name="{{ site }}"
+ Generator="{{ generator }}"
+ Hostname="{{ hostname }}"
OSName="{{ osname }}"
+ OSRelease="{{ osrelease }}"
+ VendorString="{{ target }}"
>
-
diff --git a/share/spack/templates/reports/cdash/Testing.xml b/share/spack/templates/reports/cdash/Testing.xml
new file mode 100644
index 0000000000..a5eb58c35a
--- /dev/null
+++ b/share/spack/templates/reports/cdash/Testing.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ This file has been modeled after the examples at this url:
+
+ https://www.paraview.org/Wiki/CDash:XML
+-->
+<Site BuildName="{{ buildname }}"
+ BuildStamp="{{ buildstamp }}"
+ Name="{{ site }}"
+ Generator="{{ generator }}"
+ Hostname="{{ hostname }}"
+ OSName="{{ osname }}"
+ OSRelease="{{ osrelease }}"
+ VendorString="{{ target }}"
+>
+ <Testing>
+ <StartTestTime>{{ testing.starttime }}</StartTestTime>
+{% for part in testing.parts %}
+ <Test Status="{{ part.status }}">
+ <Name>{{ part.name }}</Name>
+ <FullCommandLine>{{ part.command }}</FullCommandLine>
+ <Results>
+ <NamedMeasurement type="numeric/double" name="Execution Time">
+ <Value>{{ part.elapsed }}</Value>
+ </NamedMeasurement>
+{% if part.desc %}
+ <NamedMeasurement type="text/string" name="Description">
+ <Value>{{ part.desc }}</Value>
+ </NamedMeasurement>
+{% endif %}
+ <NamedMeasurement type="text/string" name="Completion Status">
+ <Value>{{ part.completed }}</Value>
+ </NamedMeasurement>
+{% if part.output %}
+ <Measurement>
+ <Value>{{ part.output }}</Value>
+ </Measurement>
+{% endif %}
+ </Results>
+ </Test>
+{% endfor %}
+ <EndTestTime>{{ testing.endtime }}</EndTestTime>
+ </Testing>
+</Site>