From 3c3b18d8588a82b6ace4ca5e852497f115e88479 Mon Sep 17 00:00:00 2001 From: Tamara Dahlgren <35777542+tldahlgren@users.noreply.github.com> Date: Tue, 23 Aug 2022 00:52:48 -0700 Subject: spack ci: add support for running stand-alone tests (#27877) This support requires adding the '--tests' option to 'spack ci rebuild'. Packages whose stand-alone tests are broken (in the CI environment) can be configured in gitlab-ci to be skipped by adding them to broken-tests-packages. Highlights include: - Restructured 'spack ci' help to provide better subcommand summaries; - Ensured only one InstallError (i.e., installer's) rather than allowing build_environment to have its own; and - Refactored CI and CDash reporting to keep CDash-related properties and behavior in a separate class. This allows stand-alone tests from `spack ci` to run when the `--tests` option is used. With `--tests`, stand-alone tests are run **after** a **successful** (re)build of the package. Test results are collected and report(able) using CDash. This PR adds the following features: - Adds `-t` and `--tests` to `spack ci rebuild` to run stand-alone tests; - Adds `--fail-fast` to stop stand-alone tests after the first failure; - Ensures a *single* `InstallError` across packages (i.e., removes second class from build environment); - Captures skipping tests for externals and uninstalled packages (for CDash reporting); - Copies test logs and outputs to the CI artifacts directory to facilitate debugging; - Parses stand-alone test results to report outputs from each `run_test` as separate test parts (CDash reporting); - Logs a test completion message to allow capture of timing of the last `run_test` part; - Adds the runner description to the CDash site to better distinguish entries in CDash tables; - Adds `gitlab-ci` `broken-tests-packages` to CI configuration to skip stand-alone testing for packages with known issues; - Changes `spack ci --help` so description of each subcommand is a single line; - Changes `spack ci --help` to provide the full description of each command (versus no description); and - Ensures `junit` test log file ends in an `.xml` extension (versus default where it does not). Tasks: - [x] Include the equivalent of the architecture information, or at least the host target, in the CDash output - [x] Upload stand-alone test results files as `test` artifacts - [x] Confirm tests are run in GitLab - [x] Ensure CDash results are uploaded as artifacts - [x] Resolve issues with CDash build-and test results appearing on same row of the table - [x] Add unit tests as needed - [x] Investigate why some (dependency) packages don't have test results (e.g., related from other pipelines) - [x] Ensure proper parsing and reporting of skipped tests (as `not run`) .. post- #28701 merge - [x] Restore the proper CDash URLand or mirror ONCE out-of-band testing completed --- lib/spack/spack/build_environment.py | 10 +- lib/spack/spack/ci.py | 519 ++++++++++++++++----- lib/spack/spack/cmd/__init__.py | 5 + lib/spack/spack/cmd/ci.py | 309 +++++++----- lib/spack/spack/cmd/install.py | 1 + lib/spack/spack/cmd/test.py | 39 +- lib/spack/spack/environment/environment.py | 2 +- lib/spack/spack/install_test.py | 5 + lib/spack/spack/installer.py | 52 ++- lib/spack/spack/package_base.py | 7 + lib/spack/spack/report.py | 5 +- lib/spack/spack/reporters/cdash.py | 137 ++++-- lib/spack/spack/reporters/extract.py | 212 +++++++++ lib/spack/spack/reporters/junit.py | 10 +- lib/spack/spack/schema/gitlab_ci.py | 6 + lib/spack/spack/test/ci.py | 178 +++++++ lib/spack/spack/test/cmd/ci.py | 336 +++++++++---- lib/spack/spack/test/cmd/config.py | 2 +- lib/spack/spack/test/cmd/install.py | 5 +- lib/spack/spack/test/cmd/test.py | 4 +- lib/spack/spack/test/reporters.py | 175 +++++++ lib/spack/spack/test/test_suite.py | 95 ++-- .../gitlab/cloud_pipelines/stacks/e4s/spack.yaml | 5 +- share/spack/spack-completion.bash | 2 +- share/spack/templates/reports/cdash/Site.xml | 5 +- share/spack/templates/reports/cdash/Testing.xml | 44 ++ 26 files changed, 1722 insertions(+), 448 deletions(-) create mode 100644 lib/spack/spack/reporters/extract.py create mode 100644 lib/spack/spack/test/reporters.py create mode 100644 share/spack/templates/reports/cdash/Testing.xml diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 5a780e4bd5..c292fb89bb 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -65,6 +65,7 @@ import spack.subprocess_context import spack.user_environment import spack.util.path from spack.error import NoHeadersError, NoLibrariesError +from spack.installer import InstallError from spack.util.cpus import cpus_available from spack.util.environment import ( EnvironmentModifications, @@ -1279,15 +1280,6 @@ def get_package_context(traceback, context=3): return lines -class InstallError(spack.error.SpackError): - """Raised by packages when a package fails to install. - - Any subclass of InstallError will be annotated by Spack with a - ``pkg`` attribute on failure, which the caller can use to get the - package for which the exception was raised. - """ - - class ChildError(InstallError): """Special exception class for wrapping exceptions from child processes in Spack's build environment. diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index 06dc4741ba..b1c777bb8b 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -10,7 +10,9 @@ import os import re import shutil import stat +import subprocess import tempfile +import time import zipfile from six import iteritems @@ -20,6 +22,7 @@ from six.moves.urllib.request import HTTPHandler, Request, build_opener import llnl.util.filesystem as fs import llnl.util.tty as tty +from llnl.util.lang import memoized import spack import spack.binary_distribution as bindist @@ -35,7 +38,10 @@ import spack.util.gpg as gpg_util import spack.util.spack_yaml as syaml import spack.util.web as web_util from spack.error import SpackError +from spack.reporters.cdash import CDash +from spack.reporters.cdash import build_stamp as cdash_build_stamp from spack.spec import Spec +from spack.util.pattern import Bunch JOB_RETRY_CONDITIONS = [ "always", @@ -60,69 +66,6 @@ class TemporaryDirectory(object): return False -def _create_buildgroup(opener, headers, url, project, group_name, group_type): - data = {"newbuildgroup": group_name, "project": project, "type": group_type} - - enc_data = json.dumps(data).encode("utf-8") - - request = Request(url, data=enc_data, headers=headers) - - response = opener.open(request) - response_code = response.getcode() - - if response_code != 200 and response_code != 201: - msg = "Creating buildgroup failed (response code = {0}".format(response_code) - tty.warn(msg) - return None - - response_text = response.read() - response_json = json.loads(response_text) - build_group_id = response_json["id"] - - return build_group_id - - -def _populate_buildgroup(job_names, group_name, project, site, credentials, cdash_url): - url = "{0}/api/v1/buildgroup.php".format(cdash_url) - - headers = { - "Authorization": "Bearer {0}".format(credentials), - "Content-Type": "application/json", - } - - opener = build_opener(HTTPHandler) - - parent_group_id = _create_buildgroup(opener, headers, url, project, group_name, "Daily") - group_id = _create_buildgroup( - opener, headers, url, project, "Latest {0}".format(group_name), "Latest" - ) - - if not parent_group_id or not group_id: - msg = "Failed to create or retrieve buildgroups for {0}".format(group_name) - tty.warn(msg) - return - - data = { - "project": project, - "buildgroupid": group_id, - "dynamiclist": [ - {"match": name, "parentgroupid": parent_group_id, "site": site} for name in job_names - ], - } - - enc_data = json.dumps(data).encode("utf-8") - - request = Request(url, data=enc_data, headers=headers) - request.get_method = lambda: "PUT" - - response = opener.open(request) - response_code = response.getcode() - - if response_code != 200: - msg = "Error response code ({0}) in _populate_buildgroup".format(response_code) - tty.warn(msg) - - def _is_main_phase(phase_name): return True if phase_name == "specs" else False @@ -180,12 +123,6 @@ def get_job_name(phase, strip_compiler, spec, osarch, build_group): return format_str.format(*format_args) -def _get_cdash_build_name(spec, build_group): - return "{0}@{1}%{2} arch={3} ({4})".format( - spec.name, spec.version, spec.compiler, spec.architecture, build_group - ) - - def _remove_reserved_tags(tags): """Convenience function to strip reserved tags from jobs""" return [tag for tag in tags if tag not in SPACK_RESERVED_TAGS] @@ -672,21 +609,8 @@ def generate_gitlab_ci_yaml( gitlab_ci = yaml_root["gitlab-ci"] - build_group = None - enable_cdash_reporting = False - cdash_auth_token = None - - if "cdash" in yaml_root: - enable_cdash_reporting = True - ci_cdash = yaml_root["cdash"] - build_group = ci_cdash["build-group"] - cdash_url = ci_cdash["url"] - cdash_project = ci_cdash["project"] - cdash_site = ci_cdash["site"] - - if "SPACK_CDASH_AUTH_TOKEN" in os.environ: - tty.verbose("Using CDash auth token from environment") - cdash_auth_token = os.environ.get("SPACK_CDASH_AUTH_TOKEN") + cdash_handler = CDashHandler(yaml_root.get("cdash")) if "cdash" in yaml_root else None + build_group = cdash_handler.build_group if cdash_handler else None prune_untouched_packages = os.environ.get("SPACK_PRUNE_UNTOUCHED", None) if prune_untouched_packages: @@ -820,6 +744,7 @@ def generate_gitlab_ci_yaml( job_log_dir = os.path.join(pipeline_artifacts_dir, "logs") job_repro_dir = os.path.join(pipeline_artifacts_dir, "reproduction") + job_test_dir = os.path.join(pipeline_artifacts_dir, "tests") local_mirror_dir = os.path.join(pipeline_artifacts_dir, "mirror") user_artifacts_dir = os.path.join(pipeline_artifacts_dir, "user_data") @@ -833,7 +758,8 @@ def generate_gitlab_ci_yaml( rel_concrete_env_dir = os.path.relpath(concrete_env_dir, ci_project_dir) rel_job_log_dir = os.path.relpath(job_log_dir, ci_project_dir) rel_job_repro_dir = os.path.relpath(job_repro_dir, ci_project_dir) - rel_local_mirror_dir = os.path.relpath(local_mirror_dir, ci_project_dir) + rel_job_test_dir = os.path.relpath(job_test_dir, ci_project_dir) + rel_local_mirror_dir = os.path.join(local_mirror_dir, ci_project_dir) rel_user_artifacts_dir = os.path.relpath(user_artifacts_dir, ci_project_dir) # Speed up staging by first fetching binary indices from all mirrors @@ -1101,14 +1027,23 @@ def generate_gitlab_ci_yaml( job_vars["SPACK_SPEC_NEEDS_REBUILD"] = str(rebuild_spec) - if enable_cdash_reporting: - cdash_build_name = _get_cdash_build_name(release_spec, build_group) - all_job_names.append(cdash_build_name) - job_vars["SPACK_CDASH_BUILD_NAME"] = cdash_build_name + if cdash_handler: + cdash_handler.current_spec = release_spec + build_name = cdash_handler.build_name + all_job_names.append(build_name) + job_vars["SPACK_CDASH_BUILD_NAME"] = build_name + + build_stamp = cdash_handler.build_stamp + job_vars["SPACK_CDASH_BUILD_STAMP"] = build_stamp variables.update(job_vars) - artifact_paths = [rel_job_log_dir, rel_job_repro_dir, rel_user_artifacts_dir] + artifact_paths = [ + rel_job_log_dir, + rel_job_repro_dir, + rel_job_test_dir, + rel_user_artifacts_dir, + ] if enable_artifacts_buildcache: bc_root = os.path.join(local_mirror_dir, "build_cache") @@ -1176,11 +1111,9 @@ def generate_gitlab_ci_yaml( ) # Use "all_job_names" to populate the build group for this set - if enable_cdash_reporting and cdash_auth_token: + if cdash_handler and cdash_handler.auth_token: try: - _populate_buildgroup( - all_job_names, build_group, cdash_project, cdash_site, cdash_auth_token, cdash_url - ) + cdash_handler.populate_buildgroup(all_job_names) except (SpackError, HTTPError, URLError) as err: tty.warn("Problem populating buildgroup: {0}".format(err)) else: @@ -1341,6 +1274,7 @@ def generate_gitlab_ci_yaml( "SPACK_REMOTE_MIRROR_URL": remote_mirror_url, "SPACK_JOB_LOG_DIR": rel_job_log_dir, "SPACK_JOB_REPRO_DIR": rel_job_repro_dir, + "SPACK_JOB_TEST_DIR": rel_job_test_dir, "SPACK_LOCAL_MIRROR_DIR": rel_local_mirror_dir, "SPACK_PIPELINE_TYPE": str(spack_pipeline_type), } @@ -1609,33 +1543,70 @@ def push_mirror_contents(env, specfile_path, mirror_url, sign_binaries): raise inst +def copy_files_to_artifacts(src, artifacts_dir): + """ + Copy file(s) to the given artifacts directory + + Parameters: + src (str): the glob-friendly path expression for the file(s) to copy + artifacts_dir (str): the destination directory + """ + try: + fs.copy(src, artifacts_dir) + except Exception as err: + msg = ("Unable to copy files ({0}) to artifacts {1} due to " "exception: {2}").format( + src, artifacts_dir, str(err) + ) + tty.error(msg) + + def copy_stage_logs_to_artifacts(job_spec, job_log_dir): - """Looks for spack-build-out.txt in the stage directory of the given - job_spec, and attempts to copy the file into the directory given - by job_log_dir. + """Copy selected build stage file(s) to the given artifacts directory - Arguments: + Looks for spack-build-out.txt in the stage directory of the given + job_spec, and attempts to copy the file into the directory given + by job_log_dir. - job_spec (spack.spec.Spec): Spec associated with spack install log - job_log_dir (str): Path into which build log should be copied + Parameters: + job_spec (spack.spec.Spec): spec associated with spack install log + job_log_dir (str): path into which build log should be copied """ + tty.debug("job spec: {0}".format(job_spec)) + if not job_spec: + msg = "Cannot copy stage logs: job spec ({0}) is required" + tty.error(msg.format(job_spec)) + return + try: pkg_cls = spack.repo.path.get_pkg_class(job_spec.name) job_pkg = pkg_cls(job_spec) - tty.debug("job package: {0.fullname}".format(job_pkg)) - stage_dir = job_pkg.stage.path - tty.debug("stage dir: {0}".format(stage_dir)) - build_out_src = os.path.join(stage_dir, "spack-build-out.txt") - build_out_dst = os.path.join(job_log_dir, "spack-build-out.txt") - tty.debug( - "Copying build log ({0}) to artifacts ({1})".format(build_out_src, build_out_dst) - ) - shutil.copyfile(build_out_src, build_out_dst) - except Exception as inst: - msg = ( - "Unable to copy build logs from stage to artifacts " "due to exception: {0}" - ).format(inst) - tty.error(msg) + tty.debug("job package: {0}".format(job_pkg)) + except AssertionError: + msg = "Cannot copy stage logs: job spec ({0}) must be concrete" + tty.error(msg.format(job_spec)) + return + + stage_dir = job_pkg.stage.path + tty.debug("stage dir: {0}".format(stage_dir)) + build_out_src = os.path.join(stage_dir, "spack-build-out.txt") + copy_files_to_artifacts(build_out_src, job_log_dir) + + +def copy_test_logs_to_artifacts(test_stage, job_test_dir): + """ + Copy test log file(s) to the given artifacts directory + + Parameters: + test_stage (str): test stage path + job_test_dir (str): the destination artifacts test directory + """ + tty.debug("test stage: {0}".format(test_stage)) + if not os.path.exists(test_stage): + msg = "Cannot copy test logs: job test stage ({0}) does not exist" + tty.error(msg.format(test_stage)) + return + + copy_files_to_artifacts(os.path.join(test_stage, "*", "*.txt"), job_test_dir) def download_and_extract_artifacts(url, work_dir): @@ -1985,3 +1956,323 @@ def reproduce_ci_job(url, work_dir): ) print("".join(inst_list)) + + +def process_command(cmd, cmd_args, repro_dir): + """ + Create a script for and run the command. Copy the script to the + reproducibility directory. + + Arguments: + cmd (str): name of the command being processed + cmd_args (list): string arguments to pass to the command + repro_dir (str): Job reproducibility directory + + Returns: the exit code from processing the command + """ + tty.debug("spack {0} arguments: {1}".format(cmd, cmd_args)) + + # Write the command to a shell script + script = "{0}.sh".format(cmd) + with open(script, "w") as fd: + fd.write("#!/bin/bash\n\n") + fd.write("\n# spack {0} command\n".format(cmd)) + fd.write(" ".join(['"{0}"'.format(i) for i in cmd_args])) + fd.write("\n") + + st = os.stat(script) + os.chmod(script, st.st_mode | stat.S_IEXEC) + + copy_path = os.path.join(repro_dir, script) + shutil.copyfile(script, copy_path) + + # Run the generated install.sh shell script as if it were being run in + # a login shell. + try: + cmd_process = subprocess.Popen(["bash", "./{0}".format(script)]) + cmd_process.wait() + exit_code = cmd_process.returncode + except (ValueError, subprocess.CalledProcessError, OSError) as err: + tty.error("Encountered error running {0} script".format(cmd)) + tty.error(err) + exit_code = 1 + + tty.debug("spack {0} exited {1}".format(cmd, exit_code)) + return exit_code + + +def create_buildcache(**kwargs): + """Create the buildcache at the provided mirror(s). + + Arguments: + kwargs (dict): dictionary of arguments used to create the buildcache + + List of recognized keys: + + * "env" (spack.environment.Environment): the active environment + * "buildcache_mirror_url" (str or None): URL for the buildcache mirror + * "pipeline_mirror_url" (str or None): URL for the pipeline mirror + * "pr_pipeline" (bool): True if the CI job is for a PR + * "json_path" (str): path the the spec's JSON file + """ + env = kwargs.get("env") + buildcache_mirror_url = kwargs.get("buildcache_mirror_url") + pipeline_mirror_url = kwargs.get("pipeline_mirror_url") + pr_pipeline = kwargs.get("pr_pipeline") + json_path = kwargs.get("json_path") + + sign_binaries = pr_pipeline is False and can_sign_binaries() + + # Create buildcache in either the main remote mirror, or in the + # per-PR mirror, if this is a PR pipeline + if buildcache_mirror_url: + push_mirror_contents(env, json_path, buildcache_mirror_url, sign_binaries) + + # Create another copy of that buildcache in the per-pipeline + # temporary storage mirror (this is only done if either + # artifacts buildcache is enabled or a temporary storage url + # prefix is set) + if pipeline_mirror_url: + push_mirror_contents(env, json_path, pipeline_mirror_url, sign_binaries) + + +def run_standalone_tests(**kwargs): + """Run stand-alone tests on the current spec. + + Arguments: + kwargs (dict): dictionary of arguments used to run the tests + + List of recognized keys: + + * "cdash" (CDashHandler): (optional) cdash handler instance + * "fail_fast" (bool): (optional) terminate tests after the first failure + * "log_file" (str): (optional) test log file name if NOT CDash reporting + * "job_spec" (Spec): spec that was built + * "repro_dir" (str): reproduction directory + """ + cdash = kwargs.get("cdash") + fail_fast = kwargs.get("fail_fast") + log_file = kwargs.get("log_file") + + if cdash and log_file: + tty.msg("The test log file {0} option is ignored with CDash reporting".format(log_file)) + log_file = None + + # Error out but do NOT terminate if there are missing required arguments. + job_spec = kwargs.get("job_spec") + if not job_spec: + tty.error("Job spec is required to run stand-alone tests") + return + + repro_dir = kwargs.get("repro_dir") + if not repro_dir: + tty.error("Reproduction directory is required for stand-alone tests") + return + + test_args = [ + "spack", + "-d", + "-v", + "test", + "run", + ] + if fail_fast: + test_args.append("--fail-fast") + + if cdash: + test_args.extend(cdash.args()) + else: + test_args.extend(["--log-format", "junit"]) + if log_file: + test_args.extend(["--log-file", log_file]) + test_args.append(job_spec.name) + + tty.debug("Running {0} stand-alone tests".format(job_spec.name)) + exit_code = process_command("test", test_args, repro_dir) + + tty.debug("spack test exited {0}".format(exit_code)) + + +class CDashHandler(object): + """ + Class for managing CDash data and processing. + """ + + def __init__(self, ci_cdash): + # start with the gitlab ci configuration + self.url = ci_cdash.get("url") + self.build_group = ci_cdash.get("build-group") + self.project = ci_cdash.get("project") + self.site = ci_cdash.get("site") + + # grab the authorization token when available + self.auth_token = os.environ.get("SPACK_CDASH_AUTH_TOKEN") + if self.auth_token: + tty.verbose("Using CDash auth token from environment") + + # append runner description to the site if available + runner = os.environ.get("CI_RUNNER_DESCRIPTION") + if runner: + self.site += " ({0})".format(runner) + + # track current spec, if any + self.current_spec = None + + def args(self): + return [ + "--cdash-upload-url", + self.upload_url, + "--cdash-build", + self.build_name, + "--cdash-site", + self.site, + "--cdash-buildstamp", + self.build_stamp, + ] + + @property # type: ignore + def build_name(self): + """Returns the CDash build name. + + A name will be generated if the `current_spec` property is set; + otherwise, the value will be retrieved from the environment + through the `SPACK_CDASH_BUILD_NAME` variable. + + Returns: (str) current spec's CDash build name.""" + spec = self.current_spec + if spec: + build_name = "{0}@{1}%{2} hash={3} arch={4} ({5})".format( + spec.name, + spec.version, + spec.compiler, + spec.dag_hash(), + spec.architecture, + self.build_group, + ) + tty.verbose( + "Generated CDash build name ({0}) from the {1}".format(build_name, spec.name) + ) + return build_name + + build_name = os.environ.get("SPACK_CDASH_BUILD_NAME") + tty.verbose("Using CDash build name ({0}) from the environment".format(build_name)) + return build_name + + @property # type: ignore + def build_stamp(self): + """Returns the CDash build stamp. + + The one defined by SPACK_CDASH_BUILD_STAMP environment variable + is preferred due to the representation of timestamps; otherwise, + one will be built. + + Returns: (str) current CDash build stamp""" + build_stamp = os.environ.get("SPACK_CDASH_BUILD_STAMP") + if build_stamp: + tty.verbose("Using build stamp ({0}) from the environment".format(build_stamp)) + return build_stamp + + build_stamp = cdash_build_stamp(self.build_group, time.time()) + tty.verbose("Generated new build stamp ({0})".format(build_stamp)) + return build_stamp + + @property # type: ignore + @memoized + def project_enc(self): + tty.debug("Encoding project ({0}): {1})".format(type(self.project), self.project)) + encode = urlencode({"project": self.project}) + index = encode.find("=") + 1 + return encode[index:] + + @property + def upload_url(self): + url_format = "{0}/submit.php?project={1}" + return url_format.format(self.url, self.project_enc) + + def copy_test_results(self, source, dest): + """Copy test results to artifacts directory.""" + reports = fs.join_path(source, "*_Test*.xml") + copy_files_to_artifacts(reports, dest) + + def create_buildgroup(self, opener, headers, url, group_name, group_type): + data = {"newbuildgroup": group_name, "project": self.project, "type": group_type} + + enc_data = json.dumps(data).encode("utf-8") + + request = Request(url, data=enc_data, headers=headers) + + response = opener.open(request) + response_code = response.getcode() + + if response_code not in [200, 201]: + msg = "Creating buildgroup failed (response code = {0})".format(response_code) + tty.warn(msg) + return None + + response_text = response.read() + response_json = json.loads(response_text) + build_group_id = response_json["id"] + + return build_group_id + + def populate_buildgroup(self, job_names): + url = "{0}/api/v1/buildgroup.php".format(self.url) + + headers = { + "Authorization": "Bearer {0}".format(self.auth_token), + "Content-Type": "application/json", + } + + opener = build_opener(HTTPHandler) + + parent_group_id = self.create_buildgroup( + opener, + headers, + url, + self.build_group, + "Daily", + ) + group_id = self.create_buildgroup( + opener, + headers, + url, + "Latest {0}".format(self.build_group), + "Latest", + ) + + if not parent_group_id or not group_id: + msg = "Failed to create or retrieve buildgroups for {0}".format(self.build_group) + tty.warn(msg) + return + + data = { + "dynamiclist": [ + { + "match": name, + "parentgroupid": parent_group_id, + "site": self.site, + } + for name in job_names + ], + } + + enc_data = json.dumps(data).encode("utf-8") + + request = Request(url, data=enc_data, headers=headers) + request.get_method = lambda: "PUT" + + response = opener.open(request) + response_code = response.getcode() + + if response_code != 200: + msg = "Error response code ({0}) in populate_buildgroup".format(response_code) + tty.warn(msg) + + def report_skipped(self, spec, directory_name, reason): + cli_args = self.args() + cli_args.extend(["package", [spec.name]]) + it = iter(cli_args) + kv = {x.replace("--", "").replace("-", "_"): next(it) for x in it} + + reporter = CDash(Bunch(**kv)) + reporter.test_skipped_report(directory_name, spec, reason) diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 49e5c70019..bf7f1a5959 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -640,3 +640,8 @@ def find_environment(args): return ev.Environment(env) raise ev.SpackEnvironmentError("no environment in %s" % env) + + +def first_line(docstring): + """Return the first line of the docstring.""" + return docstring.split("\n")[0] diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index 3087a7881e..7bb0497c81 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -6,13 +6,10 @@ import json import os import shutil -import stat -import subprocess import sys import tempfile -from six.moves.urllib.parse import urlencode - +import llnl.util.filesystem as fs import llnl.util.tty as tty import spack.binary_distribution as bindist @@ -34,6 +31,10 @@ CI_REBUILD_INSTALL_BASE_ARGS = ["spack", "-d", "-v"] INSTALL_FAIL_CODE = 1 +def deindent(desc): + return desc.replace(" ", "") + + def get_env_var(variable_name): if variable_name in os.environ: return os.environ.get(variable_name) @@ -45,27 +46,35 @@ def setup_parser(subparser): subparsers = subparser.add_subparsers(help="CI sub-commands") # Dynamic generation of the jobs yaml from a spack environment - generate = subparsers.add_parser("generate", help=ci_generate.__doc__) + generate = subparsers.add_parser( + "generate", + description=deindent(ci_generate.__doc__), + help=spack.cmd.first_line(ci_generate.__doc__), + ) generate.add_argument( "--output-file", default=None, - help="Path to file where generated jobs file should be " - + "written. The default is .gitlab-ci.yml in the root of the " - + "repository.", + help="""pathname for the generated gitlab ci yaml file + Path to the file where generated jobs file should +be written. Default is .gitlab-ci.yml in the root of +the repository.""", ) generate.add_argument( "--copy-to", default=None, - help="Absolute path of additional location where generated jobs " - + "yaml file should be copied. Default is not to copy.", + help="""path to additional directory for job files + This option provides an absolute path to a directory +where the generated jobs yaml file should be copied. +Default is not to copy.""", ) generate.add_argument( "--optimize", action="store_true", default=False, - help="(Experimental) run the generated document through a series of " - "optimization passes designed to reduce the size of the " - "generated file.", + help="""(Experimental) optimize the gitlab yaml file for size + Run the generated document through a series of +optimization passes designed to reduce the size +of the generated file.""", ) generate.add_argument( "--dependencies", @@ -86,53 +95,84 @@ def setup_parser(subparser): action="store_true", dest="prune_dag", default=True, - help="""Do not generate jobs for specs already up to -date on the mirror""", + help="""skip up-to-date specs + Do not generate jobs for specs that are up-to-date +on the mirror.""", ) prune_group.add_argument( "--no-prune-dag", action="store_false", dest="prune_dag", default=True, - help="""Generate jobs for specs already up to date -on the mirror""", + help="""process up-to-date specs + Generate jobs for specs even when they are up-to-date +on the mirror.""", ) generate.add_argument( "--check-index-only", action="store_true", dest="index_only", default=False, - help="""Spack always check specs against configured -binary mirrors when generating the pipeline, regardless of whether or not -DAG pruning is enabled. This flag controls whether it might attempt to -fetch remote spec files directly (ensuring no spec is rebuilt if it -is present on the mirror), or whether it should reduce pipeline generation time -by assuming all remote buildcache indices are up to date and only use those -to determine whether a given spec is up to date on mirrors. In the latter -case, specs might be needlessly rebuilt if remote buildcache indices are out -of date.""", + help="""only check spec state from buildcache indices + Spack always checks specs against configured binary +mirrors, regardless of the DAG pruning option. + If enabled, Spack will assume all remote buildcache +indices are up-to-date when assessing whether the spec +on the mirror, if present, is up-to-date. This has the +benefit of reducing pipeline generation time but at the +potential cost of needlessly rebuilding specs when the +indices are outdated. + If not enabled, Spack will fetch remote spec files +directly to assess whether the spec on the mirror is +up-to-date.""", ) generate.add_argument( "--artifacts-root", default=None, - help="""Path to root of artifacts directory. If provided, concrete -environment files (spack.yaml, spack.lock) will be generated under this -path and their location sent to generated child jobs via the custom job -variable SPACK_CONCRETE_ENVIRONMENT_PATH.""", + help="""path to the root of the artifacts directory + If provided, concrete environment files (spack.yaml, +spack.lock) will be generated under this directory. +Their location will be passed to generated child jobs +through the SPACK_CONCRETE_ENVIRONMENT_PATH variable.""", ) generate.set_defaults(func=ci_generate) # Rebuild the buildcache index associated with the mirror in the # active, gitlab-enabled environment. - index = subparsers.add_parser("rebuild-index", help=ci_reindex.__doc__) + index = subparsers.add_parser( + "rebuild-index", + description=deindent(ci_reindex.__doc__), + help=spack.cmd.first_line(ci_reindex.__doc__), + ) index.set_defaults(func=ci_reindex) # Handle steps of a ci build/rebuild - rebuild = subparsers.add_parser("rebuild", help=ci_rebuild.__doc__) + rebuild = subparsers.add_parser( + "rebuild", + description=deindent(ci_rebuild.__doc__), + help=spack.cmd.first_line(ci_rebuild.__doc__), + ) + rebuild.add_argument( + "-t", + "--tests", + action="store_true", + default=False, + help="""run stand-alone tests after the build""", + ) + rebuild.add_argument( + "--fail-fast", + action="store_true", + default=False, + help="""stop stand-alone tests after the first failure""", + ) rebuild.set_defaults(func=ci_rebuild) # Facilitate reproduction of a failed CI build job - reproduce = subparsers.add_parser("reproduce-build", help=ci_reproduce.__doc__) + reproduce = subparsers.add_parser( + "reproduce-build", + description=deindent(ci_reproduce.__doc__), + help=spack.cmd.first_line(ci_reproduce.__doc__), + ) reproduce.add_argument("job_url", help="Url of job artifacts bundle") reproduce.add_argument( "--working-dir", @@ -144,12 +184,12 @@ variable SPACK_CONCRETE_ENVIRONMENT_PATH.""", def ci_generate(args): - """Generate jobs file from a spack environment file containing CI info. - Before invoking this command, you can set the environment variable - SPACK_CDASH_AUTH_TOKEN to contain the CDash authorization token - for creating a build group for the generated workload and registering - all generated jobs under that build group. If this environment - variable is not set, no build group will be created on CDash.""" + """Generate jobs file from a CI-aware spack file. + + If you want to report the results on CDash, you will need to set + the SPACK_CDASH_AUTH_TOKEN before invoking this command. The + value must be the CDash authorization token needed to create a + build group and register all generated jobs under it.""" env = spack.cmd.require_active_env(cmd_name="ci generate") output_file = args.output_file @@ -190,8 +230,10 @@ def ci_generate(args): def ci_reindex(args): - """Rebuild the buildcache index associated with the mirror in the - active, gitlab-enabled environment.""" + """Rebuild the buildcache index for the remote mirror. + + Use the active, gitlab-enabled environment to rebuild the buildcache + index for the associated mirror.""" env = spack.cmd.require_active_env(cmd_name="ci rebuild-index") yaml_root = ev.config_dict(env.yaml) @@ -206,17 +248,16 @@ def ci_reindex(args): def ci_rebuild(args): - """Check a single spec against the remote mirror, and rebuild it from + """Rebuild a spec if it is not on the remote mirror. + + Check a single spec against the remote mirror, and rebuild it from source if the mirror does not contain the hash.""" env = spack.cmd.require_active_env(cmd_name="ci rebuild") # Make sure the environment is "gitlab-enabled", or else there's nothing # to do. yaml_root = ev.config_dict(env.yaml) - gitlab_ci = None - if "gitlab-ci" in yaml_root: - gitlab_ci = yaml_root["gitlab-ci"] - + gitlab_ci = yaml_root["gitlab-ci"] if "gitlab-ci" in yaml_root else None if not gitlab_ci: tty.die("spack ci rebuild requires an env containing gitlab-ci cfg") @@ -231,6 +272,7 @@ def ci_rebuild(args): # out as variables, or else provided by GitLab itself. pipeline_artifacts_dir = get_env_var("SPACK_ARTIFACTS_ROOT") job_log_dir = get_env_var("SPACK_JOB_LOG_DIR") + job_test_dir = get_env_var("SPACK_JOB_TEST_DIR") repro_dir = get_env_var("SPACK_JOB_REPRO_DIR") local_mirror_dir = get_env_var("SPACK_LOCAL_MIRROR_DIR") concrete_env_dir = get_env_var("SPACK_CONCRETE_ENV_DIR") @@ -240,7 +282,6 @@ def ci_rebuild(args): root_spec = get_env_var("SPACK_ROOT_SPEC") job_spec_pkg_name = get_env_var("SPACK_JOB_SPEC_PKG_NAME") compiler_action = get_env_var("SPACK_COMPILER_ACTION") - cdash_build_name = get_env_var("SPACK_CDASH_BUILD_NAME") spack_pipeline_type = get_env_var("SPACK_PIPELINE_TYPE") remote_mirror_override = get_env_var("SPACK_REMOTE_MIRROR_OVERRIDE") remote_mirror_url = get_env_var("SPACK_REMOTE_MIRROR_URL") @@ -249,6 +290,7 @@ def ci_rebuild(args): ci_project_dir = get_env_var("CI_PROJECT_DIR") pipeline_artifacts_dir = os.path.join(ci_project_dir, pipeline_artifacts_dir) job_log_dir = os.path.join(ci_project_dir, job_log_dir) + job_test_dir = os.path.join(ci_project_dir, job_test_dir) repro_dir = os.path.join(ci_project_dir, repro_dir) local_mirror_dir = os.path.join(ci_project_dir, local_mirror_dir) concrete_env_dir = os.path.join(ci_project_dir, concrete_env_dir) @@ -263,23 +305,15 @@ def ci_rebuild(args): # Query the environment manifest to find out whether we're reporting to a # CDash instance, and if so, gather some information from the manifest to # support that task. - enable_cdash = False - if "cdash" in yaml_root: - enable_cdash = True - ci_cdash = yaml_root["cdash"] - job_spec_buildgroup = ci_cdash["build-group"] - cdash_base_url = ci_cdash["url"] - cdash_project = ci_cdash["project"] - proj_enc = urlencode({"project": cdash_project}) - eq_idx = proj_enc.find("=") + 1 - cdash_project_enc = proj_enc[eq_idx:] - cdash_site = ci_cdash["site"] - tty.debug("cdash_base_url = {0}".format(cdash_base_url)) - tty.debug("cdash_project = {0}".format(cdash_project)) - tty.debug("cdash_project_enc = {0}".format(cdash_project_enc)) - tty.debug("cdash_build_name = {0}".format(cdash_build_name)) - tty.debug("cdash_site = {0}".format(cdash_site)) - tty.debug("job_spec_buildgroup = {0}".format(job_spec_buildgroup)) + cdash_handler = spack_ci.CDashHandler(yaml_root.get("cdash")) if "cdash" in yaml_root else None + if cdash_handler: + tty.debug("cdash url = {0}".format(cdash_handler.url)) + tty.debug("cdash project = {0}".format(cdash_handler.project)) + tty.debug("cdash project_enc = {0}".format(cdash_handler.project_enc)) + tty.debug("cdash build_name = {0}".format(cdash_handler.build_name)) + tty.debug("cdash build_stamp = {0}".format(cdash_handler.build_stamp)) + tty.debug("cdash site = {0}".format(cdash_handler.site)) + tty.debug("cdash build_group = {0}".format(cdash_handler.build_group)) # Is this a pipeline run on a spack PR or a merge to develop? It might # be neither, e.g. a pipeline run on some environment repository. @@ -344,6 +378,9 @@ def ci_rebuild(args): if os.path.exists(job_log_dir): shutil.rmtree(job_log_dir) + if os.path.exists(job_test_dir): + shutil.rmtree(job_test_dir) + if os.path.exists(repro_dir): shutil.rmtree(repro_dir) @@ -351,6 +388,7 @@ def ci_rebuild(args): # need for storing artifacts. The cdash_report directory will be # created internally if needed. os.makedirs(job_log_dir) + os.makedirs(job_test_dir) os.makedirs(repro_dir) # Copy the concrete environment files to the repro directory so we can @@ -468,6 +506,7 @@ def ci_rebuild(args): install_args.extend( [ "install", + "--show-log-on-error", # Print full log on fails "--keep-stage", ] ) @@ -477,22 +516,9 @@ def ci_rebuild(args): if not verify_binaries: install_args.append("--no-check-signature") - if enable_cdash: + if cdash_handler: # Add additional arguments to `spack install` for CDash reporting. - cdash_upload_url = "{0}/submit.php?project={1}".format(cdash_base_url, cdash_project_enc) - - install_args.extend( - [ - "--cdash-upload-url", - cdash_upload_url, - "--cdash-build", - cdash_build_name, - "--cdash-site", - cdash_site, - "--cdash-track", - job_spec_buildgroup, - ] - ) + install_args.extend(cdash_handler.args()) # A compiler action of 'FIND_ANY' means we are building a bootstrap # compiler or one of its deps. @@ -506,29 +532,7 @@ def ci_rebuild(args): install_args.extend(["-f", job_spec_json_path]) tty.debug("Installing {0} from source".format(job_spec.name)) - tty.debug("spack install arguments: {0}".format(install_args)) - - # Write the install command to a shell script - with open("install.sh", "w") as fd: - fd.write("#!/bin/bash\n\n") - fd.write("\n# spack install command\n") - fd.write(" ".join(['"{0}"'.format(i) for i in install_args])) - fd.write("\n") - - st = os.stat("install.sh") - os.chmod("install.sh", st.st_mode | stat.S_IEXEC) - - install_copy_path = os.path.join(repro_dir, "install.sh") - shutil.copyfile("install.sh", install_copy_path) - - # Run the generated install.sh shell script - try: - install_process = subprocess.Popen(["bash", "./install.sh"]) - install_process.wait() - install_exit_code = install_process.returncode - except (ValueError, subprocess.CalledProcessError, OSError) as inst: - tty.error("Encountered error running install script") - tty.error(inst) + install_exit_code = spack_ci.process_command("install", install_args, repro_dir) # Now do the post-install tasks tty.debug("spack install exited {0}".format(install_exit_code)) @@ -564,7 +568,7 @@ def ci_rebuild(args): extra_args={"ContentType": "text/plain"}, ) except Exception as err: - # If we got some kind of S3 (access denied or other connection + # If there is an S3 error (e.g., access denied or connection # error), the first non boto-specific class in the exception # hierarchy is Exception. Just print a warning and return msg = "Error writing to broken specs list {0}: {1}".format(broken_spec_path, err) @@ -576,28 +580,79 @@ def ci_rebuild(args): # any logs from the staging directory to artifacts now spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir) + # If the installation succeeded and we're running stand-alone tests for + # the package, run them and copy the output. Failures of any kind should + # *not* terminate the build process or preclude creating the build cache. + broken_tests = ( + "broken-tests-packages" in gitlab_ci + and job_spec.name in gitlab_ci["broken-tests-packages"] + ) + reports_dir = fs.join_path(os.getcwd(), "cdash_report") + if args.tests and broken_tests: + tty.warn( + "Unable to run stand-alone tests since listed in " + "gitlab-ci's 'broken-tests-packages'" + ) + if cdash_handler: + msg = "Package is listed in gitlab-ci's broken-tests-packages" + cdash_handler.report_skipped(job_spec, reports_dir, reason=msg) + cdash_handler.copy_test_results(reports_dir, job_test_dir) + elif args.tests: + if install_exit_code == 0: + try: + # First ensure we will use a reasonable test stage directory + stage_root = os.path.dirname(str(job_spec.package.stage.path)) + test_stage = fs.join_path(stage_root, "spack-standalone-tests") + tty.debug("Configuring test_stage to {0}".format(test_stage)) + config_test_path = "config:test_stage:{0}".format(test_stage) + cfg.add(config_test_path, scope=cfg.default_modify_scope()) + + # Run the tests, resorting to junit results if not using cdash + log_file = ( + None if cdash_handler else fs.join_path(test_stage, "ci-test-results.xml") + ) + spack_ci.run_standalone_tests( + cdash=cdash_handler, + job_spec=job_spec, + fail_fast=args.fail_fast, + log_file=log_file, + repro_dir=repro_dir, + ) + + except Exception as err: + # If there is any error, just print a warning. + msg = "Error processing stand-alone tests: {0}".format(str(err)) + tty.warn(msg) + + finally: + # Copy the test log/results files + spack_ci.copy_test_logs_to_artifacts(test_stage, job_test_dir) + if cdash_handler: + cdash_handler.copy_test_results(reports_dir, job_test_dir) + elif log_file: + spack_ci.copy_files_to_artifacts(log_file, job_test_dir) + else: + tty.warn("No recognized test results reporting option") + + else: + tty.warn("Unable to run stand-alone tests due to unsuccessful " "installation") + if cdash_handler: + msg = "Failed to install the package" + cdash_handler.report_skipped(job_spec, reports_dir, reason=msg) + cdash_handler.copy_test_results(reports_dir, job_test_dir) + # If the install succeeded, create a buildcache entry for this job spec # and push it to one or more mirrors. If the install did not succeed, # print out some instructions on how to reproduce this build failure # outside of the pipeline environment. if install_exit_code == 0: - can_sign = spack_ci.can_sign_binaries() - sign_binaries = can_sign and spack_is_pr_pipeline is False - - # Create buildcache in either the main remote mirror, or in the - # per-PR mirror, if this is a PR pipeline - if buildcache_mirror_url: - spack_ci.push_mirror_contents( - env, job_spec_json_path, buildcache_mirror_url, sign_binaries - ) - - # Create another copy of that buildcache in the per-pipeline - # temporary storage mirror (this is only done if either - # artifacts buildcache is enabled or a temporary storage url - # prefix is set) - if pipeline_mirror_url: - spack_ci.push_mirror_contents( - env, job_spec_json_path, pipeline_mirror_url, sign_binaries + if buildcache_mirror_url or pipeline_mirror_url: + spack_ci.create_buildcache( + env=env, + buildcache_mirror_url=buildcache_mirror_url, + pipeline_mirror_url=pipeline_mirror_url, + pr_pipeline=spack_is_pr_pipeline, + json_path=job_spec_json_path, ) # If this is a develop pipeline, check if the spec that we just built is @@ -611,13 +666,11 @@ def ci_rebuild(args): try: web_util.remove_url(broken_spec_path) except Exception as err: - # If we got some kind of S3 (access denied or other connection + # If there is an S3 error (e.g., access denied or connection # error), the first non boto-specific class in the exception - # hierarchy is Exception. Just print a warning and return - msg = "Error removing {0} from broken specs list: {1}".format( - broken_spec_path, err - ) - tty.warn(msg) + # hierarchy is Exception. Just print a warning and return. + msg = "Error removing {0} from broken specs list: {1}" + tty.warn(msg.format(broken_spec_path, err)) else: tty.debug("spack install exited non-zero, will not create buildcache") @@ -654,6 +707,10 @@ If this project does not have public pipelines, you will need to first: def ci_reproduce(args): + """Generate instructions for reproducing the spec rebuild job. + + Artifacts of the provided gitlab pipeline rebuild job's URL will be + used to derive instructions for reproducing the build locally.""" job_url = args.job_url work_dir = args.working_dir diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index f74b164b31..dd3d5b8c9c 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -236,6 +236,7 @@ def install_specs(specs, install_kwargs, cli_args): except spack.build_environment.InstallError as e: if cli_args.show_log_on_error: e.print_context() + assert e.pkg, "Expected InstallError to include the associated package" if not os.path.exists(e.pkg.build_log_path): tty.error("'spack install' created no log.") else: diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index ec062f259a..fcd72a123e 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -29,17 +29,14 @@ section = "admin" level = "long" -def first_line(docstring): - """Return the first line of the docstring.""" - return docstring.split("\n")[0] - - def setup_parser(subparser): sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="test_command") # Run run_parser = sp.add_parser( - "run", description=test_run.__doc__, help=first_line(test_run.__doc__) + "run", + description=test_run.__doc__, + help=spack.cmd.first_line(test_run.__doc__), ) alias_help_msg = "Provide an alias for this test-suite" @@ -83,7 +80,9 @@ def setup_parser(subparser): # List list_parser = sp.add_parser( - "list", description=test_list.__doc__, help=first_line(test_list.__doc__) + "list", + description=test_list.__doc__, + help=spack.cmd.first_line(test_list.__doc__), ) list_parser.add_argument( "-a", @@ -97,7 +96,9 @@ def setup_parser(subparser): # Find find_parser = sp.add_parser( - "find", description=test_find.__doc__, help=first_line(test_find.__doc__) + "find", + description=test_find.__doc__, + help=spack.cmd.first_line(test_find.__doc__), ) find_parser.add_argument( "filter", @@ -107,7 +108,9 @@ def setup_parser(subparser): # Status status_parser = sp.add_parser( - "status", description=test_status.__doc__, help=first_line(test_status.__doc__) + "status", + description=test_status.__doc__, + help=spack.cmd.first_line(test_status.__doc__), ) status_parser.add_argument( "names", nargs=argparse.REMAINDER, help="Test suites for which to print status" @@ -115,7 +118,9 @@ def setup_parser(subparser): # Results results_parser = sp.add_parser( - "results", description=test_results.__doc__, help=first_line(test_results.__doc__) + "results", + description=test_results.__doc__, + help=spack.cmd.first_line(test_results.__doc__), ) results_parser.add_argument( "-l", "--logs", action="store_true", help="print the test log for each matching package" @@ -142,7 +147,9 @@ def setup_parser(subparser): # Remove remove_parser = sp.add_parser( - "remove", description=test_remove.__doc__, help=first_line(test_remove.__doc__) + "remove", + description=test_remove.__doc__, + help=spack.cmd.first_line(test_remove.__doc__), ) arguments.add_common_arguments(remove_parser, ["yes_to_all"]) remove_parser.add_argument( @@ -191,6 +198,16 @@ environment variables: matching = spack.store.db.query_local(spec, hashes=hashes) if spec and not matching: tty.warn("No installed packages match spec %s" % spec) + """ + TODO: Need to write out a log message and/or CDASH Testing + output that package not installed IF continue to process + these issues here. + + if args.log_format: + # Proceed with the spec assuming the test process + # to ensure report package as skipped (e.g., for CI) + specs_to_test.append(spec) + """ specs_to_test.extend(matching) # test_stage_dir diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index 105b60642f..a0869cf533 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -1694,7 +1694,7 @@ class Environment(object): spec for already concretized but not yet installed specs. """ # use a transaction to avoid overhead of repeated calls - # to `package.installed` + # to `package.spec.installed` with spack.store.db.read_transaction(): concretized = dict(self.concretized_specs()) for spec in self.user_specs: diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py index 861db1a556..da2b73032e 100644 --- a/lib/spack/spack/install_test.py +++ b/lib/spack/spack/install_test.py @@ -12,6 +12,7 @@ import sys import six import llnl.util.filesystem as fs +import llnl.util.tty as tty import spack.error import spack.paths @@ -180,6 +181,9 @@ class TestSuite(object): if spec.external and not externals: status = "SKIPPED" skipped += 1 + elif not spec.installed: + status = "SKIPPED" + skipped += 1 else: status = "NO-TESTS" untested += 1 @@ -187,6 +191,7 @@ class TestSuite(object): self.write_test_result(spec, status) except BaseException as exc: self.fails += 1 + tty.debug("Test failure: {0}".format(str(exc))) if isinstance(exc, (SyntaxError, TestSuiteSpecError)): # Create the test log file and report the error. self.ensure_stage() diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index da5181c473..d97db35535 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -820,7 +820,7 @@ class PackageInstaller(object): if spack.store.db.prefix_failed(dep): action = "'spack install' the dependency" msg = "{0} is marked as an install failure: {1}".format(dep_id, action) - raise InstallError(err.format(request.pkg_id, msg)) + raise InstallError(err.format(request.pkg_id, msg), pkg=dep_pkg) # Attempt to get a read lock to ensure another process does not # uninstall the dependency while the requested spec is being @@ -828,7 +828,7 @@ class PackageInstaller(object): ltype, lock = self._ensure_locked("read", dep_pkg) if lock is None: msg = "{0} is write locked by another process".format(dep_id) - raise InstallError(err.format(request.pkg_id, msg)) + raise InstallError(err.format(request.pkg_id, msg), pkg=request.pkg) # Flag external and upstream packages as being installed if dep_pkg.spec.external or dep_pkg.spec.installed_upstream: @@ -883,6 +883,7 @@ class PackageInstaller(object): "Install prefix collision for {0}".format(task.pkg_id), long_msg="Prefix directory {0} already used by another " "installed spec.".format(task.pkg.spec.prefix), + pkg=task.pkg, ) # Make sure the installation directory is in the desired state @@ -1571,7 +1572,8 @@ class PackageInstaller(object): raise InstallError( "Cannot proceed with {0}: {1} uninstalled {2}: {3}".format( pkg_id, task.priority, dep_str, ",".join(task.uninstalled_deps) - ) + ), + pkg=pkg, ) # Skip the installation if the spec is not being installed locally @@ -1596,7 +1598,7 @@ class PackageInstaller(object): spack.hooks.on_install_failure(task.request.pkg.spec) if self.fail_fast: - raise InstallError(fail_fast_err) + raise InstallError(fail_fast_err, pkg=pkg) continue @@ -1718,7 +1720,7 @@ class PackageInstaller(object): ) # Terminate if requested to do so on the first failure. if self.fail_fast: - raise InstallError("{0}: {1}".format(fail_fast_err, str(exc))) + raise InstallError("{0}: {1}".format(fail_fast_err, str(exc)), pkg=pkg) # Terminate at this point if the single explicit spec has # failed to install. @@ -1727,7 +1729,7 @@ class PackageInstaller(object): # Track explicit spec id and error to summarize when done if task.explicit: - failed_explicits.append((pkg_id, str(exc))) + failed_explicits.append((pkg, pkg_id, str(exc))) finally: # Remove the install prefix if anything went wrong during @@ -1750,19 +1752,38 @@ class PackageInstaller(object): # Ensure we properly report if one or more explicit specs failed # or were not installed when should have been. missing = [ - request.pkg_id + (request.pkg, request.pkg_id) for request in self.build_requests if request.install_args.get("install_package") and request.pkg_id not in self.installed ] + if failed_explicits or missing: - for pkg_id, err in failed_explicits: + for _, pkg_id, err in failed_explicits: tty.error("{0}: {1}".format(pkg_id, err)) - for pkg_id in missing: + for _, pkg_id in missing: tty.error("{0}: Package was not installed".format(pkg_id)) + pkg = None + if len(failed_explicits) > 0: + pkg = failed_explicits[0][0] + ids = [pkg_id for _, pkg_id, _ in failed_explicits] + tty.debug( + "Associating installation failure with first failed " + "explicit package ({0}) from {1}".format(ids[0], ", ".join(ids)) + ) + + if not pkg and len(missing) > 0: + pkg = missing[0][0] + ids = [pkg_id for _, pkg_id in missing] + tty.debug( + "Associating installation failure with first " + "missing package ({0}) from {1}".format(ids[0], ", ".join(ids)) + ) + raise InstallError( - "Installation request failed. Refer to " "reported errors for failing package(s)." + "Installation request failed. Refer to reported errors for failing package(s).", + pkg=pkg, ) @@ -2060,7 +2081,7 @@ class BuildTask(object): # queue. if status == STATUS_REMOVED: msg = "Cannot create a build task for {0} with status '{1}'" - raise InstallError(msg.format(self.pkg_id, status)) + raise InstallError(msg.format(self.pkg_id, status), pkg=pkg) self.status = status @@ -2351,10 +2372,15 @@ class BuildRequest(object): class InstallError(spack.error.SpackError): - """Raised when something goes wrong during install or uninstall.""" + """Raised when something goes wrong during install or uninstall. - def __init__(self, message, long_msg=None): + The error can be annotated with a ``pkg`` attribute to allow the + caller to get the package for which the exception was raised. + """ + + def __init__(self, message, long_msg=None, pkg=None): super(InstallError, self).__init__(message, long_msg) + self.pkg = pkg class BadInstallPhase(InstallError): diff --git a/lib/spack/spack/package_base.py b/lib/spack/spack/package_base.py index 441563eeba..81801df9a5 100644 --- a/lib/spack/spack/package_base.py +++ b/lib/spack/spack/package_base.py @@ -2840,6 +2840,10 @@ def test_process(pkg, kwargs): print_test_message(logger, "Skipped tests for external package", verbose) return + if not pkg.spec.installed: + print_test_message(logger, "Skipped not installed package", verbose) + return + # run test methods from the package and all virtuals it # provides virtuals have to be deduped by name v_names = list(set([vspec.name for vspec in pkg.virtuals_provided])) @@ -2910,6 +2914,9 @@ def test_process(pkg, kwargs): # non-pass-only methods if ran_actual_test_function: fsys.touch(pkg.tested_file) + # log one more test message to provide a completion timestamp + # for CDash reporting + tty.msg("Completed testing") else: print_test_message(logger, "No tests to run", verbose) diff --git a/lib/spack/spack/report.py b/lib/spack/spack/report.py index ec81502887..8d4fb2b81d 100644 --- a/lib/spack/spack/report.py +++ b/lib/spack/spack/report.py @@ -245,6 +245,7 @@ class collect_info(object): self.cls = cls self.function = function self.filename = None + self.ctest_parsing = getattr(args, "ctest_parsing", False) if args.cdash_upload_url: self.format_name = "cdash" self.filename = "cdash_report" @@ -271,10 +272,10 @@ class collect_info(object): def __exit__(self, exc_type, exc_val, exc_tb): if self.format_name: - # Close the collector and restore the - # original PackageInstaller._install_task + # Close the collector and restore the original function self.collector.__exit__(exc_type, exc_val, exc_tb) report_data = {"specs": self.collector.specs} + report_data["ctest-parsing"] = self.ctest_parsing report_fn = getattr(self.report_writer, "%s_report" % self.type) report_fn(self.filename, report_data) diff --git a/lib/spack/spack/reporters/cdash.py b/lib/spack/spack/reporters/cdash.py index 6107aaed50..2c57306412 100644 --- a/lib/spack/spack/reporters/cdash.py +++ b/lib/spack/spack/reporters/cdash.py @@ -23,8 +23,10 @@ from llnl.util.filesystem import working_dir import spack.build_environment import spack.fetch_strategy import spack.package_base +import spack.platforms from spack.error import SpackError from spack.reporter import Reporter +from spack.reporters.extract import extract_test_parts from spack.util.crypto import checksum from spack.util.executable import which from spack.util.log_parse import parse_log_events @@ -46,6 +48,11 @@ cdash_phases = set(map_phases_to_cdash.values()) cdash_phases.add("update") +def build_stamp(track, timestamp): + buildstamp_format = "%Y%m%d-%H%M-{0}".format(track) + return time.strftime(buildstamp_format, time.localtime(timestamp)) + + class CDash(Reporter): """Generate reports of spec installations for CDash. @@ -80,6 +87,9 @@ class CDash(Reporter): packages = args.spec elif getattr(args, "specs", ""): packages = args.specs + elif getattr(args, "package", ""): + # Ensure CI 'spack test run' can output CDash results + packages = args.package else: packages = [] for file in args.specfiles: @@ -90,29 +100,36 @@ class CDash(Reporter): self.base_buildname = args.cdash_build or self.install_command self.site = args.cdash_site or socket.gethostname() self.osname = platform.system() + self.osrelease = platform.release() + self.target = spack.platforms.host().target("default_target") self.endtime = int(time.time()) - if args.cdash_buildstamp: - self.buildstamp = args.cdash_buildstamp - else: - buildstamp_format = "%Y%m%d-%H%M-{0}".format(args.cdash_track) - self.buildstamp = time.strftime(buildstamp_format, time.localtime(self.endtime)) + self.buildstamp = ( + args.cdash_buildstamp + if args.cdash_buildstamp + else build_stamp(args.cdash_track, self.endtime) + ) self.buildIds = collections.OrderedDict() self.revision = "" git = which("git") with working_dir(spack.paths.spack_root): self.revision = git("rev-parse", "HEAD", output=str).strip() + self.generator = "spack-{0}".format(spack.main.get_version()) self.multiple_packages = False + def report_build_name(self, pkg_name): + return ( + "{0} - {1}".format(self.base_buildname, pkg_name) + if self.multiple_packages + else self.base_buildname + ) + def build_report_for_package(self, directory_name, package, duration): if "stdout" not in package: # Skip reporting on packages that did not generate any output. return self.current_package_name = package["name"] - if self.multiple_packages: - self.buildname = "{0} - {1}".format(self.base_buildname, package["name"]) - else: - self.buildname = self.base_buildname + self.buildname = self.report_build_name(self.current_package_name) report_data = self.initialize_report(directory_name) for phase in cdash_phases: report_data[phase] = {} @@ -228,6 +245,7 @@ class CDash(Reporter): # Do an initial scan to determine if we are generating reports for more # than one package. When we're only reporting on a single package we # do not explicitly include the package's name in the CDash build name. + self.multipe_packages = False num_packages = 0 for spec in input_data["specs"]: # Do not generate reports for packages that were installed @@ -255,27 +273,19 @@ class CDash(Reporter): self.build_report_for_package(directory_name, package, duration) self.finalize_report() - def test_report_for_package(self, directory_name, package, duration): - if "stdout" not in package: - # Skip reporting on packages that did not generate any output. - return - - self.current_package_name = package["name"] - self.buildname = "{0} - {1}".format(self.base_buildname, package["name"]) - - report_data = self.initialize_report(directory_name) + def extract_ctest_test_data(self, package, phases, report_data): + """Extract ctest test data for the package.""" + # Track the phases we perform so we know what reports to create. + # We always report the update step because this is how we tell CDash + # what revision of Spack we are using. + assert "update" in phases - for phase in ("test", "update"): + for phase in phases: report_data[phase] = {} report_data[phase]["loglines"] = [] report_data[phase]["status"] = 0 report_data[phase]["endtime"] = self.endtime - # Track the phases we perform so we know what reports to create. - # We always report the update step because this is how we tell CDash - # what revision of Spack we are using. - phases_encountered = ["test", "update"] - # Generate a report for this package. # The first line just says "Testing package name-hash" report_data["test"]["loglines"].append( @@ -284,8 +294,7 @@ class CDash(Reporter): for line in package["stdout"].splitlines()[1:]: report_data["test"]["loglines"].append(xml.sax.saxutils.escape(line)) - self.starttime = self.endtime - duration - for phase in phases_encountered: + for phase in phases: report_data[phase]["starttime"] = self.starttime report_data[phase]["log"] = "\n".join(report_data[phase]["loglines"]) errors, warnings = parse_log_events(report_data[phase]["loglines"]) @@ -326,6 +335,19 @@ class CDash(Reporter): if phase == "update": report_data[phase]["revision"] = self.revision + def extract_standalone_test_data(self, package, phases, report_data): + """Extract stand-alone test outputs for the package.""" + + testing = {} + report_data["testing"] = testing + testing["starttime"] = self.starttime + testing["endtime"] = self.starttime + testing["generator"] = self.generator + testing["parts"] = extract_test_parts(package["name"], package["stdout"].splitlines()) + + def report_test_data(self, directory_name, package, phases, report_data): + """Generate and upload the test report(s) for the package.""" + for phase in phases: # Write the report. report_name = phase.capitalize() + ".xml" report_file_name = package["name"] + "_" + report_name @@ -333,7 +355,7 @@ class CDash(Reporter): with codecs.open(phase_report, "w", "utf-8") as f: env = spack.tengine.make_environment() - if phase != "update": + if phase not in ["update", "testing"]: # Update.xml stores site information differently # than the rest of the CTest XML files. site_template = posixpath.join(self.template_dir, "Site.xml") @@ -343,18 +365,65 @@ class CDash(Reporter): phase_template = posixpath.join(self.template_dir, report_name) t = env.get_template(phase_template) f.write(t.render(report_data)) + + tty.debug("Preparing to upload {0}".format(phase_report)) self.upload(phase_report) + def test_report_for_package(self, directory_name, package, duration, ctest_parsing=False): + if "stdout" not in package: + # Skip reporting on packages that did not generate any output. + tty.debug("Skipping report for {0}: No generated output".format(package["name"])) + return + + self.current_package_name = package["name"] + if self.base_buildname == self.install_command: + # The package list is NOT all that helpful in this case + self.buildname = "{0}-{1}".format(self.current_package_name, package["id"]) + else: + self.buildname = self.report_build_name(self.current_package_name) + self.starttime = self.endtime - duration + + report_data = self.initialize_report(directory_name) + report_data["hostname"] = socket.gethostname() + if ctest_parsing: + phases = ["test", "update"] + self.extract_ctest_test_data(package, phases, report_data) + else: + phases = ["testing"] + self.extract_standalone_test_data(package, phases, report_data) + + self.report_test_data(directory_name, package, phases, report_data) + def test_report(self, directory_name, input_data): - # Generate reports for each package in each spec. + """Generate reports for each package in each spec.""" + tty.debug("Processing test report") for spec in input_data["specs"]: duration = 0 if "time" in spec: duration = int(spec["time"]) for package in spec["packages"]: - self.test_report_for_package(directory_name, package, duration) + self.test_report_for_package( + directory_name, + package, + duration, + input_data["ctest-parsing"], + ) + self.finalize_report() + def test_skipped_report(self, directory_name, spec, reason=None): + output = "Skipped {0} package".format(spec.name) + if reason: + output += "\n{0}".format(reason) + + package = { + "name": spec.name, + "id": spec.dag_hash(), + "result": "skipped", + "stdout": output, + } + self.test_report_for_package(directory_name, package, duration=0.0, ctest_parsing=False) + def concretization_report(self, directory_name, msg): self.buildname = self.base_buildname report_data = self.initialize_report(directory_name) @@ -384,12 +453,16 @@ class CDash(Reporter): report_data["buildname"] = self.buildname report_data["buildstamp"] = self.buildstamp report_data["install_command"] = self.install_command + report_data["generator"] = self.generator report_data["osname"] = self.osname + report_data["osrelease"] = self.osrelease report_data["site"] = self.site + report_data["target"] = self.target return report_data def upload(self, filename): if not self.cdash_upload_url: + print("Cannot upload {0} due to missing upload url".format(filename)) return # Compute md5 checksum for the contents of this file. @@ -412,7 +485,7 @@ class CDash(Reporter): request.add_header("Authorization", "Bearer {0}".format(self.authtoken)) try: # By default, urllib2 only support GET and POST. - # CDash needs expects this file to be uploaded via PUT. + # CDash expects this file to be uploaded via PUT. request.get_method = lambda: "PUT" response = opener.open(request) if self.current_package_name not in self.buildIds: @@ -428,13 +501,13 @@ class CDash(Reporter): def finalize_report(self): if self.buildIds: - print("View your build results here:") + tty.msg("View your build results here:") for package_name, buildid in iteritems(self.buildIds): # Construct and display a helpful link if CDash responded with # a buildId. build_url = self.cdash_upload_url build_url = build_url[0 : build_url.find("submit.php")] build_url += "buildSummary.php?buildid={0}".format(buildid) - print("{0}: {1}".format(package_name, build_url)) + tty.msg("{0}: {1}".format(package_name, build_url)) if not self.success: raise SpackError("Errors encountered, see above for more details") diff --git a/lib/spack/spack/reporters/extract.py b/lib/spack/spack/reporters/extract.py new file mode 100644 index 0000000000..5814d791a5 --- /dev/null +++ b/lib/spack/spack/reporters/extract.py @@ -0,0 +1,212 @@ +# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os +import re +import xml.sax.saxutils +from datetime import datetime + +import llnl.util.tty as tty + +# The keys here represent the only recognized (ctest/cdash) status values +completed = { + "failed": "Completed", + "passed": "Completed", + "notrun": "No tests to run", +} + +log_regexp = re.compile(r"^==> \[([0-9:.\-]*)(?:, [0-9]*)?\] (.*)") +returns_regexp = re.compile(r"\[([0-9 ,]*)\]") + +skip_msgs = ["Testing package", "Results for", "Detected the following"] +skip_regexps = [re.compile(r"{0}".format(msg)) for msg in skip_msgs] + +status_values = ["FAILED", "PASSED", "NO-TESTS"] +status_regexps = [re.compile(r"^({0})".format(stat)) for stat in status_values] + + +def add_part_output(part, line): + if part: + part["loglines"].append(xml.sax.saxutils.escape(line)) + + +def elapsed(current, previous): + if not (current and previous): + return 0 + + diff = current - previous + tty.debug("elapsed = %s - %s = %s" % (current, previous, diff)) + return diff.total_seconds() + + +def expected_failure(line): + if not line: + return False + + match = returns_regexp.search(line) + xfail = "0" not in match.group(0) if match else False + return xfail + + +def new_part(): + return { + "command": None, + "completed": "Unknown", + "desc": None, + "elapsed": None, + "name": None, + "loglines": [], + "output": None, + "status": "passed", + } + + +def part_name(source): + # TODO: Should be passed the package prefix and only remove it + elements = [] + for e in source.replace("'", "").split(" "): + elements.append(os.path.basename(e) if os.sep in e else e) + return "_".join(elements) + + +def process_part_end(part, curr_time, last_time): + if part: + if not part["elapsed"]: + part["elapsed"] = elapsed(curr_time, last_time) + + stat = part["status"] + if stat in completed: + if stat == "passed" and expected_failure(part["desc"]): + part["completed"] = "Expected to fail" + elif part["completed"] == "Unknown": + part["completed"] = completed[stat] + part["output"] = "\n".join(part["loglines"]) + + +def timestamp(time_string): + return datetime.strptime(time_string, "%Y-%m-%d-%H:%M:%S.%f") + + +def skip(line): + for regex in skip_regexps: + match = regex.search(line) + if match: + return match + + +def status(line): + for regex in status_regexps: + match = regex.search(line) + if match: + stat = match.group(0) + stat = "notrun" if stat == "NO-TESTS" else stat + return stat.lower() + + +def extract_test_parts(default_name, outputs): + parts = [] + part = {} + testdesc = "" + last_time = None + curr_time = None + for line in outputs: + line = line.strip() + if not line: + add_part_output(part, line) + continue + + if skip(line): + continue + + # Skipped tests start with "Skipped" and end with "package" + if line.startswith("Skipped") and line.endswith("package"): + part = new_part() + part["command"] = "Not Applicable" + part["completed"] = line + part["elapsed"] = 0.0 + part["name"] = default_name + part["status"] = "notrun" + parts.append(part) + continue + + # Process Spack log messages + if line.find("==>") != -1: + match = log_regexp.search(line) + if match: + curr_time = timestamp(match.group(1)) + msg = match.group(2) + + # Skip logged message for caching build-time data + if msg.startswith("Installing"): + continue + + # New command means the start of a new test part + if msg.startswith("'") and msg.endswith("'"): + # Update the last part processed + process_part_end(part, curr_time, last_time) + + part = new_part() + part["command"] = msg + part["name"] = part_name(msg) + parts.append(part) + + # Save off the optional test description if it was + # tty.debuged *prior to* the command and reset + if testdesc: + part["desc"] = testdesc + testdesc = "" + + else: + # Update the last part processed since a new log message + # means a non-test action + process_part_end(part, curr_time, last_time) + + if testdesc: + # We had a test description but no command so treat + # as a new part (e.g., some import tests) + part = new_part() + part["name"] = "_".join(testdesc.split()) + part["command"] = "unknown" + part["desc"] = testdesc + parts.append(part) + process_part_end(part, curr_time, curr_time) + + # Assuming this is a description for the next test part + testdesc = msg + + else: + tty.debug("Did not recognize test output '{0}'".format(line)) + + # Each log message potentially represents a new test part so + # save off the last timestamp + last_time = curr_time + continue + + # Check for status values + stat = status(line) + if stat: + if part: + part["status"] = stat + add_part_output(part, line) + else: + tty.warn("No part to add status from '{0}'".format(line)) + continue + + add_part_output(part, line) + + # Process the last lingering part IF it didn't generate status + process_part_end(part, curr_time, last_time) + + # If no parts, create a skeleton to flag that the tests are not run + if not parts: + part = new_part() + stat = "notrun" + part["command"] = "Not Applicable" + part["completed"] = completed[stat] + part["elapsed"] = 0.0 + part["name"] = default_name + part["status"] = stat + parts.append(part) + + return parts diff --git a/lib/spack/spack/reporters/junit.py b/lib/spack/spack/reporters/junit.py index f845974d9d..ee080fc6ab 100644 --- a/lib/spack/spack/reporters/junit.py +++ b/lib/spack/spack/reporters/junit.py @@ -3,11 +3,10 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os.path import posixpath -import spack.build_environment -import spack.fetch_strategy -import spack.package_base +import spack.tengine from spack.reporter import Reporter __all__ = ["JUnit"] @@ -23,6 +22,11 @@ class JUnit(Reporter): self.template_file = posixpath.join("reports", "junit.xml") def build_report(self, filename, report_data): + if not (os.path.splitext(filename))[1]: + # Ensure the report name will end with the proper extension; + # otherwise, it currently defaults to the "directory" name. + filename = filename + ".xml" + # Write the report with open(filename, "w") as f: env = spack.tengine.make_environment() diff --git a/lib/spack/spack/schema/gitlab_ci.py b/lib/spack/spack/schema/gitlab_ci.py index eb8abc9682..d9da5c6ce7 100644 --- a/lib/spack/spack/schema/gitlab_ci.py +++ b/lib/spack/spack/schema/gitlab_ci.py @@ -101,6 +101,12 @@ core_shared_properties = union_dicts( "signing-job-attributes": runner_selector_schema, "rebuild-index": {"type": "boolean"}, "broken-specs-url": {"type": "string"}, + "broken-tests-packages": { + "type": "array", + "items": { + "type": "string", + }, + }, }, ) diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index 0964a1ba1a..f42518f7fb 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -176,6 +176,33 @@ def test_download_and_extract_artifacts(tmpdir, monkeypatch, working_env): ci.download_and_extract_artifacts(url, working_dir) +def test_ci_copy_stage_logs_to_artifacts_fail(tmpdir, config, mock_packages, monkeypatch, capfd): + """The copy will fail because the spec is not concrete so does not have + a package.""" + log_dir = tmpdir.join("log_dir") + s = spec.Spec("printing-package").concretized() + + ci.copy_stage_logs_to_artifacts(s, log_dir) + _, err = capfd.readouterr() + assert "Unable to copy files" in err + assert "No such file or directory" in err + + +def test_ci_copy_test_logs_to_artifacts_fail(tmpdir, capfd): + log_dir = tmpdir.join("log_dir") + + ci.copy_test_logs_to_artifacts("no-such-dir", log_dir) + _, err = capfd.readouterr() + assert "Cannot copy test logs" in err + + stage_dir = tmpdir.join("stage_dir").strpath + os.makedirs(stage_dir) + ci.copy_test_logs_to_artifacts(stage_dir, log_dir) + _, err = capfd.readouterr() + assert "Unable to copy files" in err + assert "No such file or directory" in err + + def test_setup_spack_repro_version(tmpdir, capfd, last_two_git_commits, monkeypatch): c1, c2 = last_two_git_commits repro_dir = os.path.join(tmpdir.strpath, "repro") @@ -467,3 +494,154 @@ def test_affected_specs_on_first_concretization(mutable_mock_env_path, config): affected_specs = spack.ci.get_spec_filter_list(e, ["zlib"]) hdf5_specs = [s for s in affected_specs if s.name == "hdf5"] assert len(hdf5_specs) == 2 + + +@pytest.mark.skipif( + sys.platform == "win32", reason="Reliance on bash script ot supported on Windows" +) +def test_ci_process_command(tmpdir): + repro_dir = tmpdir.join("repro_dir").strpath + os.makedirs(repro_dir) + result = ci.process_command("help", [], repro_dir) + + assert os.path.exists(fs.join_path(repro_dir, "help.sh")) + assert not result + + +@pytest.mark.skipif( + sys.platform == "win32", reason="Reliance on bash script ot supported on Windows" +) +def test_ci_process_command_fail(tmpdir, monkeypatch): + import subprocess + + err = "subprocess wait exception" + + def _fail(self, args): + raise RuntimeError(err) + + monkeypatch.setattr(subprocess.Popen, "__init__", _fail) + + repro_dir = tmpdir.join("repro_dir").strpath + os.makedirs(repro_dir) + + with pytest.raises(RuntimeError, match=err): + ci.process_command("help", [], repro_dir) + + +def test_ci_create_buildcache(tmpdir, working_env, config, mock_packages, monkeypatch): + # Monkeypatching ci method tested elsewhere to reduce number of methods + # that would need to be patched here. + monkeypatch.setattr(spack.ci, "push_mirror_contents", lambda a, b, c, d: None) + + args = { + "env": None, + "buildcache_mirror_url": "file://fake-url", + "pipeline_mirror_url": "file://fake-url", + } + ci.create_buildcache(**args) + + +def test_ci_run_standalone_tests_missing_requirements( + tmpdir, working_env, config, mock_packages, capfd +): + """This test case checks for failing prerequisite checks.""" + ci.run_standalone_tests() + err = capfd.readouterr()[1] + assert "Job spec is required" in err + + args = {"job_spec": spec.Spec("printing-package").concretized()} + ci.run_standalone_tests(**args) + err = capfd.readouterr()[1] + assert "Reproduction directory is required" in err + + +@pytest.mark.skipif( + sys.platform == "win32", reason="Reliance on bash script ot supported on Windows" +) +def test_ci_run_standalone_tests_not_installed_junit( + tmpdir, working_env, config, mock_packages, mock_test_stage, capfd +): + log_file = tmpdir.join("junit.xml").strpath + args = { + "log_file": log_file, + "job_spec": spec.Spec("printing-package").concretized(), + "repro_dir": tmpdir.join("repro_dir").strpath, + "fail_fast": True, + } + os.makedirs(args["repro_dir"]) + + ci.run_standalone_tests(**args) + err = capfd.readouterr()[1] + assert "No installed packages" in err + assert os.path.getsize(log_file) > 0 + + +@pytest.mark.skipif( + sys.platform == "win32", reason="Reliance on bash script ot supported on Windows" +) +def test_ci_run_standalone_tests_not_installed_cdash( + tmpdir, working_env, config, mock_packages, mock_test_stage, capfd +): + """Test run_standalone_tests with cdash and related options.""" + log_file = tmpdir.join("junit.xml").strpath + args = { + "log_file": log_file, + "job_spec": spec.Spec("printing-package").concretized(), + "repro_dir": tmpdir.join("repro_dir").strpath, + } + os.makedirs(args["repro_dir"]) + + # Cover when CDash handler provided (with the log file as well) + ci_cdash = { + "url": "file://fake", + "build-group": "fake-group", + "project": "ci-unit-testing", + "site": "fake-site", + } + os.environ["SPACK_CDASH_BUILD_NAME"] = "ci-test-build" + os.environ["SPACK_CDASH_BUILD_STAMP"] = "ci-test-build-stamp" + os.environ["CI_RUNNER_DESCRIPTION"] = "test-runner" + handler = ci.CDashHandler(ci_cdash) + args["cdash"] = handler + ci.run_standalone_tests(**args) + out = capfd.readouterr()[0] + # CDash *and* log file output means log file ignored + assert "xml option is ignored" in out + assert "0 passed of 0" in out + + # copy test results (though none) + artifacts_dir = tmpdir.join("artifacts") + fs.mkdirp(artifacts_dir.strpath) + handler.copy_test_results(tmpdir.strpath, artifacts_dir.strpath) + err = capfd.readouterr()[1] + assert "Unable to copy files" in err + assert "No such file or directory" in err + + +def test_ci_skipped_report(tmpdir, mock_packages, config): + """Test explicit skipping of report as well as CI's 'package' arg.""" + pkg = "trivial-smoke-test" + spec = spack.spec.Spec(pkg).concretized() + ci_cdash = { + "url": "file://fake", + "build-group": "fake-group", + "project": "ci-unit-testing", + "site": "fake-site", + } + os.environ["SPACK_CDASH_BUILD_NAME"] = "fake-test-build" + os.environ["SPACK_CDASH_BUILD_STAMP"] = "ci-test-build-stamp" + os.environ["CI_RUNNER_DESCRIPTION"] = "test-runner" + handler = ci.CDashHandler(ci_cdash) + reason = "Testing skip" + handler.report_skipped(spec, tmpdir.strpath, reason=reason) + + report = fs.join_path(tmpdir, "{0}_Testing.xml".format(pkg)) + expected = "Skipped {0} package".format(pkg) + with open(report, "r") as f: + have = [0, 0] + for line in f: + if expected in line: + have[0] += 1 + elif reason in line: + have[1] += 1 + assert all(count == 1 for count in have) diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 2435f8d92a..0ac352c1df 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -33,6 +33,7 @@ from spack.schema.gitlab_ci import schema as gitlab_ci_schema from spack.spec import CompilerSpec, Spec from spack.util.executable import which from spack.util.mock_package import MockPackageMultiRepo +from spack.util.pattern import Bunch ci_cmd = spack.main.SpackCommand("ci") env_cmd = spack.main.SpackCommand("env") @@ -257,7 +258,12 @@ def _validate_needs_graph(yaml_contents, needs_graph, artifacts): def test_ci_generate_bootstrap_gcc( - tmpdir, mutable_mock_env_path, install_mockery, mock_packages, ci_base_environment + tmpdir, + working_env, + mutable_mock_env_path, + install_mockery, + mock_packages, + ci_base_environment, ): """Test that we can bootstrap a compiler and use it as the compiler for a spec in the environment""" @@ -320,7 +326,12 @@ spack: def test_ci_generate_bootstrap_artifacts_buildcache( - tmpdir, mutable_mock_env_path, install_mockery, mock_packages, ci_base_environment + tmpdir, + working_env, + mutable_mock_env_path, + install_mockery, + mock_packages, + ci_base_environment, ): """Test that we can bootstrap a compiler when artifacts buildcache is turned on""" @@ -387,6 +398,7 @@ spack: def test_ci_generate_with_env_missing_section( tmpdir, + working_env, mutable_mock_env_path, install_mockery, mock_packages, @@ -479,6 +491,7 @@ spack: def test_ci_generate_with_custom_scripts( tmpdir, + working_env, mutable_mock_env_path, install_mockery, mock_packages, @@ -575,7 +588,12 @@ spack: def test_ci_generate_pkg_with_deps( - tmpdir, mutable_mock_env_path, install_mockery, mock_packages, ci_base_environment + tmpdir, + working_env, + mutable_mock_env_path, + install_mockery, + mock_packages, + ci_base_environment, ): """Test pipeline generation for a package w/ dependencies""" filename = str(tmpdir.join("spack.yaml")) @@ -630,7 +648,13 @@ spack: def test_ci_generate_for_pr_pipeline( - tmpdir, mutable_mock_env_path, install_mockery, mock_packages, monkeypatch, ci_base_environment + tmpdir, + working_env, + mutable_mock_env_path, + install_mockery, + mock_packages, + monkeypatch, + ci_base_environment, ): """Test that PR pipelines do not include a final stage job for rebuilding the mirror index, even if that job is specifically @@ -690,7 +714,13 @@ spack: def test_ci_generate_with_external_pkg( - tmpdir, mutable_mock_env_path, install_mockery, mock_packages, monkeypatch, ci_base_environment + tmpdir, + working_env, + mutable_mock_env_path, + install_mockery, + mock_packages, + monkeypatch, + ci_base_environment, ): """Make sure we do not generate jobs for external pkgs""" filename = str(tmpdir.join("spack.yaml")) @@ -729,22 +759,40 @@ spack: assert not any("externaltool" in key for key in yaml_contents) -@pytest.mark.xfail(reason="fails intermittently and covered by gitlab ci") -def test_ci_rebuild( - tmpdir, - mutable_mock_env_path, - install_mockery, - mock_packages, - monkeypatch, - mock_gnupghome, - mock_fetch, - ci_base_environment, - mock_binary_index, -): +def test_ci_rebuild_missing_config(tmpdir, working_env, mutable_mock_env_path): + spack_yaml_contents = """ +spack: + specs: + - archive-files +""" + + filename = str(tmpdir.join("spack.yaml")) + with open(filename, "w") as f: + f.write(spack_yaml_contents) + + with tmpdir.as_cwd(): + env_cmd("create", "test", "./spack.yaml") + env_cmd("activate", "--without-view", "--sh", "test") + out = ci_cmd("rebuild", fail_on_error=False) + assert "env containing gitlab-ci" in out + + env_cmd("deactivate") + + +def _signing_key(): + signing_key_dir = spack_paths.mock_gpg_keys_path + signing_key_path = os.path.join(signing_key_dir, "package-signing-key") + with open(signing_key_path) as fd: + key = fd.read() + return key + + +def create_rebuild_env(tmpdir, pkg_name, broken_tests=False): working_dir = tmpdir.join("working_dir") log_dir = os.path.join(working_dir.strpath, "logs") repro_dir = os.path.join(working_dir.strpath, "repro") + test_dir = os.path.join(working_dir.strpath, "test") env_dir = working_dir.join("concrete_env") mirror_dir = working_dir.join("mirror") @@ -754,39 +802,37 @@ def test_ci_rebuild( broken_specs_url = url_util.join("file://", broken_specs_path) temp_storage_url = "file:///path/to/per/pipeline/storage" + broken_tests_packages = [pkg_name] if broken_tests else [] + ci_job_url = "https://some.domain/group/project/-/jobs/42" ci_pipeline_url = "https://some.domain/group/project/-/pipelines/7" - signing_key_dir = spack_paths.mock_gpg_keys_path - signing_key_path = os.path.join(signing_key_dir, "package-signing-key") - with open(signing_key_path) as fd: - signing_key = fd.read() - spack_yaml_contents = """ spack: - definitions: - - packages: [archive-files] - specs: - - $packages - mirrors: - test-mirror: {0} - gitlab-ci: - broken-specs-url: {1} - temporary-storage-url-prefix: {2} - mappings: + definitions: + - packages: [{0}] + specs: + - $packages + mirrors: + test-mirror: {1} + gitlab-ci: + broken-specs-url: {2} + broken-tests-packages: {3} + temporary-storage-url-prefix: {4} + mappings: - match: - - archive-files + - {0} runner-attributes: tags: - donotcare image: donotcare - cdash: - build-group: Not important - url: https://my.fake.cdash - project: Not used - site: Nothing + cdash: + build-group: Not important + url: https://my.fake.cdash + project: Not used + site: Nothing """.format( - mirror_url, broken_specs_url, temp_storage_url + pkg_name, mirror_url, broken_specs_url, broken_tests_packages, temp_storage_url ) filename = str(tmpdir.join("spack.yaml")) @@ -809,43 +855,126 @@ spack: root_spec_dag_hash = None for h, s in env.specs_by_hash.items(): - if s.name == "archive-files": + if s.name == pkg_name: root_spec_dag_hash = h assert root_spec_dag_hash - def fake_cdash_register(build_name, base_url, project, site, track): - return ("fakebuildid", "fakestamp") + return Bunch( + broken_spec_file=os.path.join(broken_specs_path, root_spec_dag_hash), + ci_job_url=ci_job_url, + ci_pipeline_url=ci_pipeline_url, + env_dir=env_dir, + log_dir=log_dir, + mirror_dir=mirror_dir, + mirror_url=mirror_url, + repro_dir=repro_dir, + root_spec_dag_hash=root_spec_dag_hash, + test_dir=test_dir, + working_dir=working_dir, + ) + + +def activate_rebuild_env(tmpdir, pkg_name, rebuild_env): + env_cmd("activate", "--without-view", "--sh", "-d", ".") + + # Create environment variables as gitlab would do it + os.environ.update( + { + "SPACK_ARTIFACTS_ROOT": rebuild_env.working_dir.strpath, + "SPACK_JOB_LOG_DIR": rebuild_env.log_dir, + "SPACK_JOB_REPRO_DIR": rebuild_env.repro_dir, + "SPACK_JOB_TEST_DIR": rebuild_env.test_dir, + "SPACK_LOCAL_MIRROR_DIR": rebuild_env.mirror_dir.strpath, + "SPACK_CONCRETE_ENV_DIR": rebuild_env.env_dir.strpath, + "CI_PIPELINE_ID": "7192", + "SPACK_SIGNING_KEY": _signing_key(), + "SPACK_ROOT_SPEC": rebuild_env.root_spec_dag_hash, + "SPACK_JOB_SPEC_DAG_HASH": rebuild_env.root_spec_dag_hash, + "SPACK_JOB_SPEC_PKG_NAME": pkg_name, + "SPACK_COMPILER_ACTION": "NONE", + "SPACK_CDASH_BUILD_NAME": "(specs) {0}".format(pkg_name), + "SPACK_REMOTE_MIRROR_URL": rebuild_env.mirror_url, + "SPACK_PIPELINE_TYPE": "spack_protected_branch", + "CI_JOB_URL": rebuild_env.ci_job_url, + "CI_PIPELINE_URL": rebuild_env.ci_pipeline_url, + "CI_PROJECT_DIR": tmpdir.join("ci-project").strpath, + } + ) + + +@pytest.mark.parametrize("broken_tests", [True, False]) +def test_ci_rebuild_mock_success( + tmpdir, + config, + working_env, + mutable_mock_env_path, + install_mockery, + mock_gnupghome, + mock_stage, + mock_fetch, + mock_binary_index, + monkeypatch, + broken_tests, +): + + pkg_name = "archive-files" + rebuild_env = create_rebuild_env(tmpdir, pkg_name, broken_tests) + + monkeypatch.setattr( + spack.cmd.ci, + "CI_REBUILD_INSTALL_BASE_ARGS", + ["echo"], + ) + + with rebuild_env.env_dir.as_cwd(): + activate_rebuild_env(tmpdir, pkg_name, rebuild_env) + + out = ci_cmd("rebuild", "--tests", fail_on_error=False) + + # We didn"t really run the build so build output file(s) are missing + assert "Unable to copy files" in out + assert "No such file or directory" in out + + if broken_tests: + # We generate a skipped tests report in this case + assert "Unable to run stand-alone tests" in out + else: + # No installation means no package to test and no test log to copy + assert "Cannot copy test logs" in out + + +@pytest.mark.xfail(reason="fails intermittently and covered by gitlab ci") +def test_ci_rebuild( + tmpdir, + working_env, + mutable_mock_env_path, + install_mockery, + mock_packages, + monkeypatch, + mock_gnupghome, + mock_fetch, + ci_base_environment, + mock_binary_index, +): + pkg_name = "archive-files" + rebuild_env = create_rebuild_env(tmpdir, pkg_name) + + # Create job directories to be removed before processing (for coverage) + os.makedirs(rebuild_env.log_dir) + os.makedirs(rebuild_env.repro_dir) + os.makedirs(rebuild_env.test_dir) + + with rebuild_env.env_dir.as_cwd(): + activate_rebuild_env(tmpdir, pkg_name, rebuild_env) + + ci_cmd("rebuild", "--tests", fail_on_error=False) monkeypatch.setattr(spack.cmd.ci, "CI_REBUILD_INSTALL_BASE_ARGS", ["notcommand"]) monkeypatch.setattr(spack.cmd.ci, "INSTALL_FAIL_CODE", 127) - with env_dir.as_cwd(): - env_cmd("activate", "--without-view", "--sh", "-d", ".") - - # Create environment variables as gitlab would do it - os.environ.update( - { - "SPACK_ARTIFACTS_ROOT": working_dir.strpath, - "SPACK_JOB_LOG_DIR": log_dir, - "SPACK_JOB_REPRO_DIR": repro_dir, - "SPACK_LOCAL_MIRROR_DIR": mirror_dir.strpath, - "SPACK_CONCRETE_ENV_DIR": env_dir.strpath, - "CI_PIPELINE_ID": "7192", - "SPACK_SIGNING_KEY": signing_key, - "SPACK_ROOT_SPEC": root_spec_dag_hash, - "SPACK_JOB_SPEC_DAG_HASH": root_spec_dag_hash, - "SPACK_JOB_SPEC_PKG_NAME": "archive-files", - "SPACK_COMPILER_ACTION": "NONE", - "SPACK_CDASH_BUILD_NAME": "(specs) archive-files", - "SPACK_REMOTE_MIRROR_URL": mirror_url, - "SPACK_PIPELINE_TYPE": "spack_protected_branch", - "CI_JOB_URL": ci_job_url, - "CI_PIPELINE_URL": ci_pipeline_url, - } - ) - - ci_cmd("rebuild", fail_on_error=False) + with rebuild_env.env_dir.as_cwd(): + activate_rebuild_env(tmpdir, pkg_name, rebuild_env) expected_repro_files = [ "install.sh", @@ -854,10 +983,10 @@ spack: "spack.yaml", "spack.lock", ] - repro_files = os.listdir(repro_dir) + repro_files = os.listdir(rebuild_env.repro_dir) assert all([f in repro_files for f in expected_repro_files]) - install_script_path = os.path.join(repro_dir, "install.sh") + install_script_path = os.path.join(rebuild_env.repro_dir, "install.sh") install_line = None with open(install_script_path) as fd: for line in fd: @@ -878,17 +1007,27 @@ spack: flag_index = install_parts.index("-f") assert "archive-files.json" in install_parts[flag_index + 1] - broken_spec_file = os.path.join(broken_specs_path, root_spec_dag_hash) - with open(broken_spec_file) as fd: + with open(rebuild_env.broken_spec_file) as fd: broken_spec_content = fd.read() - assert ci_job_url in broken_spec_content - assert (ci_pipeline_url) in broken_spec_content + assert rebuild_env.ci_job_url in broken_spec_content + assert rebuild_env.ci_pipeline_url in broken_spec_content + + # Ensure also produce CDash output for skipped (or notrun) tests + test_files = os.listdir(rebuild_env.test_dir) + with open(os.path.join(rebuild_env.test_dir, test_files[0]), "r") as f: + have = False + for line in f: + if "notrun" in line: + have = True + break + assert have env_cmd("deactivate") def test_ci_nothing_to_rebuild( tmpdir, + working_env, mutable_mock_env_path, install_mockery, mock_packages, @@ -946,6 +1085,7 @@ spack: "SPACK_ARTIFACTS_ROOT": working_dir.strpath, "SPACK_JOB_LOG_DIR": "log_dir", "SPACK_JOB_REPRO_DIR": "repro_dir", + "SPACK_JOB_TEST_DIR": "test_dir", "SPACK_LOCAL_MIRROR_DIR": mirror_dir.strpath, "SPACK_CONCRETE_ENV_DIR": tmpdir.strpath, "SPACK_ROOT_SPEC": root_spec_dag_hash, @@ -1073,12 +1213,7 @@ def test_push_mirror_contents( mirror_dir = working_dir.join("mirror") mirror_url = "file://{0}".format(mirror_dir.strpath) - signing_key_dir = spack_paths.mock_gpg_keys_path - signing_key_path = os.path.join(signing_key_dir, "package-signing-key") - with open(signing_key_path) as fd: - signing_key = fd.read() - - ci.import_signing_key(signing_key) + ci.import_signing_key(_signing_key()) spack_yaml_contents = """ spack: @@ -1198,6 +1333,7 @@ spack: # Also just make sure that if something goes wrong with the # stage logs copy, no exception is thrown + ci.copy_stage_logs_to_artifacts(concrete_spec, None) ci.copy_stage_logs_to_artifacts(None, logs_dir.strpath) dl_dir = working_dir.join("download_dir") @@ -1413,7 +1549,13 @@ spack: @pytest.mark.disable_clean_stage_check def test_ci_rebuild_index( - tmpdir, mutable_mock_env_path, install_mockery, mock_packages, mock_fetch, mock_stage + tmpdir, + working_env, + mutable_mock_env_path, + install_mockery, + mock_packages, + mock_fetch, + mock_stage, ): working_dir = tmpdir.join("working_dir") @@ -2036,3 +2178,35 @@ spack: expect_out = "docker run --rm -v {0}:{0} -ti {1}".format(working_dir.strpath, image_name) assert expect_out in rep_out + + +@pytest.mark.parametrize( + "subcmd", + [ + (""), + ("generate"), + ("rebuild-index"), + ("rebuild"), + ("reproduce-build"), + ], +) +def test_ci_help(subcmd, capsys): + """Make sure `spack ci` --help describes the (sub)command help.""" + with pytest.raises(SystemExit): + ci_cmd(subcmd, "--help") + + out = str(capsys.readouterr()) + usage = "usage: spack ci {0}{1}[".format(subcmd, " " if subcmd else "") + assert usage in out + + +def test_cmd_first_line(): + """Explicitly test first_line since not picked up in test_ci_help.""" + first = "This is a test." + doc = """{0} + + Is there more to be said?""".format( + first + ) + + assert spack.cmd.first_line(doc) == first diff --git a/lib/spack/spack/test/cmd/config.py b/lib/spack/spack/test/cmd/config.py index 42152ca3a5..f9c3fae2f9 100644 --- a/lib/spack/spack/test/cmd/config.py +++ b/lib/spack/spack/test/cmd/config.py @@ -78,7 +78,7 @@ repos: ) -def test_config_edit(): +def test_config_edit(mutable_config, working_env): """Ensure `spack config edit` edits the right paths.""" dms = spack.config.default_modify_scope("compilers") diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py index 73e3cf15cf..91ef154001 100644 --- a/lib/spack/spack/test/cmd/install.py +++ b/lib/spack/spack/test/cmd/install.py @@ -49,11 +49,12 @@ def test_install_package_and_dependency( tmpdir, mock_packages, mock_archive, mock_fetch, config, install_mockery ): + log = "test" with tmpdir.as_cwd(): - install("--log-format=junit", "--log-file=test.xml", "libdwarf") + install("--log-format=junit", "--log-file={0}".format(log), "libdwarf") files = tmpdir.listdir() - filename = tmpdir.join("test.xml") + filename = tmpdir.join("{0}.xml".format(log)) assert filename in files content = filename.open().read() diff --git a/lib/spack/spack/test/cmd/test.py b/lib/spack/spack/test/cmd/test.py index bc5724e04e..e842dbc465 100644 --- a/lib/spack/spack/test/cmd/test.py +++ b/lib/spack/spack/test/cmd/test.py @@ -186,7 +186,7 @@ def test_cdash_output_test_error( report_dir = tmpdir.join("cdash_reports") print(tmpdir.listdir()) assert report_dir in tmpdir.listdir() - report_file = report_dir.join("test-error_Test.xml") + report_file = report_dir.join("test-error_Testing.xml") assert report_file in report_dir.listdir() content = report_file.open().read() assert "FAILED: Command exited with status 1" in content @@ -205,7 +205,7 @@ def test_cdash_upload_clean_test( spack_test("run", "--log-file=cdash_reports", "--log-format=cdash", "printing-package") report_dir = tmpdir.join("cdash_reports") assert report_dir in tmpdir.listdir() - report_file = report_dir.join("printing-package_Test.xml") + report_file = report_dir.join("printing-package_Testing.xml") assert report_file in report_dir.listdir() content = report_file.open().read() assert "" in content diff --git a/lib/spack/spack/test/reporters.py b/lib/spack/spack/test/reporters.py new file mode 100644 index 0000000000..72aff79210 --- /dev/null +++ b/lib/spack/spack/test/reporters.py @@ -0,0 +1,175 @@ +# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import pytest + +import llnl.util.filesystem as fs +import llnl.util.tty as tty + +import spack.reporters.cdash +import spack.reporters.extract +import spack.spec +from spack.util.pattern import Bunch + +# Use a path variable to appease Spack style line length checks +fake_install_prefix = fs.join_path( + "usr", + "spack", + "spack", + "opt", + "spack", + "linux-rhel7-broadwell", + "intel-19.0.4.227", + "fake-1.0", +) +fake_install_test_root = fs.join_path(fake_install_prefix, ".spack", "test") +fake_test_cache = fs.join_path( + "usr", "spack", ".spack", "test", "abcdefg", "fake-1.0-abcdefg", "cache", "fake" +) + + +def test_reporters_extract_no_parts(capfd): + # This test ticks three boxes: + # 1) has Installing, which is skipped; + # 2) does not define any test parts; + # 3) has a status value without a part so generates a warning + outputs = """ +==> Testing package fake-1.0-abcdefg +==> [2022-02-11-17:14:38.875259] Installing {0} to {1} +NO-TESTS +""".format( + fake_install_test_root, fake_test_cache + ).splitlines() + + parts = spack.reporters.extract.extract_test_parts("fake", outputs) + err = capfd.readouterr()[1] + + assert len(parts) == 1 + assert parts[0]["status"] == "notrun" + assert "No part to add status" in err + + +def test_reporters_extract_no_command(): + # This test ticks 2 boxes: + # 1) has a test description with no command or status + # 2) has a test description, command, and status + fake_bin = fs.join_path(fake_install_prefix, "bin", "fake") + outputs = """ +==> Testing package fake-1.0-abcdefg +==> [2022-02-15-18:44:21.250165] command with no status +==> [2022-02-15-18:44:21.250175] running test program +==> [2022-02-15-18:44:21.250200] '{0}' +PASSED +""".format( + fake_bin + ).splitlines() + + parts = spack.reporters.extract.extract_test_parts("fake", outputs) + assert len(parts) == 2 + assert parts[0]["command"] == "unknown" + assert parts[1]["loglines"] == ["PASSED"] + assert parts[1]["elapsed"] == 0.0 + + +def test_reporters_extract_missing_desc(): + fake_bin = fs.join_path(fake_install_prefix, "bin", "importer") + outputs = """ +==> Testing package fake-1.0-abcdefg +==> [2022-02-15-18:44:21.250165] '{0}' '-c' 'import fake.bin' +PASSED +==> [2022-02-15-18:44:21.250200] '{0}' '-c' 'import fake.util' +PASSED +""".format( + fake_bin + ).splitlines() + + parts = spack.reporters.extract.extract_test_parts("fake", outputs) + + assert len(parts) == 2 + assert parts[0]["desc"] is None + assert parts[1]["desc"] is None + + +def test_reporters_extract_xfail(): + fake_bin = fs.join_path(fake_install_prefix, "bin", "fake-app") + outputs = """ +==> Testing package fake-1.0-abcdefg +==> [2022-02-15-18:44:21.250165] Expecting return code in [3] +==> [2022-02-15-18:44:21.250200] '{0}' +PASSED +""".format( + fake_bin + ).splitlines() + + parts = spack.reporters.extract.extract_test_parts("fake", outputs) + + assert len(parts) == 1 + parts[0]["completed"] == "Expected to fail" + + +@pytest.mark.parametrize("state", [("not installed"), ("external")]) +def test_reporters_extract_skipped(state): + expected = "Skipped {0} package".format(state) + outputs = """ +==> Testing package fake-1.0-abcdefg +{0} +""".format( + expected + ).splitlines() + + parts = spack.reporters.extract.extract_test_parts("fake", outputs) + + assert len(parts) == 1 + parts[0]["completed"] == expected + + +def test_reporters_skip(): + # This test ticks 3 boxes: + # 1) covers an as yet uncovered skip messages + # 2) covers debug timestamps + # 3) unrecognized output + fake_bin = fs.join_path(fake_install_prefix, "bin", "fake") + unknown_message = "missing timestamp" + outputs = """ +==> Testing package fake-1.0-abcdefg +==> [2022-02-15-18:44:21.250165, 123456] Detected the following modules: fake1 +==> {0} +==> [2022-02-15-18:44:21.250175, 123456] running fake program +==> [2022-02-15-18:44:21.250200, 123456] '{1}' +INVALID +Results for test suite abcdefghijklmn +""".format( + unknown_message, fake_bin + ).splitlines() + + parts = spack.reporters.extract.extract_test_parts("fake", outputs) + + assert len(parts) == 1 + assert fake_bin in parts[0]["command"] + assert parts[0]["loglines"] == ["INVALID"] + assert parts[0]["elapsed"] == 0.0 + + +def test_reporters_report_for_package_no_stdout(tmpdir, monkeypatch, capfd): + class MockCDash(spack.reporters.cdash.CDash): + def upload(*args, **kwargs): + # Just return (Do NOT try to upload the report to the fake site) + return + + args = Bunch( + cdash_upload_url="https://fake-upload", + package="fake-package", + cdash_build="fake-cdash-build", + cdash_site="fake-site", + cdash_buildstamp=None, + cdash_track="fake-track", + ) + monkeypatch.setattr(tty, "_debug", 1) + + reporter = MockCDash(args) + pkg_data = {"name": "fake-package"} + reporter.test_report_for_package(tmpdir.strpath, pkg_data, 0, False) + err = capfd.readouterr()[1] + assert "Skipping report for" in err + assert "No generated output" in err diff --git a/lib/spack/spack/test/test_suite.py b/lib/spack/spack/test/test_suite.py index 0f3fe97f90..3d8ebace39 100644 --- a/lib/spack/spack/test/test_suite.py +++ b/lib/spack/spack/test/test_suite.py @@ -7,7 +7,7 @@ import sys import pytest -import llnl.util.filesystem as fs +import llnl.util.tty as tty import spack.install_test import spack.spec @@ -15,6 +15,33 @@ import spack.spec pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="Tests fail on Windows") +def _true(*args, **kwargs): + """Generic monkeypatch function that always returns True.""" + return True + + +@pytest.fixture +def ensure_debug(monkeypatch): + current_debug_level = tty.debug_level() + tty.set_debug(1) + + yield + + tty.set_debug(current_debug_level) + + +def ensure_results(filename, expected): + assert os.path.exists(filename) + with open(filename, "r") as fd: + lines = fd.readlines() + have = False + for line in lines: + if expected in line: + have = True + break + assert have + + def test_test_log_pathname(mock_packages, config): """Ensure test log path is reasonable.""" spec = spack.spec.Spec("libdwarf").concretized() @@ -61,31 +88,15 @@ def test_write_test_result(mock_packages, mock_test_stage): assert spec.name in msg -def test_do_test(mock_packages, install_mockery, mock_test_stage): - """Perform a stand-alone test with files to copy.""" +def test_test_uninstalled(mock_packages, install_mockery, mock_test_stage): + """Attempt to perform stand-alone test for uninstalled package.""" spec = spack.spec.Spec("trivial-smoke-test").concretized() - test_name = "test_do_test" - test_filename = "test_file.in" - - pkg = spec.package - pkg.create_extra_test_source() - - test_suite = spack.install_test.TestSuite([spec], test_name) - test_suite.current_test_spec = spec - test_suite.current_base_spec = spec - test_suite.ensure_stage() - - # Save off target paths for current spec since test suite processing - # assumes testing multiple specs. - cached_filename = fs.join_path(test_suite.current_test_cache_dir, pkg.test_source_filename) - data_filename = fs.join_path(test_suite.current_test_data_dir, test_filename) + test_suite = spack.install_test.TestSuite([spec]) - # Run the test, making sure to retain the test stage directory - # so we can ensure the files were copied. - test_suite(remove_directory=False) + test_suite() - assert os.path.exists(cached_filename) - assert os.path.exists(data_filename) + ensure_results(test_suite.results_file, "SKIPPED") + ensure_results(test_suite.log_file_for_spec(spec), "Skipped not installed") @pytest.mark.parametrize( @@ -95,27 +106,21 @@ def test_do_test(mock_packages, install_mockery, mock_test_stage): ({"externals": True}, "NO-TESTS", "No tests"), ], ) -def test_test_external(mock_packages, install_mockery, mock_test_stage, arguments, status, msg): - def ensure_results(filename, expected): - assert os.path.exists(filename) - with open(filename, "r") as fd: - lines = fd.readlines() - have = False - for line in lines: - if expected in line: - have = True - break - assert have - +def test_test_external( + mock_packages, install_mockery, mock_test_stage, monkeypatch, arguments, status, msg +): name = "trivial-smoke-test" spec = spack.spec.Spec(name).concretized() spec.external_path = "/path/to/external/{0}".format(name) + monkeypatch.setattr(spack.spec.Spec, "installed", _true) + test_suite = spack.install_test.TestSuite([spec]) test_suite(**arguments) ensure_results(test_suite.results_file, status) - ensure_results(test_suite.log_file_for_spec(spec), msg) + if arguments: + ensure_results(test_suite.log_file_for_spec(spec), msg) def test_test_stage_caches(mock_packages, install_mockery, mock_test_stage): @@ -152,21 +157,15 @@ def test_test_spec_run_once(mock_packages, install_mockery, mock_test_stage): test_suite() -def test_test_spec_verbose(mock_packages, install_mockery, mock_test_stage): +def test_test_spec_passes(mock_packages, install_mockery, mock_test_stage, monkeypatch): + spec = spack.spec.Spec("simple-standalone-test").concretized() + monkeypatch.setattr(spack.spec.Spec, "installed", _true) test_suite = spack.install_test.TestSuite([spec]) + test_suite() - test_suite(verbose=True) - passed, msg = False, False - with open(test_suite.log_file_for_spec(spec), "r") as fd: - for line in fd: - if "simple stand-alone test" in line: - msg = True - elif "PASSED" in line: - passed = True - - assert msg - assert passed + ensure_results(test_suite.results_file, "PASSED") + ensure_results(test_suite.log_file_for_spec(spec), "simple stand-alone") def test_get_test_suite(): diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml index 31d408db6e..7b06fd2dda 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml @@ -248,10 +248,13 @@ spack: - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + - spack -d ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) image: ecpe4s/ubuntu22.04-runner-x86_64:2022-07-01 + broken-tests-packages: + - gptune + mappings: - match: - hipblas diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 936593d98a..4cfde14e7a 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -612,7 +612,7 @@ _spack_ci_rebuild_index() { } _spack_ci_rebuild() { - SPACK_COMPREPLY="-h --help" + SPACK_COMPREPLY="-h --help -t --tests --fail-fast" } _spack_ci_reproduce_build() { diff --git a/share/spack/templates/reports/cdash/Site.xml b/share/spack/templates/reports/cdash/Site.xml index f0a150b6e5..e8a6c0609b 100644 --- a/share/spack/templates/reports/cdash/Site.xml +++ b/share/spack/templates/reports/cdash/Site.xml @@ -2,6 +2,9 @@ - diff --git a/share/spack/templates/reports/cdash/Testing.xml b/share/spack/templates/reports/cdash/Testing.xml new file mode 100644 index 0000000000..a5eb58c35a --- /dev/null +++ b/share/spack/templates/reports/cdash/Testing.xml @@ -0,0 +1,44 @@ + + + + + {{ testing.starttime }} +{% for part in testing.parts %} + + {{ part.name }} + {{ part.command }} + + + {{ part.elapsed }} + +{% if part.desc %} + + {{ part.desc }} + +{% endif %} + + {{ part.completed }} + +{% if part.output %} + + {{ part.output }} + +{% endif %} + + +{% endfor %} + {{ testing.endtime }} + + -- cgit v1.2.3-60-g2f50