From f591e9788d064c390e11a3fd25e9efc65825a0d7 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Fri, 30 Jul 2021 09:11:00 -0600 Subject: pipelines: Store details about specs broken on develop (#24637) When a spec fails to build on `develop`, instead of storing an empty file as the entry in the broken specs list, this change stores the full spec yaml as well as links to the failing pipeline and job. --- lib/spack/spack/cmd/ci.py | 14 ++++++++++++-- lib/spack/spack/test/cmd/ci.py | 19 ++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index cf228b3192..a605f621c3 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -22,6 +22,7 @@ import spack.config as cfg import spack.environment as ev import spack.hash_types as ht import spack.mirror +import spack.util.spack_yaml as syaml import spack.util.url as url_util import spack.util.web as web_util @@ -30,6 +31,7 @@ section = "build" level = "long" CI_REBUILD_INSTALL_BASE_ARGS = ['spack', '-d', '-v'] +INSTALL_FAIL_CODE = 1 def get_env_var(variable_name): @@ -491,7 +493,7 @@ def ci_rebuild(args): # If a spec fails to build in a spack develop pipeline, we add it to a # list of known broken full hashes. This allows spack PR pipelines to # avoid wasting compute cycles attempting to build those hashes. - if install_exit_code == 1 and spack_is_develop_pipeline: + if install_exit_code == INSTALL_FAIL_CODE and spack_is_develop_pipeline: tty.debug('Install failed on develop') if 'broken-specs-url' in gitlab_ci: broken_specs_url = gitlab_ci['broken-specs-url'] @@ -502,9 +504,17 @@ def ci_rebuild(args): tmpdir = tempfile.mkdtemp() empty_file_path = os.path.join(tmpdir, 'empty.txt') + broken_spec_details = { + 'broken-spec': { + 'job-url': get_env_var('CI_JOB_URL'), + 'pipeline-url': get_env_var('CI_PIPELINE_URL'), + 'concrete-spec-yaml': job_spec.to_dict(hash=ht.full_hash) + } + } + try: with open(empty_file_path, 'w') as efd: - efd.write('') + efd.write(syaml.dump(broken_spec_details)) web_util.push_to_url( empty_file_path, broken_spec_path, diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index ea7d063b74..4a40d1fc01 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -23,6 +23,7 @@ import spack.paths as spack_paths import spack.repo as repo import spack.util.gpg import spack.util.spack_yaml as syaml +import spack.util.url as url_util from spack.schema.buildcache_spec import schema as spec_yaml_schema from spack.schema.database_index import schema as db_idx_schema from spack.schema.gitlab_ci import schema as gitlab_ci_schema @@ -689,9 +690,13 @@ def test_ci_rebuild(tmpdir, mutable_mock_env_path, env_deactivate, mirror_dir = working_dir.join('mirror') mirror_url = 'file://{0}'.format(mirror_dir.strpath) - broken_specs_url = 's3://some-bucket/naughty-list' + broken_specs_path = os.path.join(working_dir.strpath, 'naughty-list') + broken_specs_url = url_util.join('file://', broken_specs_path) temp_storage_url = 'file:///path/to/per/pipeline/storage' + ci_job_url = 'https://some.domain/group/project/-/jobs/42' + ci_pipeline_url = 'https://some.domain/group/project/-/pipelines/7' + signing_key_dir = spack_paths.mock_gpg_keys_path signing_key_path = os.path.join(signing_key_dir, 'package-signing-key') with open(signing_key_path) as fd: @@ -743,14 +748,17 @@ spack: root_spec_build_hash = None job_spec_dag_hash = None + job_spec_full_hash = None for h, s in env.specs_by_hash.items(): if s.name == 'archive-files': root_spec_build_hash = h job_spec_dag_hash = s.dag_hash() + job_spec_full_hash = s.full_hash() assert root_spec_build_hash assert job_spec_dag_hash + assert job_spec_full_hash def fake_cdash_register(build_name, base_url, project, site, track): return ('fakebuildid', 'fakestamp') @@ -760,6 +768,7 @@ spack: monkeypatch.setattr(spack.cmd.ci, 'CI_REBUILD_INSTALL_BASE_ARGS', [ 'notcommand' ]) + monkeypatch.setattr(spack.cmd.ci, 'INSTALL_FAIL_CODE', 127) with env_dir.as_cwd(): env_cmd('activate', '--without-view', '--sh', '-d', '.') @@ -780,6 +789,8 @@ spack: set_env_var('SPACK_RELATED_BUILDS_CDASH', '') set_env_var('SPACK_REMOTE_MIRROR_URL', mirror_url) set_env_var('SPACK_PIPELINE_TYPE', 'spack_protected_branch') + set_env_var('CI_JOB_URL', ci_job_url) + set_env_var('CI_PIPELINE_URL', ci_pipeline_url) ci_cmd('rebuild', fail_on_error=False) @@ -815,6 +826,12 @@ spack: flag_index = install_parts.index('-f') assert('archive-files.yaml' in install_parts[flag_index + 1]) + broken_spec_file = os.path.join(broken_specs_path, job_spec_full_hash) + with open(broken_spec_file) as fd: + broken_spec_content = fd.read() + assert(ci_job_url in broken_spec_content) + assert(ci_pipeline_url) in broken_spec_content + env_cmd('deactivate') -- cgit v1.2.3-60-g2f50