diff options
20 files changed, 76 insertions, 13 deletions
diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index 33724e7ac8..c54596fea6 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -397,6 +397,14 @@ def _spec_matches(spec, match_string): return spec.satisfies(match_string) +def _remove_attributes(src_dict, dest_dict): + if "tags" in src_dict and "tags" in dest_dict: + # For 'tags', we remove any tags that are listed for removal + for tag in src_dict["tags"]: + while tag in dest_dict["tags"]: + dest_dict["tags"].remove(tag) + + def _copy_attributes(attrs_list, src_dict, dest_dict): for runner_attr in attrs_list: if runner_attr in src_dict: @@ -430,19 +438,23 @@ def _find_matching_config(spec, gitlab_ci): _copy_attributes(overridable_attrs, gitlab_ci, runner_attributes) - ci_mappings = gitlab_ci["mappings"] - for ci_mapping in ci_mappings: + matched = False + only_first = gitlab_ci.get("match_behavior", "first") == "first" + for ci_mapping in gitlab_ci["mappings"]: for match_string in ci_mapping["match"]: if _spec_matches(spec, match_string): + matched = True + if "remove-attributes" in ci_mapping: + _remove_attributes(ci_mapping["remove-attributes"], runner_attributes) if "runner-attributes" in ci_mapping: _copy_attributes( overridable_attrs, ci_mapping["runner-attributes"], runner_attributes ) - return runner_attributes - else: - return None + break + if matched and only_first: + break - return runner_attributes + return runner_attributes if matched else None def _pkg_name_from_spec_label(spec_label): diff --git a/lib/spack/spack/schema/gitlab_ci.py b/lib/spack/spack/schema/gitlab_ci.py index d9da5c6ce7..7c0604aaee 100644 --- a/lib/spack/spack/schema/gitlab_ci.py +++ b/lib/spack/spack/schema/gitlab_ci.py @@ -52,6 +52,15 @@ runner_selector_schema = { "properties": runner_attributes_schema_items, } +remove_attributes_schema = { + "type": "object", + "additionalProperties": False, + "required": ["tags"], + "properties": { + "tags": {"type": "array", "items": {"type": "string"}}, + }, +} + core_shared_properties = union_dicts( runner_attributes_schema_items, @@ -80,6 +89,7 @@ core_shared_properties = union_dicts( ], }, }, + "match_behavior": {"type": "string", "enum": ["first", "merge"], "default": "first"}, "mappings": { "type": "array", "items": { @@ -93,6 +103,7 @@ core_shared_properties = union_dicts( "type": "string", }, }, + "remove-attributes": remove_attributes_schema, "runner-attributes": runner_selector_schema, }, }, diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 37189af73e..dff8484199 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -1358,8 +1358,15 @@ def test_push_mirror_contents_exceptions(monkeypatch, capsys): assert expect_msg in std_out +@pytest.mark.parametrize("match_behavior", ["first", "merge"]) def test_ci_generate_override_runner_attrs( - tmpdir, mutable_mock_env_path, install_mockery, mock_packages, monkeypatch, ci_base_environment + tmpdir, + mutable_mock_env_path, + install_mockery, + mock_packages, + monkeypatch, + ci_base_environment, + match_behavior, ): """Test that we get the behavior we want with respect to the provision of runner attributes like tags, variables, and scripts, both when we @@ -1378,6 +1385,7 @@ spack: gitlab-ci: tags: - toplevel + - toplevel2 variables: ONE: toplevelvarone TWO: toplevelvartwo @@ -1388,6 +1396,7 @@ spack: - main step after_script: - post step one + match_behavior: {0} mappings: - match: - flatten-deps @@ -1400,10 +1409,12 @@ spack: - dependency-install - match: - a + remove-attributes: + tags: + - toplevel2 runner-attributes: tags: - specific-a - - toplevel variables: ONE: specificvarone TWO: specificvartwo @@ -1413,10 +1424,17 @@ spack: - custom main step after_script: - custom post step one + - match: + - a + runner-attributes: + tags: + - specific-a-2 service-job-attributes: image: donotcare tags: [donotcare] -""" +""".format( + match_behavior + ) ) with tmpdir.as_cwd(): @@ -1449,9 +1467,12 @@ spack: assert the_elt["variables"]["ONE"] == "specificvarone" assert the_elt["variables"]["TWO"] == "specificvartwo" assert "THREE" not in the_elt["variables"] - assert len(the_elt["tags"]) == 2 + assert len(the_elt["tags"]) == (2 if match_behavior == "first" else 3) assert "specific-a" in the_elt["tags"] + if match_behavior == "merge": + assert "specific-a-2" in the_elt["tags"] assert "toplevel" in the_elt["tags"] + assert "toplevel2" not in the_elt["tags"] assert len(the_elt["before_script"]) == 1 assert the_elt["before_script"][0] == "custom pre step one" assert len(the_elt["script"]) == 1 @@ -1466,8 +1487,9 @@ spack: assert the_elt["variables"]["ONE"] == "toplevelvarone" assert the_elt["variables"]["TWO"] == "toplevelvartwo" assert "THREE" not in the_elt["variables"] - assert len(the_elt["tags"]) == 1 - assert the_elt["tags"][0] == "toplevel" + assert len(the_elt["tags"]) == 2 + assert "toplevel" in the_elt["tags"] + assert "toplevel2" in the_elt["tags"] assert len(the_elt["before_script"]) == 2 assert the_elt["before_script"][0] == "pre step one" assert the_elt["before_script"][1] == "pre step two" @@ -1484,9 +1506,10 @@ spack: assert the_elt["variables"]["ONE"] == "toplevelvarone" assert the_elt["variables"]["TWO"] == "toplevelvartwo" assert the_elt["variables"]["THREE"] == "specificvarthree" - assert len(the_elt["tags"]) == 2 + assert len(the_elt["tags"]) == 3 assert "specific-one" in the_elt["tags"] assert "toplevel" in the_elt["tags"] + assert "toplevel2" in the_elt["tags"] assert len(the_elt["before_script"]) == 2 assert the_elt["before_script"][0] == "pre step one" assert the_elt["before_script"][1] == "pre step two" diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml index 67ac499003..89337ece10 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml @@ -252,6 +252,7 @@ spack: - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml index f81eb0383e..c0cdef4448 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml @@ -253,6 +253,7 @@ spack: - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml index 248d49fd40..0e575647fc 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml @@ -159,6 +159,7 @@ spack: - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml index 27de37b1f5..4c209022af 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml @@ -171,6 +171,7 @@ spack: - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml index 700d32add8..eb60dd85fe 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml @@ -45,6 +45,7 @@ spack: name: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18" entrypoint: [ "" ] + match_behavior: first mappings: - match: - cmake diff --git a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml index ae9025e366..3f5a37f887 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml @@ -55,6 +55,7 @@ spack: - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml index 34517c1fa9..ed49d553b2 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml @@ -51,6 +51,7 @@ spack: - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + match_behavior: first mappings: - match: ['os=monterey'] runner-attributes: diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-on-power/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-on-power/spack.yaml index 24a4879277..b1a30701df 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-on-power/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-on-power/spack.yaml @@ -221,6 +221,7 @@ spack: - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - spack -d ci rebuild + match_behavior: first mappings: - match: - cuda diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml index 4492380126..ce8441a5d8 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml @@ -282,6 +282,7 @@ spack: image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2022-07-01 + match_behavior: first mappings: - match: - hipblas diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml index 853e293e6c..6158234a8a 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml @@ -261,6 +261,7 @@ spack: broken-tests-packages: - gptune + match_behavior: first mappings: - match: - hipblas diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-cpu/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-cpu/spack.yaml index e6b49b9f41..f389a0d14f 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/ml-cpu/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-cpu/spack.yaml @@ -97,6 +97,7 @@ spack: - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-cuda/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-cuda/spack.yaml index eb37168665..a65c68db09 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/ml-cuda/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-cuda/spack.yaml @@ -100,6 +100,7 @@ spack: - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-rocm/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-rocm/spack.yaml index c437b170e4..d369924b8b 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/ml-rocm/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-rocm/spack.yaml @@ -103,6 +103,7 @@ spack: - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml index 4d18961d4b..0e22a2d566 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml @@ -67,6 +67,7 @@ spack: - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml index c6c895bd36..9c1347ac25 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml @@ -72,6 +72,7 @@ spack: - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml index ebb15eb7ca..b43c4ebdcf 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml @@ -72,6 +72,7 @@ spack: - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild + match_behavior: first mappings: - match: - lbann diff --git a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml index 504b4209d2..dc91bf3208 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml @@ -74,6 +74,7 @@ spack: - spack -d ci rebuild image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] } + match_behavior: first mappings: - match: - cmake |