From 10491e98a876df986ce9ee78902c440fe4a8337d Mon Sep 17 00:00:00 2001 From: Jonathon Anderson <17242663+blue42u@users.noreply.github.com> Date: Sat, 15 Oct 2022 12:29:53 -0500 Subject: CI: allow multiple matches to combine tags (#32290) Currently "spack ci generate" chooses the first matching entry in gitlab-ci:mappings to fill attributes for a generated build-job, requiring that the entire configuration matrix is listed out explicitly. This unfortunately causes significant problems in environments with large configuration spaces, for example the environment in #31598 (spack.yaml) supports 5 operating systems, 3 architectures and 130 packages with explicit size requirements, resulting in 1300 lines of configuration YAML. This patch adds a configuraiton option to the gitlab-ci schema called "match_behavior"; when it is set to "merge", all matching entries are applied in order to the final build-job, allowing a few entries to cover an entire matrix of configurations. The default for "match_behavior" is "first", which behaves as before this commit (only the runner attributes of the first match are used). In addition, match entries may now include a "remove-attributes" configuration, which allows matches to remove tags that have been aggregated by prior matches. This only makes sense to use with "match_behavior:merge". You can combine "runner-attributes" with "remove-attributes" to effectively override prior tags. --- share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/e4s-on-power/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/ml-cpu/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/ml-cuda/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/ml-rocm/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml | 1 + share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml | 1 + 17 files changed, 17 insertions(+) (limited to 'share') diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml index 67ac499003..89337ece10 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml @@ -252,6 +252,7 @@ spack: - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml index f81eb0383e..c0cdef4448 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml @@ -253,6 +253,7 @@ spack: - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml index 248d49fd40..0e575647fc 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml @@ -159,6 +159,7 @@ spack: - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml index 27de37b1f5..4c209022af 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml @@ -171,6 +171,7 @@ spack: - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml index 700d32add8..eb60dd85fe 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml @@ -45,6 +45,7 @@ spack: name: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18" entrypoint: [ "" ] + match_behavior: first mappings: - match: - cmake diff --git a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml index ae9025e366..3f5a37f887 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml @@ -55,6 +55,7 @@ spack: - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml index 34517c1fa9..ed49d553b2 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml @@ -51,6 +51,7 @@ spack: - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + match_behavior: first mappings: - match: ['os=monterey'] runner-attributes: diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-on-power/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-on-power/spack.yaml index 24a4879277..b1a30701df 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-on-power/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-on-power/spack.yaml @@ -221,6 +221,7 @@ spack: - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - spack -d ci rebuild + match_behavior: first mappings: - match: - cuda diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml index 4492380126..ce8441a5d8 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml @@ -282,6 +282,7 @@ spack: image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2022-07-01 + match_behavior: first mappings: - match: - hipblas diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml index 853e293e6c..6158234a8a 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml @@ -261,6 +261,7 @@ spack: broken-tests-packages: - gptune + match_behavior: first mappings: - match: - hipblas diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-cpu/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-cpu/spack.yaml index e6b49b9f41..f389a0d14f 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/ml-cpu/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-cpu/spack.yaml @@ -97,6 +97,7 @@ spack: - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-cuda/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-cuda/spack.yaml index eb37168665..a65c68db09 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/ml-cuda/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-cuda/spack.yaml @@ -100,6 +100,7 @@ spack: - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-rocm/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-rocm/spack.yaml index c437b170e4..d369924b8b 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/ml-rocm/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-rocm/spack.yaml @@ -103,6 +103,7 @@ spack: - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml index 4d18961d4b..0e22a2d566 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml @@ -67,6 +67,7 @@ spack: - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml index c6c895bd36..9c1347ac25 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml @@ -72,6 +72,7 @@ spack: - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + match_behavior: first mappings: - match: - llvm diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml index ebb15eb7ca..b43c4ebdcf 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml @@ -72,6 +72,7 @@ spack: - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild + match_behavior: first mappings: - match: - lbann diff --git a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml index 504b4209d2..dc91bf3208 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml @@ -74,6 +74,7 @@ spack: - spack -d ci rebuild image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] } + match_behavior: first mappings: - match: - cmake -- cgit v1.2.3-60-g2f50