gitlab ci: release fixes and improvements (#37601)

* gitlab ci: release fixes and improvements

  - use rules to reduce boilerplate in .gitlab-ci.yml
  - support copy-only pipeline jobs
  - make pipelines for release branches rebuild everything
  - make pipelines for protected tags copy-only

* gitlab ci: remove url changes used in testing

* gitlab ci: tag mirrors need public key

Make sure that mirrors associated with release branches and tags
contain the public key needed to verify the signed binaries.  This
also ensures that when stack-specific mirror contents are copied
to the root, the root mirror has the public key as well.

* review: be more specific about tags, curl flags

* Make the check in ci.yaml consistent with the .gitlab-ci.yml

---------

Co-authored-by: Ryan Krattiger <ryan.krattiger@kitware.com>

4 files changed, 218 insertions, 464 deletions

diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py
index f3d6398416..4589b5bca4 100644
--- a/lib/spack/spack/ci.py
+++ b/lib/spack/spack/ci.py
@@ -531,7 +531,7 @@ class SpackCI:
         """
 
         self.ci_config = ci_config
-        self.named_jobs = ["any", "build", "cleanup", "noop", "reindex", "signing"]
+        self.named_jobs = ["any", "build", "copy", "cleanup", "noop", "reindex", "signing"]
 
         self.ir = {
             "jobs": {},
@@ -1207,7 +1207,7 @@ def generate_gitlab_ci_yaml(
                             ).format(c_spec, release_spec)
                             tty.debug(debug_msg)
 
-                if prune_dag and not rebuild_spec:
+                if prune_dag and not rebuild_spec and spack_pipeline_type != "spack_copy_only":
                     tty.debug(
                         "Pruning {0}/{1}, does not need rebuild.".format(
                             release_spec.name, release_spec.dag_hash()
@@ -1298,8 +1298,9 @@ def generate_gitlab_ci_yaml(
                     max_length_needs = length_needs
                     max_needs_job = job_name
 
-                output_object[job_name] = job_object
-                job_id += 1
+                if spack_pipeline_type != "spack_copy_only":
+                    output_object[job_name] = job_object
+                    job_id += 1
 
     if print_summary:
         for phase in phases:
@@ -1329,6 +1330,17 @@ def generate_gitlab_ci_yaml(
         "when": ["runner_system_failure", "stuck_or_timeout_failure", "script_failure"],
     }
 
+    if spack_pipeline_type == "spack_copy_only":
+        stage_names.append("copy")
+        sync_job = copy.deepcopy(spack_ci_ir["jobs"]["copy"]["attributes"])
+        sync_job["stage"] = "copy"
+        if artifacts_root:
+            sync_job["needs"] = [
+                {"job": generate_job_name, "pipeline": "{0}".format(parent_pipeline_id)}
+            ]
+        output_object["copy"] = sync_job
+        job_id += 1
+
     if job_id > 0:
         if temp_storage_url_prefix:
             # There were some rebuild jobs scheduled, so we will need to
diff --git a/lib/spack/spack/schema/ci.py b/lib/spack/spack/schema/ci.py
index 2c22f071de..667960f372 100644
--- a/lib/spack/spack/schema/ci.py
+++ b/lib/spack/spack/schema/ci.py
@@ -92,6 +92,11 @@ named_attributes_schema = {
         {
             "type": "object",
             "additionalProperties": False,
+            "properties": {"copy-job": attributes_schema, "copy-job-remove": attributes_schema},
+        },
+        {
+            "type": "object",
+            "additionalProperties": False,
             "properties": {
                 "reindex-job": attributes_schema,
                 "reindex-job-remove": attributes_schema,
diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
index bdd664b8d1..31c3ba8064 100644
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -27,41 +27,47 @@ default:
 ########################################
 # Job templates
 ########################################
-
-.aws-pr-creds:
-  variables:
-    AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
-    AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
-
-.aws-protected-creds:
-  variables:
-    AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
-    AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
-
-.pr-refs:
-  only:
-  - /^pr[\d]+_.*$/
-
-.pr:
-  extends: [ ".pr-refs" ]
-  variables:
-    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
-    SPACK_PIPELINE_TYPE: "spack_pull_request"
-    SPACK_PRUNE_UNTOUCHED: "True"
-    SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
-
-.protected-refs:
-  only:
-  - /^develop$/
-  - /^releases\/v.*/
-  - /^v.*/
-
-.protected:
-  extends: [ ".protected-refs" ]
+.base-job:
   variables:
     SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
-    SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
-    SPACK_PIPELINE_TYPE: "spack_protected_branch"
+  rules:
+    - if: $CI_COMMIT_REF_NAME == "develop"
+      # Pipelines on develop only rebuild what is missing from the mirror
+      when: always
+      variables:
+        SPACK_PIPELINE_TYPE: "spack_protected_branch"
+        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+        AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
+        AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
+    - if: $CI_COMMIT_REF_NAME =~ /^releases\/v.*/
+      # Pipelines on release branches always rebuild everything
+      when: always
+      variables:
+        SPACK_PIPELINE_TYPE: "spack_protected_branch"
+        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+        SPACK_PRUNE_UNTOUCHED: "False"
+        SPACK_PRUNE_UP_TO_DATE: "False"
+        AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
+        AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
+    - if: $CI_COMMIT_TAG =~ /^develop-[\d]{4}-[\d]{2}-[\d]{2}$/ || $CI_COMMIT_TAG =~ /^v.*/
+      # Pipelines on tags (release or dev snapshots) only copy binaries from one mirror to another
+      when: always
+      variables:
+        SPACK_PIPELINE_TYPE: "spack_copy_only"
+        SPACK_SOURCE_MIRROR: "s3://spack-binaries/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}"
+        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+        AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
+        AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
+    - if: $CI_COMMIT_REF_NAME =~ /^pr[\d]+_.*$/
+      # Pipelines on PR branches rebuild only what's missing, and do extra pruning
+      when: always
+      variables:
+        SPACK_PIPELINE_TYPE: "spack_pull_request"
+        SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+        SPACK_PRUNE_UNTOUCHED: "True"
+        SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
+        AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
+        AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
 
 .generate-base:
   stage: generate
@@ -99,10 +105,11 @@ default:
       - always
 
 .generate:
-  extends: [ ".generate-base" ]
+  extends: [ ".base-job", ".generate-base" ]
   tags: ["spack", "public", "medium", "x86_64"]
 
 .generate-deprecated:
+  extends: [ ".base-job" ]
   stage: generate
   script:
     - uname -a || true
@@ -134,45 +141,30 @@ default:
   tags: ["spack", "public", "medium", "x86_64"]
 
 .generate-aarch64:
-  extends: [ ".generate" ]
+  extends: [ ".base-job", ".generate" ]
   tags: ["spack", "public", "medium", "aarch64"]
 
-.pr-generate:
-  extends: [ ".pr", ".generate" ]
-
-.pr-generate-deprecated:
-  extends: [ ".pr", ".generate-deprecated" ]
-
-.pr-generate-aarch64:
-  extends: [ ".pr", ".generate-aarch64" ]
-
-.protected-generate:
-  extends: [ ".protected", ".generate" ]
-
-.protected-generate-deprecated:
-  extends: [ ".protected", ".generate-deprecated" ]
-
-.protected-generate-aarch64:
-  extends: [ ".protected", ".generate-aarch64" ]
-
 .build:
+  extends: [ ".base-job" ]
   stage: build
 
-.pr-build:
-  extends: [ ".pr", ".build", ".aws-pr-creds" ]
-
-.protected-build:
-  extends: [ ".protected", ".build", ".aws-protected-creds" ]
-
 protected-publish:
+  # Copy binaries from stack-specific mirrors to a root mirror
   stage: publish
-  extends: [ ".protected", ".aws-protected-creds" ]
+  only:
+  - /^develop$/
+  - /^releases\/v.*/
+  - /^v.*/
+  - /^develop-[\d]{4}-[\d]{2}-[\d]{2}$/
   image: "ghcr.io/spack/python-aws-bash:0.0.1"
   tags: ["spack", "public", "medium", "aws", "x86_64"]
   retry:
     max: 2
     when: ["runner_system_failure", "stuck_or_timeout_failure"]
   variables:
+    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+    SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+    SPACK_PIPELINE_TYPE: "spack_protected_branch"
     AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
     AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
     KUBERNETES_CPU_REQUEST: 4000m
@@ -182,7 +174,9 @@ protected-publish:
     - spack --version
     - export COPY_SPECS_DIR=${CI_PROJECT_DIR}/jobs_scratch_dir/specs_to_copy
     - spack buildcache sync --manifest-glob "${COPY_SPECS_DIR}/*.json"
-    - spack buildcache update-index "${SPACK_COPY_BUILDCACHE}"
+    - curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub
+    - aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_COPY_BUILDCACHE}/build_cache/_pgp/spack-public-binary-key.pub"
+    - spack buildcache update-index --keys "${SPACK_COPY_BUILDCACHE}"
 
 ########################################
 # TEMPLATE FOR ADDING ANOTHER PIPELINE
@@ -208,33 +202,19 @@ protected-publish:
 #     SPACK_CI_STACK_NAME: my-super-cool-stack
 #     tags: [ "all", "tags", "your", "job", "needs"]
 #
-# my-super-cool-stack-pr-generate:
-#   extends: [ ".pr-generate", ".my-super-cool-stack" ]
-#
-# my-super-cool-stack-protected-generate:
-#   extends: [ ".protected-generate", ".my-super-cool-stack" ]
-#
-# my-super-cool-stack-pr-build:
-#   extends: [ ".pr-build", ".my-super-cool-stack" ]
-#   trigger:
-#     include:
-#       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-#         job: my-super-cool-stack-pr-generate
-#     strategy: depend
-#   needs:
-#     - artifacts: True
-#       job: my-super-cool-stack-pr-generate
+# my-super-cool-stack-generate:
+#   extends: [ ".generate", ".my-super-cool-stack" ]
 #
-# my-super-cool-stack-protected-build:
-#   extends: [ ".protected-build", ".my-super-cool-stack" ]
+# my-super-cool-stack-build:
+#   extends: [ ".build", ".my-super-cool-stack" ]
 #   trigger:
 #     include:
 #       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-#         job: my-super-cool-stack-protected-generate
+#         job: my-super-cool-stack-generate
 #     strategy: depend
 #   needs:
 #     - artifacts: True
-#       job: my-super-cool-stack-protected-generate
+#       job: my-super-cool-stack-generate
 
 ########################################
 #          E4S Mac Stack
@@ -362,35 +342,20 @@ protected-publish:
   variables:
     SPACK_CI_STACK_NAME: e4s
 
-e4s-pr-generate:
-  extends: [ ".e4s", ".pr-generate"]
-  image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
-
-e4s-protected-generate:
-  extends: [ ".e4s", ".protected-generate"]
+e4s-generate:
+  extends: [ ".e4s", ".generate"]
   image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
 
-e4s-pr-build:
-  extends: [ ".e4s", ".pr-build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: e4s-pr-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: e4s-pr-generate
-
-e4s-protected-build:
-  extends: [ ".e4s", ".protected-build" ]
+e4s-build:
+  extends: [ ".e4s", ".build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: e4s-protected-generate
+        job: e4s-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: e4s-protected-generate
+      job: e4s-generate
 
 ########################################
 # GPU Testing Pipeline
@@ -400,35 +365,20 @@ e4s-protected-build:
   variables:
     SPACK_CI_STACK_NAME: gpu-tests
 
-gpu-tests-pr-generate:
-  extends: [ ".gpu-tests", ".pr-generate"]
-  image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
-
-gpu-tests-protected-generate:
-  extends: [ ".gpu-tests", ".protected-generate"]
+gpu-tests-generate:
+  extends: [ ".gpu-tests", ".generate"]
   image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
 
-gpu-tests-pr-build:
-  extends: [ ".gpu-tests", ".pr-build" ]
+gpu-tests-build:
+  extends: [ ".gpu-tests", ".build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: gpu-tests-pr-generate
+        job: gpu-tests-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: gpu-tests-pr-generate
-
-gpu-tests-protected-build:
-  extends: [ ".gpu-tests", ".protected-build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: gpu-tests-protected-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: gpu-tests-protected-generate
+      job: gpu-tests-generate
 
 ########################################
 # E4S OneAPI Pipeline
@@ -438,35 +388,20 @@ gpu-tests-protected-build:
   variables:
     SPACK_CI_STACK_NAME: e4s-oneapi
 
-e4s-oneapi-pr-generate:
-  extends: [ ".e4s-oneapi", ".pr-generate"]
+e4s-oneapi-generate:
+  extends: [ ".e4s-oneapi", ".generate"]
   image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023-01-01
 
-e4s-oneapi-protected-generate:
-  extends: [ ".e4s-oneapi", ".protected-generate"]
-  image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023-01-01
-
-e4s-oneapi-pr-build:
-  extends: [ ".e4s-oneapi", ".pr-build" ]
+e4s-oneapi-build:
+  extends: [ ".e4s-oneapi", ".build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: e4s-oneapi-pr-generate
+        job: e4s-oneapi-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: e4s-oneapi-pr-generate
-
-e4s-oneapi-protected-build:
-  extends: [ ".e4s-oneapi", ".protected-build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: e4s-oneapi-protected-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: e4s-oneapi-protected-generate
+      job: e4s-oneapi-generate
 
 ########################################
 # E4S on Power
@@ -480,33 +415,19 @@ e4s-oneapi-protected-build:
   variables:
     SPACK_CI_STACK_NAME: e4s-power
 
-e4s-power-pr-generate:
-  extends: [ ".e4s-power", ".pr-generate", ".e4s-power-generate-tags-and-image"]
+e4s-power-generate:
+  extends: [ ".e4s-power", ".generate", ".e4s-power-generate-tags-and-image"]
 
-e4s-power-protected-generate:
-  extends: [ ".e4s-power", ".protected-generate", ".e4s-power-generate-tags-and-image"]
-
-e4s-power-pr-build:
-  extends: [ ".e4s-power", ".pr-build" ]
+e4s-power-build:
+  extends: [ ".e4s-power", ".build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: e4s-power-pr-generate
+        job: e4s-power-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: e4s-power-pr-generate
-
-e4s-power-protected-build:
-  extends: [ ".e4s-power", ".protected-build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: e4s-power-protected-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: e4s-power-protected-generate
+      job: e4s-power-generate
 
 #########################################
 # Build tests for different build-systems
@@ -516,33 +437,19 @@ e4s-power-protected-build:
   variables:
     SPACK_CI_STACK_NAME: build_systems
 
-build_systems-pr-generate:
-  extends: [ ".build_systems", ".pr-generate"]
-
-build_systems-protected-generate:
-  extends: [ ".build_systems", ".protected-generate"]
-
-build_systems-pr-build:
-  extends: [ ".build_systems", ".pr-build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: build_systems-pr-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: build_systems-pr-generate
+build_systems-generate:
+  extends: [ ".build_systems", ".generate"]
 
-build_systems-protected-build:
-  extends: [ ".build_systems", ".protected-build" ]
+build_systems-build:
+  extends: [ ".build_systems", ".build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: build_systems-protected-generate
+        job: build_systems-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: build_systems-protected-generate
+      job: build_systems-generate
 
 #########################################
 # RADIUSS
@@ -552,35 +459,19 @@ build_systems-protected-build:
   variables:
     SPACK_CI_STACK_NAME: radiuss
 
-# ---------  PRs ---------
-radiuss-pr-generate:
-  extends: [ ".radiuss", ".pr-generate" ]
-
-radiuss-pr-build:
-  extends: [ ".radiuss", ".pr-build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: radiuss-pr-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: radiuss-pr-generate
-
-# --------- Protected ---------
-radiuss-protected-generate:
-  extends: [ ".radiuss", ".protected-generate" ]
+radiuss-generate:
+  extends: [ ".radiuss", ".generate" ]
 
-radiuss-protected-build:
-  extends: [ ".radiuss", ".protected-build" ]
+radiuss-build:
+  extends: [ ".radiuss", ".build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: radiuss-protected-generate
+        job: radiuss-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: radiuss-protected-generate
+      job: radiuss-generate
 
 ########################################
 # RADIUSS for AWS
@@ -600,33 +491,19 @@ radiuss-protected-build:
   variables:
     SPACK_CI_STACK_NAME: radiuss-aws
 
-radiuss-aws-pr-generate:
-  extends: [ ".radiuss-aws", ".pr-generate", ".radiuss-aws-overrides", ".tags-x86_64_v4" ]
-
-radiuss-aws-protected-generate:
-  extends: [ ".radiuss-aws", ".protected-generate", ".radiuss-aws-overrides", ".tags-x86_64_v4" ]
-
-radiuss-aws-pr-build:
-  extends: [ ".radiuss-aws", ".pr-build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: radiuss-aws-pr-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: radiuss-aws-pr-generate
+radiuss-aws-generate:
+  extends: [ ".radiuss-aws", ".generate", ".radiuss-aws-overrides", ".tags-x86_64_v4" ]
 
-radiuss-aws-protected-build:
-  extends: [ ".radiuss-aws", ".protected-build" ]
+radiuss-aws-build:
+  extends: [ ".radiuss-aws", ".build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: radiuss-aws-protected-generate
+        job: radiuss-aws-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: radiuss-aws-protected-generate
+      job: radiuss-aws-generate
 
 
 # Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64)
@@ -636,33 +513,19 @@ radiuss-aws-protected-build:
   variables:
     SPACK_CI_STACK_NAME: radiuss-aws-aarch64
 
-radiuss-aws-aarch64-pr-generate:
-  extends: [ ".radiuss-aws-aarch64", ".pr-generate-aarch64", ".radiuss-aws-overrides" ]
+radiuss-aws-aarch64-generate:
+  extends: [ ".radiuss-aws-aarch64", ".generate-aarch64", ".radiuss-aws-overrides" ]
 
-radiuss-aws-aarch64-protected-generate:
-  extends: [ ".radiuss-aws-aarch64", ".protected-generate-aarch64", ".radiuss-aws-overrides" ]
-
-radiuss-aws-aarch64-pr-build:
-  extends: [ ".radiuss-aws-aarch64", ".pr-build" ]
+radiuss-aws-aarch64-build:
+  extends: [ ".radiuss-aws-aarch64", ".build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: radiuss-aws-aarch64-pr-generate
+        job: radiuss-aws-aarch64-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: radiuss-aws-aarch64-pr-generate
-
-radiuss-aws-aarch64-protected-build:
-  extends: [ ".radiuss-aws-aarch64", ".protected-build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: radiuss-aws-aarch64-protected-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: radiuss-aws-aarch64-protected-generate
+      job: radiuss-aws-aarch64-generate
 
 ########################################
 # ECP Data & Vis SDK
@@ -672,35 +535,20 @@ radiuss-aws-aarch64-protected-build:
   variables:
     SPACK_CI_STACK_NAME: data-vis-sdk
 
-data-vis-sdk-pr-generate:
-  extends: [ ".data-vis-sdk", ".pr-generate"]
+data-vis-sdk-generate:
+  extends: [ ".data-vis-sdk", ".generate"]
   image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
 
-data-vis-sdk-protected-generate:
-  extends: [ ".data-vis-sdk", ".protected-generate"]
-  image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
-
-data-vis-sdk-pr-build:
-  extends: [ ".data-vis-sdk", ".pr-build" ]
+data-vis-sdk-build:
+  extends: [ ".data-vis-sdk", ".build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: data-vis-sdk-pr-generate
+        job: data-vis-sdk-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: data-vis-sdk-pr-generate
-
-data-vis-sdk-protected-build:
-  extends: [ ".data-vis-sdk", ".protected-build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: data-vis-sdk-protected-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: data-vis-sdk-protected-generate
+      job: data-vis-sdk-generate
 
 ########################################
 # AWS AHUG Applications (x86_64)
@@ -717,34 +565,19 @@ data-vis-sdk-protected-build:
   variables:
     SPACK_CI_STACK_NAME: aws-ahug
 
-aws-ahug-pr-generate:
-  extends: [ ".aws-ahug", ".pr-generate", ".aws-ahug-overrides", ".tags-x86_64_v4" ]
+aws-ahug-generate:
+  extends: [ ".aws-ahug", ".generate", ".aws-ahug-overrides", ".tags-x86_64_v4" ]
 
-aws-ahug-protected-generate:
-  extends: [ ".aws-ahug", ".protected-generate", ".aws-ahug-overrides", ".tags-x86_64_v4" ]
-
-aws-ahug-pr-build:
-  extends: [ ".aws-ahug", ".pr-build" ]
+aws-ahug-build:
+  extends: [ ".aws-ahug", ".build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: aws-ahug-pr-generate
+        job: aws-ahug-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: aws-ahug-pr-generate
-
-aws-ahug-protected-build:
-  extends: [ ".aws-ahug", ".protected-build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: aws-ahug-protected-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: aws-ahug-protected-generate
-
+      job: aws-ahug-generate
 
 # Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64)
 .aws-ahug-aarch64:
@@ -752,33 +585,19 @@ aws-ahug-protected-build:
   variables:
     SPACK_CI_STACK_NAME: aws-ahug-aarch64
 
-aws-ahug-aarch64-pr-generate:
-  extends: [ ".aws-ahug-aarch64", ".pr-generate-aarch64", ".aws-ahug-overrides" ]
-
-aws-ahug-aarch64-protected-generate:
-  extends: [ ".aws-ahug-aarch64", ".protected-generate-aarch64", ".aws-ahug-overrides" ]
-
-aws-ahug-aarch64-pr-build:
-  extends: [ ".aws-ahug-aarch64", ".pr-build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: aws-ahug-aarch64-pr-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: aws-ahug-aarch64-pr-generate
+aws-ahug-aarch64-generate:
+  extends: [ ".aws-ahug-aarch64", ".generate-aarch64", ".aws-ahug-overrides" ]
 
-aws-ahug-aarch64-protected-build:
-  extends: [ ".aws-ahug-aarch64", ".protected-build" ]
+aws-ahug-aarch64-build:
+  extends: [ ".aws-ahug-aarch64", ".build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: aws-ahug-aarch64-protected-generate
+        job: aws-ahug-aarch64-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: aws-ahug-aarch64-protected-generate
+      job: aws-ahug-aarch64-generate
 
 ########################################
 # AWS ISC Applications (x86_64)
@@ -795,34 +614,19 @@ aws-ahug-aarch64-protected-build:
   variables:
     SPACK_CI_STACK_NAME: aws-isc
 
-aws-isc-pr-generate:
-  extends: [ ".aws-isc", ".pr-generate", ".aws-isc-overrides", ".tags-x86_64_v4" ]
-
-aws-isc-protected-generate:
-  extends: [ ".aws-isc", ".protected-generate", ".aws-isc-overrides", ".tags-x86_64_v4" ]
+aws-isc-generate:
+  extends: [ ".aws-isc", ".generate", ".aws-isc-overrides", ".tags-x86_64_v4" ]
 
-aws-isc-pr-build:
-  extends: [ ".aws-isc", ".pr-build" ]
+aws-isc-build:
+  extends: [ ".aws-isc", ".build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: aws-isc-pr-generate
+        job: aws-isc-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: aws-isc-pr-generate
-
-aws-isc-protected-build:
-  extends: [ ".aws-isc", ".protected-build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: aws-isc-protected-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: aws-isc-protected-generate
-
+      job: aws-isc-generate
 
 # Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64)
 
@@ -831,33 +635,19 @@ aws-isc-protected-build:
   variables:
     SPACK_CI_STACK_NAME: aws-isc-aarch64
 
-aws-isc-aarch64-pr-generate:
-  extends: [ ".aws-isc-aarch64", ".pr-generate-aarch64", ".aws-isc-overrides" ]
-
-aws-isc-aarch64-protected-generate:
-  extends: [ ".aws-isc-aarch64", ".protected-generate-aarch64", ".aws-isc-overrides" ]
-
-aws-isc-aarch64-pr-build:
-  extends: [ ".aws-isc-aarch64", ".pr-build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: aws-isc-aarch64-pr-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: aws-isc-aarch64-pr-generate
+aws-isc-aarch64-generate:
+  extends: [ ".aws-isc-aarch64", ".generate-aarch64", ".aws-isc-overrides" ]
 
-aws-isc-aarch64-protected-build:
-  extends: [ ".aws-isc-aarch64", ".protected-build" ]
+aws-isc-aarch64-build:
+  extends: [ ".aws-isc-aarch64", ".build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: aws-isc-aarch64-protected-generate
+        job: aws-isc-aarch64-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: aws-isc-aarch64-protected-generate
+      job: aws-isc-aarch64-generate
 
 
 ########################################
@@ -868,35 +658,20 @@ aws-isc-aarch64-protected-build:
   variables:
     SPACK_CI_STACK_NAME: tutorial
 
-tutorial-pr-generate:
-  extends: [ ".tutorial", ".pr-generate"]
+tutorial-generate:
+  extends: [ ".tutorial", ".generate"]
   image: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-05-07
 
-tutorial-protected-generate:
-  extends: [ ".tutorial", ".protected-generate"]
-  image: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-05-07
-
-tutorial-pr-build:
-  extends: [ ".tutorial", ".pr-build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: tutorial-pr-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: tutorial-pr-generate
-
-tutorial-protected-build:
-  extends: [ ".tutorial", ".protected-build" ]
+tutorial-build:
+  extends: [ ".tutorial", ".build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: tutorial-protected-generate
+        job: tutorial-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: tutorial-protected-generate
+      job: tutorial-generate
 
 #######################################
 # Machine Learning - Linux x86_64 (CPU)
@@ -906,37 +681,20 @@ tutorial-protected-build:
   variables:
     SPACK_CI_STACK_NAME: ml-linux-x86_64-cpu
 
-.ml-linux-x86_64-cpu-generate:
-  extends: [ .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ]
+ml-linux-x86_64-cpu-generate:
+  extends: [ ".generate", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ]
   image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
 
-ml-linux-x86_64-cpu-pr-generate:
-  extends: [ ".pr-generate", ".ml-linux-x86_64-cpu-generate" ]
-
-ml-linux-x86_64-cpu-protected-generate:
-  extends: [ ".protected-generate", ".ml-linux-x86_64-cpu-generate" ]
-
-ml-linux-x86_64-cpu-pr-build:
-  extends: [ ".pr-build", ".ml-linux-x86_64-cpu" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: ml-linux-x86_64-cpu-pr-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: ml-linux-x86_64-cpu-pr-generate
-
-ml-linux-x86_64-cpu-protected-build:
-  extends: [ ".protected-build", ".ml-linux-x86_64-cpu" ]
+ml-linux-x86_64-cpu-build:
+  extends: [ ".build", ".ml-linux-x86_64-cpu" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: ml-linux-x86_64-cpu-protected-generate
+        job: ml-linux-x86_64-cpu-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: ml-linux-x86_64-cpu-protected-generate
+      job: ml-linux-x86_64-cpu-generate
 
 ########################################
 # Machine Learning - Linux x86_64 (CUDA)
@@ -946,37 +704,20 @@ ml-linux-x86_64-cpu-protected-build:
   variables:
     SPACK_CI_STACK_NAME: ml-linux-x86_64-cuda
 
-.ml-linux-x86_64-cuda-generate:
-  extends: [ .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ]
+ml-linux-x86_64-cuda-generate:
+  extends: [ ".generate", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ]
   image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
 
-ml-linux-x86_64-cuda-pr-generate:
-  extends: [ ".pr-generate", ".ml-linux-x86_64-cuda-generate" ]
-
-ml-linux-x86_64-cuda-protected-generate:
-  extends: [ ".protected-generate", ".ml-linux-x86_64-cuda-generate" ]
-
-ml-linux-x86_64-cuda-pr-build:
-  extends: [ ".pr-build", ".ml-linux-x86_64-cuda" ]
+ml-linux-x86_64-cuda-build:
+  extends: [ ".build", ".ml-linux-x86_64-cuda"  ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: ml-linux-x86_64-cuda-pr-generate
+        job: ml-linux-x86_64-cuda-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: ml-linux-x86_64-cuda-pr-generate
-
-ml-linux-x86_64-cuda-protected-build:
-  extends: [ ".protected-build", ".ml-linux-x86_64-cuda"  ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: ml-linux-x86_64-cuda-protected-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: ml-linux-x86_64-cuda-protected-generate
+      job: ml-linux-x86_64-cuda-generate
 
 ########################################
 # Machine Learning - Linux x86_64 (ROCm)
@@ -986,38 +727,20 @@ ml-linux-x86_64-cuda-protected-build:
   variables:
     SPACK_CI_STACK_NAME: ml-linux-x86_64-rocm
 
-.ml-linux-x86_64-rocm-generate:
-  extends: [ .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ]
+ml-linux-x86_64-rocm-generate:
+  extends: [ ".generate", .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ]
   image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
 
-ml-linux-x86_64-rocm-pr-generate:
-  extends: [ ".pr-generate", ".ml-linux-x86_64-rocm-generate" ]
-
-ml-linux-x86_64-rocm-protected-generate:
-  extends: [ ".protected-generate", ".ml-linux-x86_64-rocm-generate" ]
-
-ml-linux-x86_64-rocm-pr-build:
-  extends: [ ".pr-build", ".ml-linux-x86_64-rocm"  ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: ml-linux-x86_64-rocm-pr-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: ml-linux-x86_64-rocm-pr-generate
-
-ml-linux-x86_64-rocm-protected-build:
-  extends: [ ".protected-build", ".ml-linux-x86_64-rocm" ]
+ml-linux-x86_64-rocm-build:
+  extends: [ ".build", ".ml-linux-x86_64-rocm" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: ml-linux-x86_64-rocm-protected-generate
+        job: ml-linux-x86_64-rocm-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: ml-linux-x86_64-rocm-protected-generate
-
+      job: ml-linux-x86_64-rocm-generate
 
 ########################################
 # Deprecated CI testing
@@ -1026,30 +749,16 @@ ml-linux-x86_64-rocm-protected-build:
   variables:
     SPACK_CI_STACK_NAME: deprecated
 
-deprecated-ci-pr-generate:
-  extends: [ ".pr-generate-deprecated", ".deprecated-ci" ]
-
-deprecated-ci-protected-generate:
-  extends: [ ".protected-generate-deprecated", ".deprecated-ci" ]
-
-deprecated-ci-pr-build:
-  extends: [ ".pr-build", ".deprecated-ci" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: deprecated-ci-pr-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: deprecated-ci-pr-generate
+deprecated-ci-generate:
+  extends: [ ".generate-deprecated", ".deprecated-ci" ]
 
-deprecated-ci-protected-build:
-  extends: [ ".protected-build", ".deprecated-ci" ]
+deprecated-ci-build:
+  extends: [ ".build", ".deprecated-ci" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: deprecated-ci-protected-generate
+        job: deprecated-ci-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: deprecated-ci-protected-generate
+      job: deprecated-ci-generate
diff --git a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
index bf9df5878d..f65760c923 100644
--- a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
@@ -36,6 +36,34 @@ ci:
         - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
         - aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
 
+  - copy-job:
+      tags: ["service", "x86_64"]
+      image: "ghcr.io/spack/python-aws-bash:0.0.1"
+      before_script:
+      - - if [[ $CI_COMMIT_TAG == "v"* ]]; then export SPACK_REPLACE_VERSION=$(echo "$CI_COMMIT_TAG" | sed 's/\(v[[:digit:]]\+\.[[:digit:]]\+\).*/releases\/\1/'); fi
+        - if [[ $CI_COMMIT_TAG == "develop-"* ]]; then export SPACK_REPLACE_VERSION=develop; fi
+        - export SPACK_BUILDCACHE_SOURCE=${SPACK_SOURCE_MIRROR//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}}
+      script:
+      - - cd ${SPACK_CONCRETE_ENV_DIR}
+        - spack env activate --without-view .
+        - echo Copying environment specs from ${SRC_MIRROR} to ${SPACK_BUILDCACHE_DESTINATION}
+        - spack buildcache sync "${SPACK_BUILDCACHE_SOURCE}" "${SPACK_BUILDCACHE_DESTINATION}"
+        - curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub
+        - aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_BUILDCACHE_DESTINATION}/build_cache/_pgp/spack-public-binary-key.pub"
+        - spack buildcache update-index --keys "${SPACK_BUILDCACHE_DESTINATION}"
+      when: "always"
+      retry:
+        max: 2
+        when:
+        - "runner_system_failure"
+        - "stuck_or_timeout_failure"
+        - "script_failure"
+      interruptible: true
+      variables:
+        CI_JOB_SIZE: "medium"
+        KUBERNETES_CPU_REQUEST: "4000m"
+        KUBERNETES_MEMORY_REQUEST: "16G"
+
   - reindex-job:
       tags: ["service"]
       variables: