gitlab ci: Rework how mirrors are configured (#39939)

Improve how mirrors are used in gitlab ci, where we have until now thought
of them as only a string.

By configuring ci mirrors ahead of time using the proposed mirror templates,
and by taking advantage of the expressiveness that spack now has for mirrors,
this PR will allow us to easily switch the protocol/url we use for fetching
binary dependencies.

This change also deprecates some gitlab functionality and marks it for
removal in Spack 0.23:

    - arguments to "spack ci generate":
        * --buildcache-destination
        * --copy-to
    - gitlab configuration options:
        * enable-artifacts-buildcache
        * temporary-storage-url-prefix

30 files changed, 77 insertions, 81 deletions

diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
index b5e57e3e84..905901ff29 100644
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -3,6 +3,12 @@ stages: [ "generate", "build", "publish" ]
 variables:
   SPACK_DISABLE_LOCAL_CONFIG: "1"
   SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/tmp/_user_cache/"
+  # PR_MIRROR_FETCH_DOMAIN: "https://binaries-prs.spack.io"
+  PR_MIRROR_FETCH_DOMAIN: "s3://spack-binaries-prs"
+  PR_MIRROR_PUSH_DOMAIN: "s3://spack-binaries-prs"
+  # PROTECTED_MIRROR_FETCH_DOMAIN: "https://binaries.spack.io"
+  PROTECTED_MIRROR_FETCH_DOMAIN: "s3://spack-binaries"
+  PROTECTED_MIRROR_PUSH_DOMAIN: "s3://spack-binaries"
 
 default:
   image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
@@ -68,7 +74,9 @@ default:
 ########################################
 .base-job:
   variables:
-    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+    PIPELINE_MIRROR_TEMPLATE: "single-src-protected-mirrors.yaml.in"
+    # TODO: We can remove this when we drop the "deprecated" stack
+    PUSH_BUILDCACHE_DEPRECATED: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
 
   rules:
     - if: $CI_COMMIT_REF_NAME == "develop"
@@ -76,7 +84,7 @@ default:
       when: always
       variables:
         SPACK_PIPELINE_TYPE: "spack_protected_branch"
-        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+        SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
         SPACK_REQUIRE_SIGNING: "True"
         AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
         AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
@@ -86,7 +94,7 @@ default:
       when: always
       variables:
         SPACK_PIPELINE_TYPE: "spack_protected_branch"
-        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+        SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
         SPACK_PRUNE_UNTOUCHED: "False"
         SPACK_PRUNE_UP_TO_DATE: "False"
         SPACK_REQUIRE_SIGNING: "True"
@@ -98,8 +106,8 @@ default:
       when: always
       variables:
         SPACK_PIPELINE_TYPE: "spack_copy_only"
-        SPACK_SOURCE_MIRROR: "s3://spack-binaries/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}"
-        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+        SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
+        PIPELINE_MIRROR_TEMPLATE: "copy-only-protected-mirrors.yaml.in"
         AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
         AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
         OIDC_TOKEN_AUDIENCE: "protected_binary_mirror"
@@ -108,9 +116,16 @@ default:
       when: always
       variables:
         SPACK_PIPELINE_TYPE: "spack_pull_request"
-        SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+        # TODO: We can remove this when we drop the "deprecated" stack
+        PUSH_BUILDCACHE_DEPRECATED: "${PR_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
         SPACK_PRUNE_UNTOUCHED: "True"
         SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
+        # TODO: Change sync script to include target in branch name.  Then we could
+        # TODO: have multiple types of "PR" pipeline here.  It would be better if we could
+        # TODO: keep just this one and use a regex to capture the target branch, but so
+        # TODO: far gitlab doesn't support that.
+        PR_TARGET_REF_NAME: "develop"
+        PIPELINE_MIRROR_TEMPLATE: "multi-src-mirrors.yaml.in"
         AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
         AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
         OIDC_TOKEN_AUDIENCE: "pr_binary_mirror"
@@ -126,13 +141,15 @@ default:
     - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
     - spack env activate --without-view .
     - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
+    - spack python -c "import os,sys; print(os.path.expandvars(sys.stdin.read()))"
+      < "${SPACK_CI_CONFIG_ROOT}/${PIPELINE_MIRROR_TEMPLATE}" > "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
+    - spack config add -f "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
     - spack
       --config-scope "${SPACK_CI_CONFIG_ROOT}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
       ${CI_STACK_CONFIG_SCOPES}
       ci generate --check-index-only
-      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
       --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
       --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
   after_script:
@@ -182,7 +199,7 @@ default:
     - spack env activate --without-view .
     - spack
       ci generate --check-index-only
-      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
+      --buildcache-destination "${PUSH_BUILDCACHE_DEPRECATED}"
       --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
       --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
   after_script:
@@ -219,8 +236,7 @@ protected-publish:
     max: 2
     when: ["runner_system_failure", "stuck_or_timeout_failure"]
   variables:
-    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
-    SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+    SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
     SPACK_PIPELINE_TYPE: "spack_protected_branch"
     AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
     AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
@@ -253,11 +269,6 @@ protected-publish:
 # you should inlclude your custom definitions at the end of the of the
 # extends list.
 #
-# Also note that if extending .base-job, the mirror url given in your
-# spack.yaml should take the form:
-#
-#     s3://spack-binaries/develop/${SPACK_CI_STACK_NAME}
-#
 ########################################
 # My Super Cool Pipeline
 ########################################
diff --git a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
index 5f7e904ba5..9aad850b5d 100644
--- a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
@@ -21,7 +21,8 @@ ci:
         - k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust $k
         - k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k
       script::
-      - - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+      - - spack config blame mirrors
+        - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
       - - spack python ${CI_PROJECT_DIR}/share/spack/gitlab/cloud_pipelines/scripts/common/aggregate_package_logs.spack.py
           --prefix /home/software/spack:${CI_PROJECT_DIR}
           --log install_times.json
@@ -40,10 +41,10 @@ ci:
       image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
       tags: ["aws"]
       script:
-      - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
+      - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_BUILDCACHE_DESTINATION}/build_cache /tmp
         - /sign.sh
-        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
-        - aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
+        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_BUILDCACHE_DESTINATION}/build_cache
+        - aws s3 cp /tmp/public_keys ${SPACK_BUILDCACHE_DESTINATION}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
       id_tokens:
         GITLAB_OIDC_TOKEN:
           aud: "${OIDC_TOKEN_AUDIENCE}"
@@ -54,14 +55,14 @@ ci:
       before_script:
       - - if [[ $CI_COMMIT_TAG == "v"* ]]; then export SPACK_REPLACE_VERSION=$(echo "$CI_COMMIT_TAG" | sed 's/\(v[[:digit:]]\+\.[[:digit:]]\+\).*/releases\/\1/'); fi
         - if [[ $CI_COMMIT_TAG == "develop-"* ]]; then export SPACK_REPLACE_VERSION=develop; fi
-        - export SPACK_BUILDCACHE_SOURCE=${SPACK_SOURCE_MIRROR//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}}
+        - export SPACK_COPY_ONLY_SOURCE=${SPACK_BUILDCACHE_SOURCE//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}}
       script:
       - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
-        - echo Copying environment specs from ${SRC_MIRROR} to ${SPACK_BUILDCACHE_DESTINATION}
-        - spack buildcache sync "${SPACK_BUILDCACHE_SOURCE}" "${SPACK_BUILDCACHE_DESTINATION}"
+        - echo Copying environment specs from ${SPACK_COPY_ONLY_SOURCE} to ${SPACK_COPY_ONLY_DESTINATION}
+        - spack buildcache sync "${SPACK_COPY_ONLY_SOURCE}" "${SPACK_COPY_ONLY_DESTINATION}"
         - curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub
-        - aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_BUILDCACHE_DESTINATION}/build_cache/_pgp/spack-public-binary-key.pub"
-        - spack buildcache update-index --keys "${SPACK_BUILDCACHE_DESTINATION}"
+        - aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_COPY_ONLY_DESTINATION}/build_cache/_pgp/spack-public-binary-key.pub"
+        - spack buildcache update-index --keys "${SPACK_COPY_ONLY_DESTINATION}"
       when: "always"
       retry:
         max: 2
@@ -89,6 +90,7 @@ ci:
         GITLAB_OIDC_TOKEN:
           aud: "${OIDC_TOKEN_AUDIENCE}"
 
+  # TODO: Remove this block in Spack 0.23
   - cleanup-job:
       tags: ["service"]
       variables:
diff --git a/share/spack/gitlab/cloud_pipelines/configs/copy-only-protected-mirrors.yaml.in b/share/spack/gitlab/cloud_pipelines/configs/copy-only-protected-mirrors.yaml.in
new file mode 100644
index 0000000000..39e5c733b2
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/copy-only-protected-mirrors.yaml.in
@@ -0,0 +1,11 @@
+mirrors:
+  buildcache-source:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
+  buildcache-destination:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
diff --git a/share/spack/gitlab/cloud_pipelines/configs/multi-src-mirrors.yaml.in b/share/spack/gitlab/cloud_pipelines/configs/multi-src-mirrors.yaml.in
new file mode 100644
index 0000000000..0ad46d5fc9
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/multi-src-mirrors.yaml.in
@@ -0,0 +1,16 @@
+mirrors:
+  buildcache-source:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/${PR_TARGET_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/${PR_TARGET_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
+  buildcache-destination:
+    fetch: ${PR_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PR_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
+  buildcache-shared:
+    fetch: ${PR_MIRROR_FETCH_DOMAIN}/shared_pr_mirror/${SPACK_CI_STACK_NAME}
+    push: ${PR_MIRROR_PUSH_DOMAIN}/shared_pr_mirror/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
diff --git a/share/spack/gitlab/cloud_pipelines/configs/single-src-pr-mirrors.yaml.in b/share/spack/gitlab/cloud_pipelines/configs/single-src-pr-mirrors.yaml.in
new file mode 100644
index 0000000000..0a2775a4a2
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/single-src-pr-mirrors.yaml.in
@@ -0,0 +1,6 @@
+mirrors:
+  buildcache-destination:
+    fetch: ${PR_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PR_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
diff --git a/share/spack/gitlab/cloud_pipelines/configs/single-src-protected-mirrors.yaml.in b/share/spack/gitlab/cloud_pipelines/configs/single-src-protected-mirrors.yaml.in
new file mode 100644
index 0000000000..a55cd72737
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/single-src-protected-mirrors.yaml.in
@@ -0,0 +1,6 @@
+mirrors:
+  buildcache-destination:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
index 1c4e2de308..abd8f4d024 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
@@ -131,9 +131,6 @@ spack:
     - - $compiler
     - - $target
 
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-isc-aarch64" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
index 0a898d1a75..038761ac18 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
@@ -142,9 +142,6 @@ spack:
     - - $compiler
     - - $target
 
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-isc" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml
index 5ce6d1c869..85cf766068 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml
@@ -30,8 +30,6 @@ spack:
   - $optimized_configs
   # - $optimized_libs
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-icelake" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml
index 5708338a2b..50ba40992a 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml
@@ -30,9 +30,6 @@ spack:
   - $optimized_configs
   - $optimized_libs
 
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-neoverse_n1" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
index f2df769610..50ba40992a 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
@@ -30,9 +30,6 @@ spack:
   - $optimized_configs
   - $optimized_libs
 
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-neoverse_v1" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml
index 029dd67351..85cf766068 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml
@@ -30,8 +30,6 @@ spack:
   - $optimized_configs
   # - $optimized_libs
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-skylake" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
index 78a3ea785c..d154894830 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
@@ -21,7 +21,5 @@ spack:
         - - $default_specs
         - - $arch
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/build_systems" }
-
   cdash:
     build-group: Build Systems
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
index 9963f4b777..bf298d606d 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
@@ -58,8 +58,6 @@ spack:
     - ["~paraview +visit"]
     - [$^visit_specs]
 
-  mirrors: {mirror: s3://spack-binaries/develop/data-vis-sdk}
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
index 83e3d9f290..413fdf34eb 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
@@ -172,7 +172,5 @@ spack:
   # - variorum            # variorum: /opt/cray/pe/cce/15.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld: /opt/cray/pe/lib64/libpals.so.0: undefined reference to `json_array_append_new@@libjansson.so.4'
   # - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos~shylu # openblas: ftn-2307 ftn: ERROR in command line: The "-m" option must be followed by 0, 1, 2, 3 or 4.; make[2]: *** [<builtin>: spotrf2.o] Error 1; make[1]: *** [Makefile:27: lapacklib] Error 2; make: *** [Makefile:250: netlib] Error 2
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-cray-rhel" }
-
   cdash:
     build-group: E4S Cray
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
index dace63659e..c141cd9bf9 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
@@ -171,7 +171,5 @@ spack:
   # - variorum
   # - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos~shylu
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-cray-sles" }
-
   cdash:
     build-group: E4S Cray SLES
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml
index b4e8114df6..db903c15c4 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml
@@ -340,8 +340,6 @@ spack:
   # - tasmanian +cuda cuda_arch=90  # tasmanian: conflicts with cuda@12
   # - upcxx +cuda cuda_arch=90      # upcxx: needs NVIDIA driver
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-arm-neoverse_v1" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
index ec86c35b33..d170b0a272 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
@@ -236,8 +236,6 @@ spack:
 
   - py-scipy
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-oneapi" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
index 72e06b060d..5f8f3d0e62 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
@@ -251,8 +251,6 @@ spack:
   # - trilinos +cuda cuda_arch=70           # trilinos: https://github.com/trilinos/Trilinos/issues/11630
   # - upcxx +cuda cuda_arch=70              # upcxx: needs NVIDIA driver
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-power" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml
index cd9addbef0..885dbb538b 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml
@@ -335,8 +335,6 @@ spack:
   # - lbann ~cuda +rocm amdgpu_target=gfx90a    # aluminum: https://github.com/spack/spack/issues/38807
   # - papi +rocm amdgpu_target=gfx90a           # papi: https://github.com/spack/spack/issues/27898
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-rocm-external" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
index 65ab32e80d..410a379ee0 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
@@ -382,8 +382,6 @@ spack:
   # - lbann ~cuda +rocm amdgpu_target=gfx90a    # aluminum: https://github.com/spack/spack/issues/38807
   # - papi +rocm amdgpu_target=gfx90a           # papi: https://github.com/spack/spack/issues/27898
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml
index 69a8859745..263d8e29b3 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml
@@ -49,8 +49,6 @@ spack:
   # FAILURES
   # - kokkos +wrapper +cuda cuda_arch=80 ^cuda@12.0.0     # https://github.com/spack/spack/issues/35378
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/gpu-tests" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
index c137b138ee..6d8a0b7491 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
@@ -82,8 +82,6 @@ spack:
   # - r-xgboost
   - xgboost
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/ml-darwin-aarch64-mps" }
-
   ci:
     pipeline-gen:
     - build-job-remove:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
index fa7bf02755..71670d5a91 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
@@ -76,9 +76,6 @@ spack:
     # - r-xgboost
     - xgboost
 
-  mirrors:
-    mirror: s3://spack-binaries/develop/ml-linux-x86_64-cpu
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
index 5a24d42f23..8829169038 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
@@ -79,9 +79,6 @@ spack:
     # - r-xgboost
     - xgboost
 
-  mirrors:
-    mirror: s3://spack-binaries/develop/ml-linux-x86_64-cuda
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml
index e49d43db3d..620a95715b 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml
@@ -82,9 +82,6 @@ spack:
     # - r-xgboost
     - xgboost
 
-  mirrors:
-    mirror: s3://spack-binaries/develop/ml-linux-x86_64-rocm
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml
index b05b45f763..6453d2a5fe 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml
@@ -38,8 +38,6 @@ spack:
     - - $compiler
     - - $target
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/radiuss-aws-aarch64" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml
index fd297ede91..ca7de563c4 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml
@@ -28,7 +28,7 @@ spack:
     - mfem +cuda ^hypre+cuda
     - raja
     - raja +cuda
-    - umpire 
+    - umpire
     - umpire +cuda
 
   - compiler:
@@ -44,8 +44,6 @@ spack:
     - - $compiler
     - - $target
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/radiuss-aws" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
index c80bcf10ee..ca8e1a9905 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
@@ -40,9 +40,6 @@ spack:
       - xbraid
       - zfp
 
-  mirrors:
-    mirror: "s3://spack-binaries/develop/radiuss"
-
   specs:
   - matrix:
     - [$radiuss]
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
index 4b39be8846..0bc36ce8e4 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
@@ -49,8 +49,6 @@ spack:
   - $clang_packages
   - $gcc_spack_built_packages
 
-  mirrors:
-    mirror: s3://spack-binaries/develop/tutorial
   ci:
     pipeline-gen:
     - build-job: