gitlab ci: Rework how mirrors are configured (#39939)

Improve how mirrors are used in gitlab ci, where we have until now thought
of them as only a string.

By configuring ci mirrors ahead of time using the proposed mirror templates,
and by taking advantage of the expressiveness that spack now has for mirrors,
this PR will allow us to easily switch the protocol/url we use for fetching
binary dependencies.

This change also deprecates some gitlab functionality and marks it for
removal in Spack 0.23:

    - arguments to "spack ci generate":
        * --buildcache-destination
        * --copy-to
    - gitlab configuration options:
        * enable-artifacts-buildcache
        * temporary-storage-url-prefix

36 files changed, 273 insertions, 150 deletions

diff --git a/lib/spack/docs/pipelines.rst b/lib/spack/docs/pipelines.rst
index d594879aab..4ebe90fb0b 100644
--- a/lib/spack/docs/pipelines.rst
+++ b/lib/spack/docs/pipelines.rst
@@ -213,6 +213,16 @@ pipeline jobs.
 ``spack ci generate``
 ^^^^^^^^^^^^^^^^^^^^^
 
+Throughout this documentation, references to the "mirror" mean the target
+mirror which is checked for the presence of up-to-date specs, and where
+any scheduled jobs should push built binary packages.  In the past, this
+defaulted to the mirror at index 0 in the mirror configs, and could be
+overridden using the ``--buildcache-destination`` argument. Starting with
+Spack 0.23, ``spack ci generate`` will require you to identify this mirror
+by the name "buildcache-destination".  While you can configure any number
+of mirrors as sources for your pipelines, you will need to identify the
+destination mirror by name.
+
 Concretizes the specs in the active environment, stages them (as described in
 :ref:`staging_algorithm`), and writes the resulting ``.gitlab-ci.yml`` to disk.
 During concretization of the environment, ``spack ci generate`` also writes a
diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py
index bf5aaa79a3..fca2836254 100644
--- a/lib/spack/spack/ci.py
+++ b/lib/spack/spack/ci.py
@@ -49,6 +49,7 @@ JOB_RETRY_CONDITIONS = ["always"]
 
 TEMP_STORAGE_MIRROR_NAME = "ci_temporary_mirror"
 SPACK_RESERVED_TAGS = ["public", "protected", "notary"]
+# TODO: Remove this in Spack 0.23
 SHARED_PR_MIRROR_URL = "s3://spack-binaries-prs/shared_pr_mirror"
 JOB_NAME_FORMAT = (
     "{name}{@version} {/hash:7} {%compiler.name}{@compiler.version}{arch=architecture}"
@@ -678,7 +679,7 @@ def generate_gitlab_ci_yaml(
         remote_mirror_override (str): Typically only needed when one spack.yaml
             is used to populate several mirrors with binaries, based on some
             criteria.  Spack protected pipelines populate different mirrors based
-            on branch name, facilitated by this option.
+            on branch name, facilitated by this option.  DEPRECATED
     """
     with spack.concretize.disable_compiler_existence_check():
         with env.write_transaction():
@@ -775,17 +776,39 @@ def generate_gitlab_ci_yaml(
             "instead.",
         )
 
-    if "mirrors" not in yaml_root or len(yaml_root["mirrors"].values()) < 1:
-        tty.die("spack ci generate requires an env containing a mirror")
+    pipeline_mirrors = spack.mirror.MirrorCollection(binary=True)
+    deprecated_mirror_config = False
+    buildcache_destination = None
+    if "buildcache-destination" in pipeline_mirrors:
+        if remote_mirror_override:
+            tty.die(
+                "Using the deprecated --buildcache-destination cli option and "
+                "having a mirror named 'buildcache-destination' at the same time "
+                "is not allowed"
+            )
+        buildcache_destination = pipeline_mirrors["buildcache-destination"]
+    else:
+        deprecated_mirror_config = True
+        # TODO: This will be an error in Spack 0.23
 
-    ci_mirrors = yaml_root["mirrors"]
-    mirror_urls = [url for url in ci_mirrors.values()]
-    remote_mirror_url = mirror_urls[0]
+    # TODO: Remove this block in spack 0.23
+    remote_mirror_url = None
+    if deprecated_mirror_config:
+        if "mirrors" not in yaml_root or len(yaml_root["mirrors"].values()) < 1:
+            tty.die("spack ci generate requires an env containing a mirror")
+
+        ci_mirrors = yaml_root["mirrors"]
+        mirror_urls = [url for url in ci_mirrors.values()]
+        remote_mirror_url = mirror_urls[0]
 
     spack_buildcache_copy = os.environ.get("SPACK_COPY_BUILDCACHE", None)
     if spack_buildcache_copy:
         buildcache_copies = {}
-        buildcache_copy_src_prefix = remote_mirror_override or remote_mirror_url
+        buildcache_copy_src_prefix = (
+            buildcache_destination.fetch_url
+            if buildcache_destination
+            else remote_mirror_override or remote_mirror_url
+        )
         buildcache_copy_dest_prefix = spack_buildcache_copy
 
     # Check for a list of "known broken" specs that we should not bother
@@ -797,6 +820,7 @@ def generate_gitlab_ci_yaml(
 
     enable_artifacts_buildcache = False
     if "enable-artifacts-buildcache" in ci_config:
+        tty.warn("Support for enable-artifacts-buildcache will be removed in Spack 0.23")
         enable_artifacts_buildcache = ci_config["enable-artifacts-buildcache"]
 
     rebuild_index_enabled = True
@@ -805,13 +829,15 @@ def generate_gitlab_ci_yaml(
 
     temp_storage_url_prefix = None
     if "temporary-storage-url-prefix" in ci_config:
+        tty.warn("Support for temporary-storage-url-prefix will be removed in Spack 0.23")
         temp_storage_url_prefix = ci_config["temporary-storage-url-prefix"]
 
     # If a remote mirror override (alternate buildcache destination) was
     # specified, add it here in case it has already built hashes we might
     # generate.
+    # TODO: Remove this block in Spack 0.23
     mirrors_to_check = None
-    if remote_mirror_override:
+    if deprecated_mirror_config and remote_mirror_override:
         if spack_pipeline_type == "spack_protected_branch":
             # Overriding the main mirror in this case might result
             # in skipping jobs on a release pipeline because specs are
@@ -831,8 +857,9 @@ def generate_gitlab_ci_yaml(
             cfg.default_modify_scope(),
         )
 
+    # TODO: Remove this block in Spack 0.23
     shared_pr_mirror = None
-    if spack_pipeline_type == "spack_pull_request":
+    if deprecated_mirror_config and spack_pipeline_type == "spack_pull_request":
         stack_name = os.environ.get("SPACK_CI_STACK_NAME", "")
         shared_pr_mirror = url_util.join(SHARED_PR_MIRROR_URL, stack_name)
         spack.mirror.add(
@@ -884,6 +911,7 @@ def generate_gitlab_ci_yaml(
     job_log_dir = os.path.join(pipeline_artifacts_dir, "logs")
     job_repro_dir = os.path.join(pipeline_artifacts_dir, "reproduction")
     job_test_dir = os.path.join(pipeline_artifacts_dir, "tests")
+    # TODO: Remove this line in Spack 0.23
     local_mirror_dir = os.path.join(pipeline_artifacts_dir, "mirror")
     user_artifacts_dir = os.path.join(pipeline_artifacts_dir, "user_data")
 
@@ -898,11 +926,11 @@ def generate_gitlab_ci_yaml(
     rel_job_log_dir = os.path.relpath(job_log_dir, ci_project_dir)
     rel_job_repro_dir = os.path.relpath(job_repro_dir, ci_project_dir)
     rel_job_test_dir = os.path.relpath(job_test_dir, ci_project_dir)
+    # TODO: Remove this line in Spack 0.23
     rel_local_mirror_dir = os.path.join(local_mirror_dir, ci_project_dir)
     rel_user_artifacts_dir = os.path.relpath(user_artifacts_dir, ci_project_dir)
 
     # Speed up staging by first fetching binary indices from all mirrors
-    # (including the override mirror we may have just added above).
     try:
         bindist.binary_index.update()
     except bindist.FetchCacheError as e:
@@ -1113,6 +1141,7 @@ def generate_gitlab_ci_yaml(
                 },
             )
 
+            # TODO: Remove this block in Spack 0.23
             if enable_artifacts_buildcache:
                 bc_root = os.path.join(local_mirror_dir, "build_cache")
                 job_object["artifacts"]["paths"].extend(
@@ -1142,10 +1171,12 @@ def generate_gitlab_ci_yaml(
         _print_staging_summary(spec_labels, stages, mirrors_to_check, rebuild_decisions)
 
     # Clean up remote mirror override if enabled
-    if remote_mirror_override:
-        spack.mirror.remove("ci_pr_mirror", cfg.default_modify_scope())
-    if spack_pipeline_type == "spack_pull_request":
-        spack.mirror.remove("ci_shared_pr_mirror", cfg.default_modify_scope())
+    # TODO: Remove this block in Spack 0.23
+    if deprecated_mirror_config:
+        if remote_mirror_override:
+            spack.mirror.remove("ci_pr_mirror", cfg.default_modify_scope())
+        if spack_pipeline_type == "spack_pull_request":
+            spack.mirror.remove("ci_shared_pr_mirror", cfg.default_modify_scope())
 
     tty.debug("{0} build jobs generated in {1} stages".format(job_id, stage_id))
 
@@ -1176,10 +1207,28 @@ def generate_gitlab_ci_yaml(
             sync_job["needs"] = [
                 {"job": generate_job_name, "pipeline": "{0}".format(parent_pipeline_id)}
             ]
+
+        if "variables" not in sync_job:
+            sync_job["variables"] = {}
+
+        sync_job["variables"]["SPACK_COPY_ONLY_DESTINATION"] = (
+            buildcache_destination.fetch_url
+            if buildcache_destination
+            else remote_mirror_override or remote_mirror_url
+        )
+
+        if "buildcache-source" in pipeline_mirrors:
+            buildcache_source = pipeline_mirrors["buildcache-source"].fetch_url
+        else:
+            # TODO: Remove this condition in Spack 0.23
+            buildcache_source = os.environ.get("SPACK_SOURCE_MIRROR", None)
+        sync_job["variables"]["SPACK_BUILDCACHE_SOURCE"] = buildcache_source
+
         output_object["copy"] = sync_job
         job_id += 1
 
     if job_id > 0:
+        # TODO: Remove this block in Spack 0.23
         if temp_storage_url_prefix:
             # There were some rebuild jobs scheduled, so we will need to
             # schedule a job to clean up the temporary storage location
@@ -1213,6 +1262,13 @@ def generate_gitlab_ci_yaml(
             signing_job["when"] = "always"
             signing_job["retry"] = {"max": 2, "when": ["always"]}
             signing_job["interruptible"] = True
+            if "variables" not in signing_job:
+                signing_job["variables"] = {}
+            signing_job["variables"]["SPACK_BUILDCACHE_DESTINATION"] = (
+                buildcache_destination.push_url  # need the s3 url for aws s3 sync
+                if buildcache_destination
+                else remote_mirror_override or remote_mirror_url
+            )
 
             output_object["sign-pkgs"] = signing_job
 
@@ -1221,13 +1277,13 @@ def generate_gitlab_ci_yaml(
             stage_names.append("stage-rebuild-index")
             final_job = spack_ci_ir["jobs"]["reindex"]["attributes"]
 
-            index_target_mirror = mirror_urls[0]
-            if remote_mirror_override:
-                index_target_mirror = remote_mirror_override
             final_job["stage"] = "stage-rebuild-index"
+            target_mirror = remote_mirror_override or remote_mirror_url
+            if buildcache_destination:
+                target_mirror = buildcache_destination.push_url
             final_job["script"] = _unpack_script(
                 final_job["script"],
-                op=lambda cmd: cmd.replace("{index_target_mirror}", index_target_mirror),
+                op=lambda cmd: cmd.replace("{index_target_mirror}", target_mirror),
             )
 
             final_job["when"] = "always"
@@ -1249,20 +1305,24 @@ def generate_gitlab_ci_yaml(
             "SPACK_CONCRETE_ENV_DIR": rel_concrete_env_dir,
             "SPACK_VERSION": spack_version,
             "SPACK_CHECKOUT_VERSION": version_to_clone,
+            # TODO: Remove this line in Spack 0.23
             "SPACK_REMOTE_MIRROR_URL": remote_mirror_url,
             "SPACK_JOB_LOG_DIR": rel_job_log_dir,
             "SPACK_JOB_REPRO_DIR": rel_job_repro_dir,
             "SPACK_JOB_TEST_DIR": rel_job_test_dir,
+            # TODO: Remove this line in Spack 0.23
             "SPACK_LOCAL_MIRROR_DIR": rel_local_mirror_dir,
             "SPACK_PIPELINE_TYPE": str(spack_pipeline_type),
             "SPACK_CI_STACK_NAME": os.environ.get("SPACK_CI_STACK_NAME", "None"),
+            # TODO: Remove this line in Spack 0.23
             "SPACK_CI_SHARED_PR_MIRROR_URL": shared_pr_mirror or "None",
             "SPACK_REBUILD_CHECK_UP_TO_DATE": str(prune_dag),
             "SPACK_REBUILD_EVERYTHING": str(rebuild_everything),
             "SPACK_REQUIRE_SIGNING": os.environ.get("SPACK_REQUIRE_SIGNING", "False"),
         }
 
-        if remote_mirror_override:
+        # TODO: Remove this block in Spack 0.23
+        if deprecated_mirror_config and remote_mirror_override:
             (output_object["variables"]["SPACK_REMOTE_MIRROR_OVERRIDE"]) = remote_mirror_override
 
         spack_stack_name = os.environ.get("SPACK_CI_STACK_NAME", None)
@@ -2002,43 +2062,23 @@ def process_command(name, commands, repro_dir, run=True, exit_on_failure=True):
 
 
 def create_buildcache(
-    input_spec: spack.spec.Spec,
-    *,
-    pipeline_mirror_url: Optional[str] = None,
-    buildcache_mirror_url: Optional[str] = None,
-    sign_binaries: bool = False,
+    input_spec: spack.spec.Spec, *, destination_mirror_urls: List[str], sign_binaries: bool = False
 ) -> List[PushResult]:
     """Create the buildcache at the provided mirror(s).
 
     Arguments:
         input_spec: Installed spec to package and push
-        buildcache_mirror_url: URL for the buildcache mirror
-        pipeline_mirror_url: URL for the pipeline mirror
+        destination_mirror_urls: List of urls to push to
         sign_binaries: Whether or not to sign buildcache entry
 
     Returns: A list of PushResults, indicating success or failure.
     """
     results = []
 
-    # Create buildcache in either the main remote mirror, or in the
-    # per-PR mirror, if this is a PR pipeline
-    if buildcache_mirror_url:
-        results.append(
-            PushResult(
-                success=push_mirror_contents(input_spec, buildcache_mirror_url, sign_binaries),
-                url=buildcache_mirror_url,
-            )
-        )
-
-    # Create another copy of that buildcache in the per-pipeline
-    # temporary storage mirror (this is only done if either
-    # artifacts buildcache is enabled or a temporary storage url
-    # prefix is set)
-    if pipeline_mirror_url:
+    for mirror_url in destination_mirror_urls:
         results.append(
             PushResult(
-                success=push_mirror_contents(input_spec, pipeline_mirror_url, sign_binaries),
-                url=pipeline_mirror_url,
+                success=push_mirror_contents(input_spec, mirror_url, sign_binaries), url=mirror_url
             )
         )
 
diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py
index cf2ee11c04..6c57319302 100644
--- a/lib/spack/spack/cmd/ci.py
+++ b/lib/spack/spack/cmd/ci.py
@@ -191,6 +191,14 @@ def ci_generate(args):
     """
     env = spack.cmd.require_active_env(cmd_name="ci generate")
 
+    if args.copy_to:
+        tty.warn("The flag --copy-to is deprecated and will be removed in Spack 0.23")
+
+    if args.buildcache_destination:
+        tty.warn(
+            "The flag --buildcache-destination is deprecated and will be removed in Spack 0.23"
+        )
+
     output_file = args.output_file
     copy_yaml_to = args.copy_to
     run_optimizer = args.optimize
@@ -264,12 +272,6 @@ def ci_rebuild(args):
     if not ci_config:
         tty.die("spack ci rebuild requires an env containing ci cfg")
 
-    tty.msg(
-        "SPACK_BUILDCACHE_DESTINATION={0}".format(
-            os.environ.get("SPACK_BUILDCACHE_DESTINATION", None)
-        )
-    )
-
     # Grab the environment variables we need.  These either come from the
     # pipeline generation step ("spack ci generate"), where they were written
     # out as variables, or else provided by GitLab itself.
@@ -277,6 +279,7 @@ def ci_rebuild(args):
     job_log_dir = os.environ.get("SPACK_JOB_LOG_DIR")
     job_test_dir = os.environ.get("SPACK_JOB_TEST_DIR")
     repro_dir = os.environ.get("SPACK_JOB_REPRO_DIR")
+    # TODO: Remove this in Spack 0.23
     local_mirror_dir = os.environ.get("SPACK_LOCAL_MIRROR_DIR")
     concrete_env_dir = os.environ.get("SPACK_CONCRETE_ENV_DIR")
     ci_pipeline_id = os.environ.get("CI_PIPELINE_ID")
@@ -285,9 +288,12 @@ def ci_rebuild(args):
     job_spec_pkg_name = os.environ.get("SPACK_JOB_SPEC_PKG_NAME")
     job_spec_dag_hash = os.environ.get("SPACK_JOB_SPEC_DAG_HASH")
     spack_pipeline_type = os.environ.get("SPACK_PIPELINE_TYPE")
+    # TODO: Remove this in Spack 0.23
     remote_mirror_override = os.environ.get("SPACK_REMOTE_MIRROR_OVERRIDE")
+    # TODO: Remove this in Spack 0.23
     remote_mirror_url = os.environ.get("SPACK_REMOTE_MIRROR_URL")
     spack_ci_stack_name = os.environ.get("SPACK_CI_STACK_NAME")
+    # TODO: Remove this in Spack 0.23
     shared_pr_mirror_url = os.environ.get("SPACK_CI_SHARED_PR_MIRROR_URL")
     rebuild_everything = os.environ.get("SPACK_REBUILD_EVERYTHING")
     require_signing = os.environ.get("SPACK_REQUIRE_SIGNING")
@@ -344,21 +350,36 @@ def ci_rebuild(args):
 
     full_rebuild = True if rebuild_everything and rebuild_everything.lower() == "true" else False
 
+    pipeline_mirrors = spack.mirror.MirrorCollection(binary=True)
+    deprecated_mirror_config = False
+    buildcache_destination = None
+    if "buildcache-destination" in pipeline_mirrors:
+        buildcache_destination = pipeline_mirrors["buildcache-destination"]
+    else:
+        deprecated_mirror_config = True
+        # TODO: This will be an error in Spack 0.23
+
     # If no override url exists, then just push binary package to the
     # normal remote mirror url.
+    # TODO: Remove in Spack 0.23
     buildcache_mirror_url = remote_mirror_override or remote_mirror_url
+    if buildcache_destination:
+        buildcache_mirror_url = buildcache_destination.push_url
 
     # Figure out what is our temporary storage mirror: Is it artifacts
     # buildcache?  Or temporary-storage-url-prefix?  In some cases we need to
     # force something or pipelines might not have a way to propagate build
     # artifacts from upstream to downstream jobs.
+    # TODO: Remove this in Spack 0.23
     pipeline_mirror_url = None
 
+    # TODO: Remove this in Spack 0.23
     temp_storage_url_prefix = None
     if "temporary-storage-url-prefix" in ci_config:
         temp_storage_url_prefix = ci_config["temporary-storage-url-prefix"]
         pipeline_mirror_url = url_util.join(temp_storage_url_prefix, ci_pipeline_id)
 
+    # TODO: Remove this in Spack 0.23
     enable_artifacts_mirror = False
     if "enable-artifacts-buildcache" in ci_config:
         enable_artifacts_mirror = ci_config["enable-artifacts-buildcache"]
@@ -454,12 +475,14 @@ def ci_rebuild(args):
     # If we decided there should be a temporary storage mechanism, add that
     # mirror now so it's used when we check for a hash match already
     # built for this spec.
+    # TODO: Remove this block in Spack 0.23
     if pipeline_mirror_url:
         mirror = spack.mirror.Mirror(pipeline_mirror_url, name=spack_ci.TEMP_STORAGE_MIRROR_NAME)
         spack.mirror.add(mirror, cfg.default_modify_scope())
         pipeline_mirrors.append(pipeline_mirror_url)
 
     # Check configured mirrors for a built spec with a matching hash
+    # TODO: Remove this block in Spack 0.23
     mirrors_to_check = None
     if remote_mirror_override:
         if spack_pipeline_type == "spack_protected_branch":
@@ -477,7 +500,8 @@ def ci_rebuild(args):
             )
         pipeline_mirrors.append(remote_mirror_override)
 
-    if spack_pipeline_type == "spack_pull_request":
+    # TODO: Remove this in Spack 0.23
+    if deprecated_mirror_config and spack_pipeline_type == "spack_pull_request":
         if shared_pr_mirror_url != "None":
             pipeline_mirrors.append(shared_pr_mirror_url)
 
@@ -499,6 +523,7 @@ def ci_rebuild(args):
         tty.msg("No need to rebuild {0}, found hash match at: ".format(job_spec_pkg_name))
         for match in matches:
             tty.msg("    {0}".format(match["mirror_url"]))
+        # TODO: Remove this block in Spack 0.23
         if enable_artifacts_mirror:
             matching_mirror = matches[0]["mirror_url"]
             build_cache_dir = os.path.join(local_mirror_dir, "build_cache")
@@ -513,7 +538,8 @@ def ci_rebuild(args):
     # only want to keep the mirror being used by the current pipeline as it's binary
     # package destination.  This ensures that the when we rebuild everything, we only
     # consume binary dependencies built in this pipeline.
-    if full_rebuild:
+    # TODO: Remove this in Spack 0.23
+    if deprecated_mirror_config and full_rebuild:
         spack_ci.remove_other_mirrors(pipeline_mirrors, cfg.default_modify_scope())
 
     # No hash match anywhere means we need to rebuild spec
@@ -678,21 +704,25 @@ def ci_rebuild(args):
     # print out some instructions on how to reproduce this build failure
     # outside of the pipeline environment.
     if install_exit_code == 0:
-        if buildcache_mirror_url or pipeline_mirror_url:
-            for result in spack_ci.create_buildcache(
-                input_spec=job_spec,
-                buildcache_mirror_url=buildcache_mirror_url,
-                pipeline_mirror_url=pipeline_mirror_url,
-                sign_binaries=spack_ci.can_sign_binaries(),
-            ):
-                msg = tty.msg if result.success else tty.warn
-                msg(
-                    "{} {} to {}".format(
-                        "Pushed" if result.success else "Failed to push",
-                        job_spec.format("{name}{@version}{/hash:7}", color=clr.get_color_when()),
-                        result.url,
-                    )
+        mirror_urls = [buildcache_mirror_url]
+
+        # TODO: Remove this block in Spack 0.23
+        if pipeline_mirror_url:
+            mirror_urls.append(pipeline_mirror_url)
+
+        for result in spack_ci.create_buildcache(
+            input_spec=job_spec,
+            destination_mirror_urls=mirror_urls,
+            sign_binaries=spack_ci.can_sign_binaries(),
+        ):
+            msg = tty.msg if result.success else tty.warn
+            msg(
+                "{} {} to {}".format(
+                    "Pushed" if result.success else "Failed to push",
+                    job_spec.format("{name}{@version}{/hash:7}", color=clr.get_color_when()),
+                    result.url,
                 )
+            )
 
         # If this is a develop pipeline, check if the spec that we just built is
         # on the broken-specs list. If so, remove it.
diff --git a/lib/spack/spack/schema/ci.py b/lib/spack/spack/schema/ci.py
index 92edf2f139..9ba65b2682 100644
--- a/lib/spack/spack/schema/ci.py
+++ b/lib/spack/spack/schema/ci.py
@@ -141,6 +141,7 @@ core_shared_properties = union_dicts(
     }
 )
 
+# TODO: Remove in Spack 0.23
 ci_properties = {
     "anyOf": [
         {
@@ -166,6 +167,7 @@ ci_properties = {
 properties = {
     "ci": {
         "oneOf": [
+            # TODO: Replace with core-shared-properties in Spack 0.23
             ci_properties,
             # Allow legacy format under `ci` for `config update ci`
             spack.schema.gitlab_ci.gitlab_ci_properties,
diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py
index 1b9833894e..53ed838252 100644
--- a/lib/spack/spack/test/ci.py
+++ b/lib/spack/spack/test/ci.py
@@ -451,9 +451,7 @@ def test_ci_create_buildcache(tmpdir, working_env, config, mock_packages, monkey
     monkeypatch.setattr(spack.ci, "push_mirror_contents", lambda a, b, c: True)
 
     results = ci.create_buildcache(
-        None,
-        buildcache_mirror_url="file:///fake-url-one",
-        pipeline_mirror_url="file:///fake-url-two",
+        None, destination_mirror_urls=["file:///fake-url-one", "file:///fake-url-two"]
     )
 
     assert len(results) == 2
@@ -463,7 +461,7 @@ def test_ci_create_buildcache(tmpdir, working_env, config, mock_packages, monkey
     assert result2.success
     assert result2.url == "file:///fake-url-two"
 
-    results = ci.create_buildcache(None, buildcache_mirror_url="file:///fake-url-one")
+    results = ci.create_buildcache(None, destination_mirror_urls=["file:///fake-url-one"])
 
     assert len(results) == 1
     assert results[0].success
diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py
index d02e1caa2d..84e9e66bf0 100644
--- a/lib/spack/spack/test/cmd/ci.py
+++ b/lib/spack/spack/test/cmd/ci.py
@@ -2209,3 +2209,50 @@ spack:
             assert all([t in rebuild_tags for t in ["spack", "service"]])
             expected_vars = ["CI_JOB_SIZE", "KUBERNETES_CPU_REQUEST", "KUBERNETES_MEMORY_REQUEST"]
             assert all([v in rebuild_vars for v in expected_vars])
+
+
+def test_ci_generate_mirror_config(
+    tmpdir,
+    mutable_mock_env_path,
+    install_mockery,
+    mock_packages,
+    monkeypatch,
+    ci_base_environment,
+    mock_binary_index,
+):
+    """Make sure the correct mirror gets used as the buildcache destination"""
+    filename = str(tmpdir.join("spack.yaml"))
+    with open(filename, "w") as f:
+        f.write(
+            """\
+spack:
+  specs:
+    - archive-files
+  mirrors:
+    some-mirror: file:///this/is/a/source/mirror
+    buildcache-destination: file:///push/binaries/here
+  ci:
+    pipeline-gen:
+    - submapping:
+      - match:
+          - archive-files
+        build-job:
+          tags:
+            - donotcare
+          image: donotcare
+"""
+        )
+
+    with tmpdir.as_cwd():
+        env_cmd("create", "test", "./spack.yaml")
+        outputfile = str(tmpdir.join(".gitlab-ci.yml"))
+
+        with ev.read("test"):
+            ci_cmd("generate", "--output-file", outputfile)
+            with open(outputfile) as of:
+                pipeline_doc = syaml.load(of.read())
+                assert "rebuild-index" in pipeline_doc
+                reindex_job = pipeline_doc["rebuild-index"]
+                assert "script" in reindex_job
+                reindex_step = reindex_job["script"][0]
+                assert "file:///push/binaries/here" in reindex_step
diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
index b5e57e3e84..905901ff29 100644
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -3,6 +3,12 @@ stages: [ "generate", "build", "publish" ]
 variables:
   SPACK_DISABLE_LOCAL_CONFIG: "1"
   SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/tmp/_user_cache/"
+  # PR_MIRROR_FETCH_DOMAIN: "https://binaries-prs.spack.io"
+  PR_MIRROR_FETCH_DOMAIN: "s3://spack-binaries-prs"
+  PR_MIRROR_PUSH_DOMAIN: "s3://spack-binaries-prs"
+  # PROTECTED_MIRROR_FETCH_DOMAIN: "https://binaries.spack.io"
+  PROTECTED_MIRROR_FETCH_DOMAIN: "s3://spack-binaries"
+  PROTECTED_MIRROR_PUSH_DOMAIN: "s3://spack-binaries"
 
 default:
   image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
@@ -68,7 +74,9 @@ default:
 ########################################
 .base-job:
   variables:
-    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+    PIPELINE_MIRROR_TEMPLATE: "single-src-protected-mirrors.yaml.in"
+    # TODO: We can remove this when we drop the "deprecated" stack
+    PUSH_BUILDCACHE_DEPRECATED: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
 
   rules:
     - if: $CI_COMMIT_REF_NAME == "develop"
@@ -76,7 +84,7 @@ default:
       when: always
       variables:
         SPACK_PIPELINE_TYPE: "spack_protected_branch"
-        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+        SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
         SPACK_REQUIRE_SIGNING: "True"
         AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
         AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
@@ -86,7 +94,7 @@ default:
       when: always
       variables:
         SPACK_PIPELINE_TYPE: "spack_protected_branch"
-        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+        SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
         SPACK_PRUNE_UNTOUCHED: "False"
         SPACK_PRUNE_UP_TO_DATE: "False"
         SPACK_REQUIRE_SIGNING: "True"
@@ -98,8 +106,8 @@ default:
       when: always
       variables:
         SPACK_PIPELINE_TYPE: "spack_copy_only"
-        SPACK_SOURCE_MIRROR: "s3://spack-binaries/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}"
-        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+        SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
+        PIPELINE_MIRROR_TEMPLATE: "copy-only-protected-mirrors.yaml.in"
         AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
         AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
         OIDC_TOKEN_AUDIENCE: "protected_binary_mirror"
@@ -108,9 +116,16 @@ default:
       when: always
       variables:
         SPACK_PIPELINE_TYPE: "spack_pull_request"
-        SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+        # TODO: We can remove this when we drop the "deprecated" stack
+        PUSH_BUILDCACHE_DEPRECATED: "${PR_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
         SPACK_PRUNE_UNTOUCHED: "True"
         SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
+        # TODO: Change sync script to include target in branch name.  Then we could
+        # TODO: have multiple types of "PR" pipeline here.  It would be better if we could
+        # TODO: keep just this one and use a regex to capture the target branch, but so
+        # TODO: far gitlab doesn't support that.
+        PR_TARGET_REF_NAME: "develop"
+        PIPELINE_MIRROR_TEMPLATE: "multi-src-mirrors.yaml.in"
         AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
         AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
         OIDC_TOKEN_AUDIENCE: "pr_binary_mirror"
@@ -126,13 +141,15 @@ default:
     - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
     - spack env activate --without-view .
     - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
+    - spack python -c "import os,sys; print(os.path.expandvars(sys.stdin.read()))"
+      < "${SPACK_CI_CONFIG_ROOT}/${PIPELINE_MIRROR_TEMPLATE}" > "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
+    - spack config add -f "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
     - spack
       --config-scope "${SPACK_CI_CONFIG_ROOT}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
       ${CI_STACK_CONFIG_SCOPES}
       ci generate --check-index-only
-      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
       --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
       --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
   after_script:
@@ -182,7 +199,7 @@ default:
     - spack env activate --without-view .
     - spack
       ci generate --check-index-only
-      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
+      --buildcache-destination "${PUSH_BUILDCACHE_DEPRECATED}"
       --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
       --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
   after_script:
@@ -219,8 +236,7 @@ protected-publish:
     max: 2
     when: ["runner_system_failure", "stuck_or_timeout_failure"]
   variables:
-    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
-    SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+    SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
     SPACK_PIPELINE_TYPE: "spack_protected_branch"
     AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
     AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
@@ -253,11 +269,6 @@ protected-publish:
 # you should inlclude your custom definitions at the end of the of the
 # extends list.
 #
-# Also note that if extending .base-job, the mirror url given in your
-# spack.yaml should take the form:
-#
-#     s3://spack-binaries/develop/${SPACK_CI_STACK_NAME}
-#
 ########################################
 # My Super Cool Pipeline
 ########################################
diff --git a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
index 5f7e904ba5..9aad850b5d 100644
--- a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
@@ -21,7 +21,8 @@ ci:
         - k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust $k
         - k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k
       script::
-      - - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+      - - spack config blame mirrors
+        - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
       - - spack python ${CI_PROJECT_DIR}/share/spack/gitlab/cloud_pipelines/scripts/common/aggregate_package_logs.spack.py
           --prefix /home/software/spack:${CI_PROJECT_DIR}
           --log install_times.json
@@ -40,10 +41,10 @@ ci:
       image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
       tags: ["aws"]
       script:
-      - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
+      - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_BUILDCACHE_DESTINATION}/build_cache /tmp
         - /sign.sh
-        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
-        - aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
+        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_BUILDCACHE_DESTINATION}/build_cache
+        - aws s3 cp /tmp/public_keys ${SPACK_BUILDCACHE_DESTINATION}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
       id_tokens:
         GITLAB_OIDC_TOKEN:
           aud: "${OIDC_TOKEN_AUDIENCE}"
@@ -54,14 +55,14 @@ ci:
       before_script:
       - - if [[ $CI_COMMIT_TAG == "v"* ]]; then export SPACK_REPLACE_VERSION=$(echo "$CI_COMMIT_TAG" | sed 's/\(v[[:digit:]]\+\.[[:digit:]]\+\).*/releases\/\1/'); fi
         - if [[ $CI_COMMIT_TAG == "develop-"* ]]; then export SPACK_REPLACE_VERSION=develop; fi
-        - export SPACK_BUILDCACHE_SOURCE=${SPACK_SOURCE_MIRROR//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}}
+        - export SPACK_COPY_ONLY_SOURCE=${SPACK_BUILDCACHE_SOURCE//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}}
       script:
       - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
-        - echo Copying environment specs from ${SRC_MIRROR} to ${SPACK_BUILDCACHE_DESTINATION}
-        - spack buildcache sync "${SPACK_BUILDCACHE_SOURCE}" "${SPACK_BUILDCACHE_DESTINATION}"
+        - echo Copying environment specs from ${SPACK_COPY_ONLY_SOURCE} to ${SPACK_COPY_ONLY_DESTINATION}
+        - spack buildcache sync "${SPACK_COPY_ONLY_SOURCE}" "${SPACK_COPY_ONLY_DESTINATION}"
         - curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub
-        - aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_BUILDCACHE_DESTINATION}/build_cache/_pgp/spack-public-binary-key.pub"
-        - spack buildcache update-index --keys "${SPACK_BUILDCACHE_DESTINATION}"
+        - aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_COPY_ONLY_DESTINATION}/build_cache/_pgp/spack-public-binary-key.pub"
+        - spack buildcache update-index --keys "${SPACK_COPY_ONLY_DESTINATION}"
       when: "always"
       retry:
         max: 2
@@ -89,6 +90,7 @@ ci:
         GITLAB_OIDC_TOKEN:
           aud: "${OIDC_TOKEN_AUDIENCE}"
 
+  # TODO: Remove this block in Spack 0.23
   - cleanup-job:
       tags: ["service"]
       variables:
diff --git a/share/spack/gitlab/cloud_pipelines/configs/copy-only-protected-mirrors.yaml.in b/share/spack/gitlab/cloud_pipelines/configs/copy-only-protected-mirrors.yaml.in
new file mode 100644
index 0000000000..39e5c733b2
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/copy-only-protected-mirrors.yaml.in
@@ -0,0 +1,11 @@
+mirrors:
+  buildcache-source:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
+  buildcache-destination:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
diff --git a/share/spack/gitlab/cloud_pipelines/configs/multi-src-mirrors.yaml.in b/share/spack/gitlab/cloud_pipelines/configs/multi-src-mirrors.yaml.in
new file mode 100644
index 0000000000..0ad46d5fc9
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/multi-src-mirrors.yaml.in
@@ -0,0 +1,16 @@
+mirrors:
+  buildcache-source:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/${PR_TARGET_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/${PR_TARGET_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
+  buildcache-destination:
+    fetch: ${PR_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PR_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
+  buildcache-shared:
+    fetch: ${PR_MIRROR_FETCH_DOMAIN}/shared_pr_mirror/${SPACK_CI_STACK_NAME}
+    push: ${PR_MIRROR_PUSH_DOMAIN}/shared_pr_mirror/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
diff --git a/share/spack/gitlab/cloud_pipelines/configs/single-src-pr-mirrors.yaml.in b/share/spack/gitlab/cloud_pipelines/configs/single-src-pr-mirrors.yaml.in
new file mode 100644
index 0000000000..0a2775a4a2
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/single-src-pr-mirrors.yaml.in
@@ -0,0 +1,6 @@
+mirrors:
+  buildcache-destination:
+    fetch: ${PR_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PR_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
diff --git a/share/spack/gitlab/cloud_pipelines/configs/single-src-protected-mirrors.yaml.in b/share/spack/gitlab/cloud_pipelines/configs/single-src-protected-mirrors.yaml.in
new file mode 100644
index 0000000000..a55cd72737
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/single-src-protected-mirrors.yaml.in
@@ -0,0 +1,6 @@
+mirrors:
+  buildcache-destination:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
index 1c4e2de308..abd8f4d024 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
@@ -131,9 +131,6 @@ spack:
     - - $compiler
     - - $target
 
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-isc-aarch64" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
index 0a898d1a75..038761ac18 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
@@ -142,9 +142,6 @@ spack:
     - - $compiler
     - - $target
 
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-isc" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml
index 5ce6d1c869..85cf766068 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml
@@ -30,8 +30,6 @@ spack:
   - $optimized_configs
   # - $optimized_libs
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-icelake" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml
index 5708338a2b..50ba40992a 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml
@@ -30,9 +30,6 @@ spack:
   - $optimized_configs
   - $optimized_libs
 
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-neoverse_n1" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
index f2df769610..50ba40992a 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
@@ -30,9 +30,6 @@ spack:
   - $optimized_configs
   - $optimized_libs
 
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-neoverse_v1" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml
index 029dd67351..85cf766068 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml
@@ -30,8 +30,6 @@ spack:
   - $optimized_configs
   # - $optimized_libs
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-skylake" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
index 78a3ea785c..d154894830 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
@@ -21,7 +21,5 @@ spack:
         - - $default_specs
         - - $arch
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/build_systems" }
-
   cdash:
     build-group: Build Systems
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
index 9963f4b777..bf298d606d 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
@@ -58,8 +58,6 @@ spack:
     - ["~paraview +visit"]
     - [$^visit_specs]
 
-  mirrors: {mirror: s3://spack-binaries/develop/data-vis-sdk}
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
index 83e3d9f290..413fdf34eb 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
@@ -172,7 +172,5 @@ spack:
   # - variorum            # variorum: /opt/cray/pe/cce/15.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld: /opt/cray/pe/lib64/libpals.so.0: undefined reference to `json_array_append_new@@libjansson.so.4'
   # - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos~shylu # openblas: ftn-2307 ftn: ERROR in command line: The "-m" option must be followed by 0, 1, 2, 3 or 4.; make[2]: *** [<builtin>: spotrf2.o] Error 1; make[1]: *** [Makefile:27: lapacklib] Error 2; make: *** [Makefile:250: netlib] Error 2
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-cray-rhel" }
-
   cdash:
     build-group: E4S Cray
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
index dace63659e..c141cd9bf9 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
@@ -171,7 +171,5 @@ spack:
   # - variorum
   # - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos~shylu
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-cray-sles" }
-
   cdash:
     build-group: E4S Cray SLES
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml
index b4e8114df6..db903c15c4 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml
@@ -340,8 +340,6 @@ spack:
   # - tasmanian +cuda cuda_arch=90  # tasmanian: conflicts with cuda@12
   # - upcxx +cuda cuda_arch=90      # upcxx: needs NVIDIA driver
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-arm-neoverse_v1" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
index ec86c35b33..d170b0a272 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
@@ -236,8 +236,6 @@ spack:
 
   - py-scipy
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-oneapi" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
index 72e06b060d..5f8f3d0e62 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
@@ -251,8 +251,6 @@ spack:
   # - trilinos +cuda cuda_arch=70           # trilinos: https://github.com/trilinos/Trilinos/issues/11630
   # - upcxx +cuda cuda_arch=70              # upcxx: needs NVIDIA driver
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-power" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml
index cd9addbef0..885dbb538b 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml
@@ -335,8 +335,6 @@ spack:
   # - lbann ~cuda +rocm amdgpu_target=gfx90a    # aluminum: https://github.com/spack/spack/issues/38807
   # - papi +rocm amdgpu_target=gfx90a           # papi: https://github.com/spack/spack/issues/27898
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-rocm-external" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
index 65ab32e80d..410a379ee0 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
@@ -382,8 +382,6 @@ spack:
   # - lbann ~cuda +rocm amdgpu_target=gfx90a    # aluminum: https://github.com/spack/spack/issues/38807
   # - papi +rocm amdgpu_target=gfx90a           # papi: https://github.com/spack/spack/issues/27898
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml
index 69a8859745..263d8e29b3 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml
@@ -49,8 +49,6 @@ spack:
   # FAILURES
   # - kokkos +wrapper +cuda cuda_arch=80 ^cuda@12.0.0     # https://github.com/spack/spack/issues/35378
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/gpu-tests" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
index c137b138ee..6d8a0b7491 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
@@ -82,8 +82,6 @@ spack:
   # - r-xgboost
   - xgboost
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/ml-darwin-aarch64-mps" }
-
   ci:
     pipeline-gen:
     - build-job-remove:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
index fa7bf02755..71670d5a91 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
@@ -76,9 +76,6 @@ spack:
     # - r-xgboost
     - xgboost
 
-  mirrors:
-    mirror: s3://spack-binaries/develop/ml-linux-x86_64-cpu
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
index 5a24d42f23..8829169038 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
@@ -79,9 +79,6 @@ spack:
     # - r-xgboost
     - xgboost
 
-  mirrors:
-    mirror: s3://spack-binaries/develop/ml-linux-x86_64-cuda
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml
index e49d43db3d..620a95715b 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml
@@ -82,9 +82,6 @@ spack:
     # - r-xgboost
     - xgboost
 
-  mirrors:
-    mirror: s3://spack-binaries/develop/ml-linux-x86_64-rocm
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml
index b05b45f763..6453d2a5fe 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml
@@ -38,8 +38,6 @@ spack:
     - - $compiler
     - - $target
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/radiuss-aws-aarch64" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml
index fd297ede91..ca7de563c4 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml
@@ -28,7 +28,7 @@ spack:
     - mfem +cuda ^hypre+cuda
     - raja
     - raja +cuda
-    - umpire 
+    - umpire
     - umpire +cuda
 
   - compiler:
@@ -44,8 +44,6 @@ spack:
     - - $compiler
     - - $target
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/radiuss-aws" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
index c80bcf10ee..ca8e1a9905 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
@@ -40,9 +40,6 @@ spack:
       - xbraid
       - zfp
 
-  mirrors:
-    mirror: "s3://spack-binaries/develop/radiuss"
-
   specs:
   - matrix:
     - [$radiuss]
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
index 4b39be8846..0bc36ce8e4 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
@@ -49,8 +49,6 @@ spack:
   - $clang_packages
   - $gcc_spack_built_packages
 
-  mirrors:
-    mirror: s3://spack-binaries/develop/tutorial
   ci:
     pipeline-gen:
     - build-job: