ci: Support secure binary signing on protected pipelines (#30753)

This PR supports the creation of securely signed binaries built from spack
develop as well as release branches and tags. Specifically:

- remove internal pr mirror url generation logic in favor of buildcache destination
on command line
    - with a single mirror url specified in the spack.yaml, this makes it clearer where 
    binaries from various pipelines are pushed
- designate some tags as reserved: ['public', 'protected', 'notary']
    - these tags are stripped from all jobs by default and provisioned internally
    based on pipeline type
- update gitlab ci yaml to include pipelines on more protected branches than just
develop (so include releases and tags)
    - binaries from all protected pipelines are pushed into mirrors including the
    branch name so releases, tags, and develop binaries are kept separate
- update rebuild jobs running on protected pipelines to run on special runners
provisioned with an intermediate signing key
    - protected rebuild jobs no longer use "SPACK_SIGNING_KEY" env var to
    obtain signing key (in fact, final signing key is nowhere available to rebuild jobs)
    - these intermediate signatures are verified at the end of each pipeline by a new
    signing job to ensure binaries were produced by a protected pipeline
- optionallly schedule a signing/notary job at the end of the pipeline to sign all
packges in the mirror
    - add signing-job-attributes to gitlab-ci section of spack environment to allow
    configuration
    - signing job runs on special runner (separate from protected rebuild runners)
    provisioned with public intermediate key and secret signing key

8 files changed, 212 insertions, 94 deletions

diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
index 321c2b6af6..b0936dfc4e 100644
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -1,4 +1,4 @@
-stages: [ "generate", "build" ]
+stages: [ "generate", "build", "publish" ]
 
 default:
   image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
@@ -9,16 +9,25 @@ default:
 
 .pr:
   only:
+  - /^pr[\d]+_.*$/
   - /^github\/pr[\d]+_.*$/
   variables:
-    SPACK_PR_BRANCH: ${CI_COMMIT_REF_NAME}
+    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}"
     SPACK_PIPELINE_TYPE: "spack_pull_request"
     SPACK_PRUNE_UNTOUCHED: "True"
 
-.develop:
+.protected-refs:
   only:
+  - /^develop$/
+  - /^releases\/v.*/
+  - /^v.*/
   - /^github\/develop$/
+
+.protected:
+  extends: [ ".protected-refs" ]
   variables:
+    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+    SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
     SPACK_PIPELINE_TYPE: "spack_protected_branch"
 
 .generate:
@@ -29,12 +38,13 @@ default:
     - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
     - spack env activate --without-view .
     - spack ci generate --check-index-only
+      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
       --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
       --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
   artifacts:
     paths:
       - "${CI_PROJECT_DIR}/jobs_scratch_dir"
-  tags: ["spack", "public", "medium", "x86_64"]
+  tags: ["spack", "aws", "public", "medium", "x86_64"]
   interruptible: true
   retry:
     max: 2
@@ -45,8 +55,8 @@ default:
 .pr-generate:
   extends: [ ".pr", ".generate" ]
 
-.develop-generate:
-  extends: [ ".develop", ".generate" ]
+.protected-generate:
+  extends: [ ".protected", ".generate" ]
 
 .build:
   stage: build
@@ -57,12 +67,24 @@ default:
     AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
     AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
 
-.develop-build:
-  extends: [ ".develop", ".build" ]
+.protected-build:
+  extends: [ ".protected", ".build" ]
   variables:
     AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
     AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
-    SPACK_SIGNING_KEY: ${PACKAGE_SIGNING_KEY}
+
+protected-publish:
+  stage: publish
+  extends: [ ".protected-refs" ]
+  image: "ghcr.io/spack/python-aws-bash:0.0.1"
+  tags: ["spack", "public", "medium", "aws", "x86_64"]
+  variables:
+    AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
+    AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
+  script:
+    - . "./share/spack/setup-env.sh"
+    - spack --version
+    - spack buildcache update-index --mirror-url "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
 
 ########################################
 # TEMPLATE FOR ADDING ANOTHER PIPELINE
@@ -83,8 +105,8 @@ default:
 # my-super-cool-stack-pr-generate:
 #   extends: [ ".my-super-cool-stack", ".pr-generate"]
 #
-# my-super-cool-stack-develop-generate:
-#   extends: [ ".my-super-cool-stack", ".develop-generate"]
+# my-super-cool-stack-protected-generate:
+#   extends: [ ".my-super-cool-stack", ".protected-generate"]
 #
 # my-super-cool-stack-pr-build:
 #   extends: [ ".my-super-cool-stack", ".pr-build" ]
@@ -94,24 +116,62 @@ default:
 #         job: my-super-cool-stack-pr-generate
 #     strategy: depend
 #
-# my-super-cool-stack-develop-build:
-#   extends: [ ".my-super-cool-stack", ".develop-build" ]
+# my-super-cool-stack-protected-build:
+#   extends: [ ".my-super-cool-stack", ".protected-build" ]
 #   trigger:
 #     include:
 #       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-#         job: my-super-cool-stack-develop-generate
+#         job: my-super-cool-stack-protected-generate
 #     strategy: depend
 
 ########################################
-# E4S Mac Stack
+#          E4S Mac Stack
+#
+# With no near-future plans to have
+# protected aws runners running mac
+# builds, it seems best to decouple
+# them from the rest of the stacks for
+# the time being.  This way they can
+# still run on UO runners and be signed
+# using the previous approach.
 ########################################
 .e4s-mac:
   variables:
     SPACK_CI_STACK_NAME: e4s-mac
   allow_failure: True
 
+.mac-pr:
+  only:
+  - /^pr[\d]+_.*$/
+  - /^github\/pr[\d]+_.*$/
+  variables:
+    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}"
+    SPACK_PRUNE_UNTOUCHED: "True"
+
+.mac-protected:
+  only:
+  - /^develop$/
+  - /^releases\/v.*/
+  - /^v.*/
+  - /^github\/develop$/
+  variables:
+    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+
+.mac-pr-build:
+  extends: [ ".mac-pr", ".build" ]
+  variables:
+    AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
+    AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
+
+.mac-protected-build:
+  extends: [ ".mac-protected", ".build" ]
+  variables:
+    AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
+    AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
+    SPACK_SIGNING_KEY: ${PACKAGE_SIGNING_KEY}
+
 e4s-mac-pr-generate:
-  extends: [".e4s-mac", ".pr"]
+  extends: [".e4s-mac", ".mac-pr"]
   stage: generate
   script:
     - tmp="$(mktemp -d)"; export SPACK_USER_CONFIG_PATH="$tmp"; export SPACK_USER_CACHE_PATH="$tmp"
@@ -135,8 +195,8 @@ e4s-mac-pr-generate:
       - stuck_or_timeout_failure
   timeout: 60 minutes
 
-e4s-mac-develop-generate:
-  extends: [".e4s-mac", ".develop"]
+e4s-mac-protected-generate:
+  extends: [".e4s-mac", ".mac-protected"]
   stage: generate
   script:
     - tmp="$(mktemp -d)"; export SPACK_USER_CONFIG_PATH="$tmp"; export SPACK_USER_CACHE_PATH="$tmp"
@@ -161,7 +221,7 @@ e4s-mac-develop-generate:
   timeout: 60 minutes
 
 e4s-mac-pr-build:
-  extends: [ ".e4s-mac", ".pr-build" ]
+  extends: [ ".e4s-mac", ".mac-pr-build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
@@ -171,16 +231,16 @@ e4s-mac-pr-build:
     - artifacts: True
       job: e4s-mac-pr-generate
 
-e4s-mac-develop-build:
-  extends: [ ".e4s-mac", ".develop-build" ]
+e4s-mac-protected-build:
+  extends: [ ".e4s-mac", ".mac-protected-build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: e4s-mac-develop-generate
+        job: e4s-mac-protected-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: e4s-mac-develop-generate
+      job: e4s-mac-protected-generate
 
 ########################################
 # E4S pipeline
@@ -192,8 +252,8 @@ e4s-mac-develop-build:
 e4s-pr-generate:
   extends: [ ".e4s", ".pr-generate"]
 
-e4s-develop-generate:
-  extends: [ ".e4s", ".develop-generate"]
+e4s-protected-generate:
+  extends: [ ".e4s", ".protected-generate"]
 
 e4s-pr-build:
   extends: [ ".e4s", ".pr-build" ]
@@ -206,16 +266,16 @@ e4s-pr-build:
     - artifacts: True
       job: e4s-pr-generate
 
-e4s-develop-build:
-  extends: [ ".e4s", ".develop-build" ]
+e4s-protected-build:
+  extends: [ ".e4s", ".protected-build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: e4s-develop-generate
+        job: e4s-protected-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: e4s-develop-generate
+      job: e4s-protected-generate
 
 ########################################
 # E4S on Power
@@ -231,8 +291,8 @@ e4s-develop-build:
 # e4s-on-power-pr-generate:
 #   extends: [ ".e4s-on-power", ".pr-generate", ".power-e4s-generate-tags-and-image"]
 
-# e4s-on-power-develop-generate:
-#   extends: [ ".e4s-on-power", ".develop-generate", ".power-e4s-generate-tags-and-image"]
+# e4s-on-power-protected-generate:
+#   extends: [ ".e4s-on-power", ".protected-generate", ".power-e4s-generate-tags-and-image"]
 
 # e4s-on-power-pr-build:
 #   extends: [ ".e4s-on-power", ".pr-build" ]
@@ -245,16 +305,16 @@ e4s-develop-build:
 #     - artifacts: True
 #       job: e4s-on-power-pr-generate
 
-# e4s-on-power-develop-build:
-#   extends: [ ".e4s-on-power", ".develop-build" ]
+# e4s-on-power-protected-build:
+#   extends: [ ".e4s-on-power", ".protected-build" ]
 #   trigger:
 #     include:
 #       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-#         job: e4s-on-power-develop-generate
+#         job: e4s-on-power-protected-generate
 #     strategy: depend
 #   needs:
 #     - artifacts: True
-#       job: e4s-on-power-develop-generate
+#       job: e4s-on-power-protected-generate
 
 #########################################
 # Build tests for different build-systems
@@ -266,8 +326,8 @@ e4s-develop-build:
 build_systems-pr-generate:
   extends: [ ".build_systems", ".pr-generate"]
 
-build_systems-develop-generate:
-  extends: [ ".build_systems", ".develop-generate"]
+build_systems-protected-generate:
+  extends: [ ".build_systems", ".protected-generate"]
 
 build_systems-pr-build:
   extends: [ ".build_systems", ".pr-build" ]
@@ -280,16 +340,16 @@ build_systems-pr-build:
     - artifacts: True
       job: build_systems-pr-generate
 
-build_systems-develop-build:
-  extends: [ ".build_systems", ".develop-build" ]
+build_systems-protected-build:
+  extends: [ ".build_systems", ".protected-build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: build_systems-develop-generate
+        job: build_systems-protected-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: build_systems-develop-generate
+      job: build_systems-protected-generate
 
 #########################################
 # RADIUSS
@@ -313,20 +373,20 @@ radiuss-pr-build:
     - artifacts: True
       job: radiuss-pr-generate
 
-# ---------  Develop ---------
-radiuss-develop-generate:
-  extends: [ ".radiuss", ".develop-generate" ]
+# --------- Protected ---------
+radiuss-protected-generate:
+  extends: [ ".radiuss", ".protected-generate" ]
 
-radiuss-develop-build:
-  extends: [ ".radiuss", ".develop-build" ]
+radiuss-protected-build:
+  extends: [ ".radiuss", ".protected-build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: radiuss-develop-generate
+        job: radiuss-protected-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: radiuss-develop-generate
+      job: radiuss-protected-generate
 
 ########################################
 # ECP Data & Vis SDK
@@ -338,8 +398,8 @@ radiuss-develop-build:
 data-vis-sdk-pr-generate:
   extends: [ ".data-vis-sdk", ".pr-generate"]
 
-data-vis-sdk-develop-generate:
-  extends: [ ".data-vis-sdk", ".develop-generate"]
+data-vis-sdk-protected-generate:
+  extends: [ ".data-vis-sdk", ".protected-generate"]
 
 data-vis-sdk-pr-build:
   extends: [ ".data-vis-sdk", ".pr-build" ]
@@ -352,16 +412,16 @@ data-vis-sdk-pr-build:
     - artifacts: True
       job: data-vis-sdk-pr-generate
 
-data-vis-sdk-develop-build:
-  extends: [ ".data-vis-sdk", ".develop-build" ]
+data-vis-sdk-protected-build:
+  extends: [ ".data-vis-sdk", ".protected-build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: data-vis-sdk-develop-generate
+        job: data-vis-sdk-protected-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: data-vis-sdk-develop-generate
+      job: data-vis-sdk-protected-generate
 
 ########################################
 # Spack Tutorial
@@ -373,8 +433,8 @@ data-vis-sdk-develop-build:
 tutorial-pr-generate:
   extends: [ ".tutorial", ".pr-generate"]
 
-tutorial-develop-generate:
-  extends: [ ".tutorial", ".develop-generate"]
+tutorial-protected-generate:
+  extends: [ ".tutorial", ".protected-generate"]
 
 tutorial-pr-build:
   extends: [ ".tutorial", ".pr-build" ]
@@ -387,13 +447,13 @@ tutorial-pr-build:
     - artifacts: True
       job: tutorial-pr-generate
 
-tutorial-develop-build:
-  extends: [ ".tutorial", ".develop-build" ]
+tutorial-protected-build:
+  extends: [ ".tutorial", ".protected-build" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: tutorial-develop-generate
+        job: tutorial-protected-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: tutorial-develop-generate
+      job: tutorial-protected-generate
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
index 5a742718d4..a633d10099 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
@@ -29,7 +29,7 @@ spack:
         - - $default_specs
         - - $arch
 
-  mirrors: { "mirror": "s3://spack-binaries/build_systems" }
+  mirrors: { "mirror": "s3://spack-binaries/develop/build_systems" }
 
   gitlab-ci:
     script:
@@ -38,6 +38,8 @@ spack:
       - cd ${SPACK_CONCRETE_ENV_DIR}
       - spack env activate --without-view .
       - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
+      - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
+      - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
       - spack -d ci rebuild
 
     image:
@@ -48,7 +50,7 @@ spack:
       - match:
           - cmake
         runner-attributes:
-          tags: [ "spack", "public", "large", "x86_64"]
+          tags: [ "spack", "large", "x86_64"]
           variables:
             CI_JOB_SIZE: large
             KUBERNETES_CPU_REQUEST: 8000m
@@ -61,7 +63,7 @@ spack:
           - openjpeg
           - sqlite
         runner-attributes:
-          tags: [ "spack", "public", "medium", "x86_64" ]
+          tags: [ "spack", "medium", "x86_64" ]
           variables:
             CI_JOB_SIZE: "medium"
             KUBERNETES_CPU_REQUEST: "2000m"
@@ -85,7 +87,7 @@ spack:
           - xz
           - zlib
         runner-attributes:
-          tags: [ "spack", "public", "medium", "x86_64" ]
+          tags: [ "spack", "medium", "x86_64" ]
           variables:
             CI_JOB_SIZE: "small"
             KUBERNETES_CPU_REQUEST: "500m"
@@ -94,18 +96,27 @@ spack:
       - match:
           - 'os=ubuntu18.04'
         runner-attributes:
-          tags: ["spack", "public", "x86_64"]
+          tags: ["spack", "x86_64"]
           variables:
             CI_JOB_SIZE: "default"
 
+    broken-specs-url: "s3://spack-binaries/broken-specs"
 
-    broken-specs-url: "s3://spack-binaries-develop/broken-specs"
     service-job-attributes:
       before_script:
         - . "./share/spack/setup-env.sh"
         - spack --version
       image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
       tags: ["spack", "public", "x86_64"]
+
+    signing-job-attributes:
+      image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
+      tags: ["spack", "aws"]
+      script:
+        - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
+        - /sign.sh
+        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
+
   cdash:
     build-group: Build tests for different build systems
     url: https://cdash.spack.io
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
index 85095e5619..ae9025e366 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
@@ -42,7 +42,7 @@ spack:
         +zfp
         +visit
 
-  mirrors: { "mirror": "s3://spack-binaries/data-vis-sdk" }
+  mirrors: { "mirror": "s3://spack-binaries/develop/data-vis-sdk" }
 
   gitlab-ci:
     image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
@@ -52,13 +52,15 @@ spack:
       - cd ${SPACK_CONCRETE_ENV_DIR}
       - spack env activate --without-view .
       - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
+      - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
+      - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
       - spack -d ci rebuild
     mappings:
       - match:
           - llvm
           - qt
         runner-attributes:
-          tags: [ "spack", "public", "huge", "x86_64" ]
+          tags: [ "spack", "huge", "x86_64" ]
           variables:
             CI_JOB_SIZE: huge
             KUBERNETES_CPU_REQUEST: 11000m
@@ -72,7 +74,7 @@ spack:
             - visit
             - vtk-m
         runner-attributes:
-          tags: [ "spack", "public", "large", "x86_64" ]
+          tags: [ "spack", "large", "x86_64" ]
           variables:
             CI_JOB_SIZE: large
             KUBERNETES_CPU_REQUEST: 8000m
@@ -98,7 +100,7 @@ spack:
             - raja
             - vtk-h
         runner-attributes:
-          tags: [ "spack", "public", "medium", "x86_64" ]
+          tags: [ "spack", "medium", "x86_64" ]
           variables:
             CI_JOB_SIZE: "medium"
             KUBERNETES_CPU_REQUEST: "2000m"
@@ -133,7 +135,7 @@ spack:
             - util-linux-uuid
 
         runner-attributes:
-          tags: [ "spack", "public", "small", "x86_64" ]
+          tags: [ "spack", "small", "x86_64" ]
           variables:
             CI_JOB_SIZE: "small"
             KUBERNETES_CPU_REQUEST: "500m"
@@ -141,11 +143,12 @@ spack:
 
       - match: ['@:']
         runner-attributes:
-          tags: ["spack", "public", "x86_64"]
+          tags: ["spack", "x86_64"]
           variables:
             CI_JOB_SIZE: "default"
 
-    broken-specs-url: "s3://spack-binaries-develop/broken-specs"
+    broken-specs-url: "s3://spack-binaries/broken-specs"
+
     service-job-attributes:
       image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
       before_script:
@@ -153,6 +156,14 @@ spack:
         - spack --version
       tags: ["spack", "public", "medium", "x86_64"]
 
+    signing-job-attributes:
+      image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
+      tags: ["spack", "aws"]
+      script:
+        - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
+        - /sign.sh
+        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
+
   cdash:
     build-group: Data and Vis SDK
     url: https://cdash.spack.io
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml
index dc86cb7dbe..6cfaae4a15 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml
@@ -32,7 +32,7 @@ spack:
     - - $easy_specs
     - - $arch
 
-  mirrors: { "mirror": "s3://spack-binaries/e4s-mac" }
+  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-mac" }
 
   gitlab-ci:
 
@@ -51,7 +51,9 @@ spack:
         runner-attributes:
           tags:
           - omicron
-    broken-specs-url: "s3://spack-binaries-develop/broken-specs"
+
+    broken-specs-url: "s3://spack-binaries/broken-specs"
+
     service-job-attributes:
       before_script:
         - . "./share/spack/setup-env.sh"
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
index ce31a0156e..c4eaaddbae 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
@@ -222,7 +222,7 @@ spack:
     - - $cuda_specs
     - - $arch
 
-  mirrors: { "mirror": "s3://spack-binaries/e4s" }
+  mirrors: { "mirror": "s3://spack-binaries/develop/e4s" }
 
   gitlab-ci:
 
@@ -233,6 +233,8 @@ spack:
       - spack env activate --without-view .
       - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
       - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
+      - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
+      - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
       - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
 
     image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
@@ -240,7 +242,7 @@ spack:
       - match:
           - llvm
         runner-attributes:
-          tags: [ "spack", "public", "huge", "x86_64" ]
+          tags: [ "spack", "huge", "x86_64" ]
           variables:
             CI_JOB_SIZE: huge
             KUBERNETES_CPU_REQUEST: 11000m
@@ -265,7 +267,7 @@ spack:
           - vtk-m
           - warpx
         runner-attributes:
-          tags: [ "spack", "public", "large", "x86_64" ]
+          tags: [ "spack", "large", "x86_64" ]
           variables:
             CI_JOB_SIZE: large
             KUBERNETES_CPU_REQUEST: 8000m
@@ -333,7 +335,7 @@ spack:
           - vtk-h
           - zfp
         runner-attributes:
-          tags: [ "spack", "public", "medium", "x86_64" ]
+          tags: [ "spack", "medium", "x86_64" ]
           variables:
             CI_JOB_SIZE: "medium"
             KUBERNETES_CPU_REQUEST: "2000m"
@@ -394,7 +396,7 @@ spack:
           - zlib
           - zstd
         runner-attributes:
-          tags: [ "spack", "public", "small", "x86_64" ]
+          tags: [ "spack", "small", "x86_64" ]
           variables:
             CI_JOB_SIZE: "small"
             KUBERNETES_CPU_REQUEST: "500m"
@@ -402,11 +404,12 @@ spack:
 
       - match: ['os=ubuntu18.04']
         runner-attributes:
-          tags: ["spack", "public", "x86_64"]
+          tags: ["spack", "x86_64"]
           variables:
             CI_JOB_SIZE: "default"
 
-    broken-specs-url: "s3://spack-binaries-develop/broken-specs"
+    broken-specs-url: "s3://spack-binaries/broken-specs"
+
     service-job-attributes:
       before_script:
         - . "./share/spack/setup-env.sh"
@@ -414,6 +417,14 @@ spack:
       image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
       tags: ["spack", "public", "x86_64"]
 
+    signing-job-attributes:
+      image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
+      tags: ["spack", "aws"]
+      script:
+        - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
+        - /sign.sh
+        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
+
   cdash:
     build-group: New PR testing workflow
     url: https://cdash.spack.io
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
index 846b45f707..ebb15eb7ca 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
@@ -54,7 +54,7 @@ spack:
       - zfp
 
   mirrors:
-    mirror: "s3://spack-binaries/radiuss"
+    mirror: "s3://spack-binaries/develop/radiuss"
 
   specs:
   - matrix:
@@ -69,6 +69,8 @@ spack:
       - cd ${SPACK_CONCRETE_ENV_DIR}
       - spack env activate --without-view .
       - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
+      - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
+      - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
       - spack -d ci rebuild
     mappings:
       - match:
@@ -76,7 +78,7 @@ spack:
           - openblas
           - rust
         runner-attributes:
-          tags: ["spack", "public", "large", "x86_64"]
+          tags: ["spack", "large", "x86_64"]
           variables:
             CI_JOB_SIZE: large
             KUBERNETES_CPU_REQUEST: 8000m
@@ -96,7 +98,7 @@ spack:
           - vtk-h
           - vtk-m
         runner-attributes:
-          tags: ["spack", "public", "medium", "x86_64"]
+          tags: ["spack", "medium", "x86_64"]
           variables:
             CI_JOB_SIZE: "medium"
             KUBERNETES_CPU_REQUEST: "2000m"
@@ -150,7 +152,7 @@ spack:
           - zfp
           - zlib
         runner-attributes:
-          tags: ["spack", "public", "small", "x86_64"]
+          tags: ["spack", "small", "x86_64"]
           variables:
             CI_JOB_SIZE: "small"
             KUBERNETES_CPU_REQUEST: "500m"
@@ -158,10 +160,12 @@ spack:
 
       - match: ['os=ubuntu18.04']
         runner-attributes:
-          tags: ["spack", "public", "x86_64"]
+          tags: ["spack", "x86_64"]
           variables:
             CI_JOB_SIZE: "default"
 
+    broken-specs-url: "s3://spack-binaries/broken-specs"
+
     service-job-attributes:
       before_script:
         - . "./share/spack/setup-env.sh"
@@ -169,6 +173,14 @@ spack:
       image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
       tags: ["spack", "public", "x86_64"]
 
+    signing-job-attributes:
+      image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
+      tags: ["spack", "aws"]
+      script:
+        - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
+        - /sign.sh
+        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
+
   cdash:
     build-group: RADIUSS
     url: https://cdash.spack.io
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
index 14bd78b3b6..504b4209d2 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
@@ -59,7 +59,7 @@ spack:
   - $gcc_spack_built_packages
 
   mirrors:
-    mirror: 's3://spack-binaries/tutorial'
+    mirror: 's3://spack-binaries/develop/tutorial'
 
   gitlab-ci:
     script:
@@ -69,6 +69,8 @@ spack:
       - cd ${SPACK_CONCRETE_ENV_DIR}
       - spack env activate --without-view .
       - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
+      - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
+      - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
       - spack -d ci rebuild
 
     image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] }
@@ -81,7 +83,7 @@ spack:
           - netlib-lapack
           - trilinos
         runner-attributes:
-          tags: ["spack", "public", "large", "x86_64"]
+          tags: ["spack", "large", "x86_64"]
           variables:
             CI_JOB_SIZE: large
             KUBERNETES_CPU_REQUEST: 8000m
@@ -99,7 +101,7 @@ spack:
             - py-scipy
             - slurm
         runner-attributes:
-          tags: ["spack", "public", "medium", "x86_64"]
+          tags: ["spack", "medium", "x86_64"]
           variables:
             CI_JOB_SIZE: "medium"
             KUBERNETES_CPU_REQUEST: "2000m"
@@ -129,7 +131,7 @@ spack:
             - tar
             - util-linux-uuid
         runner-attributes:
-          tags: ["spack", "public", "small", "x86_64"]
+          tags: ["spack", "small", "x86_64"]
           variables:
             CI_JOB_SIZE: "small"
             KUBERNETES_CPU_REQUEST: "500m"
@@ -137,11 +139,12 @@ spack:
 
       - match: ['@:']
         runner-attributes:
-          tags: ["spack", "public", "x86_64"]
+          tags: ["spack", "x86_64"]
           variables:
             CI_JOB_SIZE: default
 
-    broken-specs-url: "s3://spack-binaries-develop/broken-specs"
+    broken-specs-url: "s3://spack-binaries/broken-specs"
+
     service-job-attributes:
       image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] }
       before_script:
@@ -149,6 +152,14 @@ spack:
         - spack --version
       tags: ["spack", "public", "x86_64"]
 
+    signing-job-attributes:
+      image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
+      tags: ["spack", "aws"]
+      script:
+        - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
+        - /sign.sh
+        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
+
   cdash:
     build-group: Spack Tutorial
     url: https://cdash.spack.io
diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash
index 5e4498f395..19508dd17c 100755
--- a/share/spack/spack-completion.bash
+++ b/share/spack/spack-completion.bash
@@ -626,7 +626,7 @@ _spack_ci() {
 }
 
 _spack_ci_generate() {
-    SPACK_COMPREPLY="-h --help --output-file --copy-to --optimize --dependencies --prune-dag --no-prune-dag --check-index-only --artifacts-root"
+    SPACK_COMPREPLY="-h --help --output-file --copy-to --optimize --dependencies --buildcache-destination --prune-dag --no-prune-dag --check-index-only --artifacts-root"
 }
 
 _spack_ci_rebuild_index() {