CI: remove redundant sections (#38514)

Refactor gitlab ci configs so that mac and cray jobs can reuse as much higher level
configuration as possible.

* CI: remove redundant sections
* CI: Include base linux CI configs in cray stacks

Relocation and runner mapping is consistent between cray and linux runners.

* Export user cache path in before script
* CI: add GPG root for mac runners
* Disable user configs

Metal runners share a ~ directory

* Disable user config and add configs in activate env

9 files changed, 70 insertions, 716 deletions

diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
index 4b927a04ae..358d3af121 100644
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -19,6 +19,11 @@ default:
     SPACK_TARGET_PLATFORM: "darwin"
     SPACK_TARGET_ARCH: "x86_64"
 
+.darwin_aarch64:
+  variables:
+    SPACK_TARGET_PLATFORM: "darwin"
+    SPACK_TARGET_ARCH: "aarch64"
+
 .linux_x86_64_v3:
   variables:
     SPACK_TARGET_PLATFORM: "linux"
@@ -60,6 +65,8 @@ default:
 .base-job:
   variables:
     SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+    SPACK_DISABLE_LOCAL_CONFIG: "1"
+    SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/tmp/_user_cache/"
   rules:
     - if: $CI_COMMIT_REF_NAME == "develop"
       # Pipelines on develop only rebuild what is missing from the mirror
@@ -101,7 +108,7 @@ default:
         AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
         AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
 
-.generate-base:
+.generate-common:
   stage: generate
   script:
     - uname -a || true
@@ -136,41 +143,17 @@ default:
     when:
       - always
 
-.generate:
-  extends: [ ".base-job", ".generate-base" ]
-  tags: ["spack", "public", "medium", "x86_64"]
-
-.darwin-generate-base:
-  stage: generate
-  script:
-  - export SPACK_DISABLE_LOCAL_CONFIG=1
-  - export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache
-  - uname -a || true
-  - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
-  - nproc || true
-  - . "./share/spack/setup-env.sh"
-  - spack --version
-  - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
-  - spack env activate --without-view .
-  - spack -d ci generate --check-index-only
-      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
-      --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
-      --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
-  after_script:
-  - cat /proc/loadavg || true
-  artifacts:
-    paths:
-    - "${CI_PROJECT_DIR}/jobs_scratch_dir"
-  interruptible: true
-  timeout: 60 minutes
-  retry:
-    max: 2
-    when:
-    - always
+# Generate without tags for cases using external runners
+.generate-base:
+  extends: [ ".base-job", ".generate-common" ]
 
-.darwin-generate:
-  extends: [ ".base-job", ".darwin-generate-base" ]
+.generate-x86_64:
+  extends: [ ".generate-base" ]
+  tags: ["spack", "public", "medium", "x86_64"]
 
+.generate-aarch64:
+  extends: [ ".generate-base" ]
+  tags: ["spack", "public", "medium", "aarch64"]
 
 .generate-deprecated:
   extends: [ ".base-job" ]
@@ -204,10 +187,6 @@ default:
       - always
   tags: ["spack", "public", "medium", "x86_64"]
 
-.generate-aarch64:
-  extends: [ ".base-job", ".generate" ]
-  tags: ["spack", "public", "medium", "aarch64"]
-
 .build:
   extends: [ ".base-job" ]
   stage: build
@@ -294,7 +273,7 @@ protected-publish:
     SPACK_CI_STACK_NAME: e4s
 
 e4s-generate:
-  extends: [ ".e4s", ".generate"]
+  extends: [ ".e4s", ".generate-x86_64"]
   image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
 
 e4s-build:
@@ -317,7 +296,7 @@ e4s-build:
     SPACK_CI_STACK_NAME: gpu-tests
 
 gpu-tests-generate:
-  extends: [ ".gpu-tests", ".generate"]
+  extends: [ ".gpu-tests", ".generate-x86_64"]
   image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
 
 gpu-tests-build:
@@ -340,7 +319,7 @@ gpu-tests-build:
     SPACK_CI_STACK_NAME: e4s-oneapi
 
 e4s-oneapi-generate:
-  extends: [ ".e4s-oneapi", ".generate"]
+  extends: [ ".e4s-oneapi", ".generate-x86_64"]
   image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023.07.21
 
 e4s-oneapi-build:
@@ -367,7 +346,7 @@ e4s-oneapi-build:
     SPACK_CI_STACK_NAME: e4s-power
 
 e4s-power-generate:
-  extends: [ ".e4s-power", ".generate", ".e4s-power-generate-tags-and-image"]
+  extends: [ ".e4s-power", ".generate-x86_64", ".e4s-power-generate-tags-and-image"]
 
 e4s-power-build:
   extends: [ ".e4s-power", ".build" ]
@@ -389,7 +368,7 @@ e4s-power-build:
     SPACK_CI_STACK_NAME: build_systems
 
 build_systems-generate:
-  extends: [ ".build_systems", ".generate"]
+  extends: [ ".build_systems", ".generate-x86_64"]
 
 build_systems-build:
   extends: [ ".build_systems", ".build" ]
@@ -411,7 +390,7 @@ build_systems-build:
     SPACK_CI_STACK_NAME: radiuss
 
 radiuss-generate:
-  extends: [ ".radiuss", ".generate" ]
+  extends: [ ".radiuss", ".generate-x86_64" ]
 
 radiuss-build:
   extends: [ ".radiuss", ".build" ]
@@ -443,7 +422,7 @@ radiuss-build:
     SPACK_CI_STACK_NAME: radiuss-aws
 
 radiuss-aws-generate:
-  extends: [ ".radiuss-aws", ".generate", ".radiuss-aws-overrides", ".tags-x86_64_v4" ]
+  extends: [ ".radiuss-aws", ".generate-x86_64", ".radiuss-aws-overrides", ".tags-x86_64_v4" ]
 
 radiuss-aws-build:
   extends: [ ".radiuss-aws", ".build" ]
@@ -487,7 +466,7 @@ radiuss-aws-aarch64-build:
     SPACK_CI_STACK_NAME: data-vis-sdk
 
 data-vis-sdk-generate:
-  extends: [ ".data-vis-sdk", ".generate"]
+  extends: [ ".data-vis-sdk", ".generate-x86_64"]
   image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
 
 data-vis-sdk-build:
@@ -517,7 +496,7 @@ data-vis-sdk-build:
     SPACK_CI_STACK_NAME: aws-isc
 
 aws-isc-generate:
-  extends: [ ".aws-isc", ".generate", ".aws-isc-overrides", ".tags-x86_64_v4" ]
+  extends: [ ".aws-isc", ".generate-x86_64", ".aws-isc-overrides", ".tags-x86_64_v4" ]
 
 aws-isc-build:
   extends: [ ".aws-isc", ".build" ]
@@ -561,7 +540,7 @@ aws-isc-aarch64-build:
     SPACK_CI_STACK_NAME: tutorial
 
 tutorial-generate:
-  extends: [ ".tutorial", ".generate"]
+  extends: [ ".tutorial", ".generate-x86_64"]
   image: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-05-07
 
 tutorial-build:
@@ -584,7 +563,7 @@ tutorial-build:
     SPACK_CI_STACK_NAME: ml-linux-x86_64-cpu
 
 ml-linux-x86_64-cpu-generate:
-  extends: [ ".generate", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ]
+  extends: [ ".generate-x86_64", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ]
   image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
 
 ml-linux-x86_64-cpu-build:
@@ -607,7 +586,7 @@ ml-linux-x86_64-cpu-build:
     SPACK_CI_STACK_NAME: ml-linux-x86_64-cuda
 
 ml-linux-x86_64-cuda-generate:
-  extends: [ ".generate", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ]
+  extends: [ ".generate-x86_64", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ]
   image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
 
 ml-linux-x86_64-cuda-build:
@@ -630,7 +609,7 @@ ml-linux-x86_64-cuda-build:
     SPACK_CI_STACK_NAME: ml-linux-x86_64-rocm
 
 ml-linux-x86_64-rocm-generate:
-  extends: [ ".generate", .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ]
+  extends: [ ".generate-x86_64", .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ]
   image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
 
 ml-linux-x86_64-rocm-build:
@@ -648,12 +627,13 @@ ml-linux-x86_64-rocm-build:
 # Machine Learning - Darwin aarch64 (MPS)
 ########################################
 .ml-darwin-aarch64-mps:
+  extends: [".darwin_aarch64"]
   variables:
     SPACK_CI_STACK_NAME: ml-darwin-aarch64-mps
 
 ml-darwin-aarch64-mps-generate:
   tags: [ "macos-ventura", "apple-clang-14", "aarch64-macos" ]
-  extends: [ ".ml-darwin-aarch64-mps", ".darwin-generate"]
+  extends: [ ".ml-darwin-aarch64-mps", ".generate-base"]
 
 ml-darwin-aarch64-mps-build:
   extends: [ ".ml-darwin-aarch64-mps", ".build" ]
@@ -717,7 +697,7 @@ deprecated-ci-build:
     SPACK_CI_STACK_NAME: aws-pcluster-icelake
 
 # aws-pcluster-generate-icelake:
-#   extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".generate", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
+#   extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".generate-x86_64", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
 
 # aws-pcluster-build-icelake:
 #   extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".build" ]
@@ -736,7 +716,7 @@ deprecated-ci-build:
     SPACK_CI_STACK_NAME: aws-pcluster-skylake
 
 # aws-pcluster-generate-skylake:
-#   extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".generate", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
+#   extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".generate-x86_64", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
 
 # aws-pcluster-build-skylake:
 #   extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".build" ]
@@ -810,42 +790,12 @@ deprecated-ci-build:
         SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
 
 .generate-cray:
-  extends: [ ".base-cray-job" ]
+  extends: [ ".generate-common", ".base-cray-job" ]
   stage: generate
-  script:
+  before_script:
     - echo $PATH
     - module avail
     - module list
-    - export SPACK_DISABLE_LOCAL_CONFIG=1
-    - export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache
-    - uname -a || true
-    - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
-    - nproc || true
-    - . "./share/spack/setup-env.sh"
-    - spack --version
-    - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
-    - spack env activate --without-view .
-    - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
-    - spack
-      --config-scope "${SPACK_CI_CONFIG_ROOT}"
-      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
-      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
-      ${CI_STACK_CONFIG_SCOPES}
-      ci generate --check-index-only
-      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
-      --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
-      --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
-  after_script:
-    - cat /proc/loadavg || true
-  artifacts:
-    paths:
-      - "${CI_PROJECT_DIR}/jobs_scratch_dir"
-  interruptible: true
-  timeout: 60 minutes
-  retry:
-    max: 2
-    when:
-      - always
 
 .generate-cray-rhel:
   tags: [ "cray-rhel-zen4", "public" ]
@@ -903,4 +853,4 @@ e4s-cray-sles-build:
     strategy: depend
   needs:
     - artifacts: True
-      job: e4s-cray-sles-generate
\ No newline at end of file
+      job: e4s-cray-sles-generate
diff --git a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
index 53bc5f63d7..1b3e723d5a 100644
--- a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
@@ -10,19 +10,17 @@ ci:
   - build-job:
       before_script-:
       - - spack list --count  # ensure that spack's cache is populated
-      script::
-      - - spack compiler find
-        - cd ${SPACK_CONCRETE_ENV_DIR}
-        - spack env activate --without-view .
+      - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
+        - spack compiler find
         - if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; fi
-        - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
-        - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
+      - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
         # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
-        - k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust $k
+      - - k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust $k
         # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification)
         - k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust $k
         - k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k
-        - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+      script::
+      - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
       after_script:
       - - cat /proc/loadavg || true
       variables:
@@ -47,8 +45,7 @@ ci:
         - if [[ $CI_COMMIT_TAG == "develop-"* ]]; then export SPACK_REPLACE_VERSION=develop; fi
         - export SPACK_BUILDCACHE_SOURCE=${SPACK_SOURCE_MIRROR//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}}
       script:
-      - - cd ${SPACK_CONCRETE_ENV_DIR}
-        - spack env activate --without-view .
+      - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
         - echo Copying environment specs from ${SRC_MIRROR} to ${SPACK_BUILDCACHE_DESTINATION}
         - spack buildcache sync "${SPACK_BUILDCACHE_SOURCE}" "${SPACK_BUILDCACHE_DESTINATION}"
         - curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub
@@ -89,12 +86,16 @@ ci:
         KUBERNETES_MEMORY_REQUEST: "500M"
 
   - any-job:
-      image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
       tags: ["spack"]
+      image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
+      variables:
+        # Disable local configs to avoid issues on shell runners
+        SPACK_DISABLE_LOCAL_CONFIG: "1"
       before_script:
+      - - export SPACK_USER_CACHE_PATH="${CI_PROJECT_DIR}/_user_cache/"
       - - uname -a || true
         - grep -E "vendor|model name" /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
-        - nproc
+        - nproc || true
       - - . "./share/spack/setup-env.sh"
         - spack --version
         - spack arch
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml
index 3df3e1cc7d..a74ef3f98f 100644
--- a/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml
@@ -5,287 +5,3 @@ ci:
   - build-job:
       variables:
         CI_GPG_KEY_ROOT: /etc/protected-runner
-  - match_behavior: first
-    submapping:
-    - match:
-      - hipblas
-      - llvm
-      - llvm-amdgpu
-      - pango
-      - paraview
-      - py-tensorflow
-      - py-torch
-      - qt
-      - rocblas
-      - visit
-      build-job:
-        tags: [ "spack", "huge" ]
-        variables:
-          CI_JOB_SIZE: huge
-          SPACK_BUILD_JOBS: "12"
-
-    - match:
-      - ascent
-      - atk
-      - axom
-      - cistem
-      - cmake
-      - ctffind
-      - cuda
-      - dealii
-      - dray
-      - dyninst
-      - ecp-data-vis-sdk
-      - gcc
-      - ginkgo
-      - hdf5
-      - hpx
-      - kokkos-kernels
-      - kokkos-nvcc-wrapper
-      - lbann
-      - magma
-      - mesa
-      - mfem
-      - mpich
-      - netlib-lapack
-      - nvhpc
-      - oce
-      - openblas
-      - openfoam
-      - openturns
-      - parallelio
-      - plumed
-      - precice
-      #- py-tensorflow
-      #- qt
-      - raja
-      - relion
-      #- rocblas
-      - rocfft
-      - rocsolver
-      - rocsparse
-      - rust
-      - slate
-      - strumpack
-      - sundials
-      - trilinos
-      - umpire
-      #- visit
-      - vtk
-      - vtk-h
-      - vtk-m
-      - warpx
-      - wrf
-      - wxwidgets
-      build-job:
-        tags: [ "spack", "large" ]
-        variables:
-          CI_JOB_SIZE: large
-          SPACK_BUILD_JOBS: "8"
-
-    - match:
-      - adios2
-      - amrex
-      - archer
-      - ascent
-      - autoconf-archive
-      - axom
-      - binutils
-      - blaspp
-      - blt
-      - boost
-      - butterflypack
-      - cabana
-      - caliper
-      - camp
-      - chai
-      - conduit
-      - curl
-      - datatransferkit
-      - double-conversion
-      - dray
-      - eigen
-      - faodel
-      - ffmpeg
-      - fftw
-      - fortrilinos
-      - gettext
-      - gperftools
-      - gptune
-      - hdf5
-      - heffte
-      - hpctoolkit
-      - hwloc
-      - hydrogen
-      - hypre
-      - kokkos
-      - lammps
-      - lapackpp
-      - legion
-      - libtool
-      - libxml2
-      - libzmq
-      - llvm-openmp-ompt
-      - mbedtls
-      - mfem
-      - mpich
-      - mvapich2
-      - nasm
-      - netlib-scalapack
-      - omega-h
-      - openblas
-      - openjpeg
-      - openmpi
-      - openpmd-api
-      - pagmo2
-      - papyrus
-      - parsec
-      - pdt
-      - pegtl
-      - petsc
-      - pumi
-      - py-beniget
-      - py-cinemasci
-      - pygmo
-      - py-ipython-genutils
-      - py-packaging
-      - py-petsc4py
-      - py-scipy
-      - py-statsmodels
-      - py-warlock
-      - py-warpx
-      - raja
-      - samrai
-      - slepc
-      - slurm
-      - sqlite
-      - strumpack
-      - sundials
-      - superlu-dist
-      - tasmanian
-      - tau
-      - upcxx
-      - vtk
-      - vtk-h
-      - vtk-m
-      - zfp
-      build-job:
-        tags: [ "spack", "medium" ]
-        variables:
-          CI_JOB_SIZE: "medium"
-          SPACK_BUILD_JOBS: "2"
-
-    - match:
-      - alsa-lib
-      - ant
-      - antlr
-      - argobots
-      - autoconf-archive
-      - automake
-      - berkeley-db
-      - bison
-      - blt
-      - bzip2
-      - camp
-      - cmake
-      - curl
-      - czmq
-      - darshan-util
-      - diffutils
-      - docbook-xml
-      - exmcutils
-      - expat
-      - findutils
-      - flit
-      - freetype
-      - gawk
-      - gdbm
-      - gettext
-      - glib
-      - gmake
-      - gotcha
-      - hpcviewer
-      - hwloc
-      - jansson
-      - json-c
-      - libbsd
-      - libedit
-      - libevent
-      - libfabric
-      - libffi
-      - libgcrypt
-      - libiconv
-      - libidn2
-      - libjpeg-turbo
-      - libmd
-      - libnrm
-      - libpciaccess
-      - libpng
-      - libsigsegv
-      - libsodium
-      - libunistring
-      - libunwind
-      - libxml2
-      - libyaml
-      - libzmq
-      - lua
-      - lua-luaposix
-      - lz4
-      - m4
-      - meson
-      - metis
-      - mpfr
-      - ncurses
-      - ninja
-      - numactl
-      - openblas
-      - openjdk
-      - openssh
-      - openssl
-      - papi
-      - parallel-netcdf
-      - pcre
-      - pcre2
-      - pdsh
-      - perl
-      - perl-data-dumper
-      - pkgconf
-      - py-alembic
-      - py-cffi
-      - py-cycler
-      - py-decorator
-      - py-idna
-      - py-jsonschema
-      - py-kiwisolver
-      - py-mistune
-      - py-pycparser
-      - py-setuptools
-      - py-setuptools-scm
-      - py-six
-      - py-testpath
-      - py-wheel
-      - qhull
-      - readline
-      - sed
-      - slurm
-      - snappy
-      - sqlite
-      - superlu
-      - swig
-      - tar
-      - tcl
-      - texinfo
-      - tut
-      - unzip
-      - util-linux-uuid
-      - util-macros
-      - xz
-      - yaml-cpp
-      - zfp
-      - zlib
-      - zstd
-      build-job:
-        tags: [ "spack", "small" ]
-        variables:
-          CI_JOB_SIZE: "small"
-          SPACK_BUILD_JOBS: "1"
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml
index 3df3e1cc7d..a74ef3f98f 100644
--- a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml
@@ -5,287 +5,3 @@ ci:
   - build-job:
       variables:
         CI_GPG_KEY_ROOT: /etc/protected-runner
-  - match_behavior: first
-    submapping:
-    - match:
-      - hipblas
-      - llvm
-      - llvm-amdgpu
-      - pango
-      - paraview
-      - py-tensorflow
-      - py-torch
-      - qt
-      - rocblas
-      - visit
-      build-job:
-        tags: [ "spack", "huge" ]
-        variables:
-          CI_JOB_SIZE: huge
-          SPACK_BUILD_JOBS: "12"
-
-    - match:
-      - ascent
-      - atk
-      - axom
-      - cistem
-      - cmake
-      - ctffind
-      - cuda
-      - dealii
-      - dray
-      - dyninst
-      - ecp-data-vis-sdk
-      - gcc
-      - ginkgo
-      - hdf5
-      - hpx
-      - kokkos-kernels
-      - kokkos-nvcc-wrapper
-      - lbann
-      - magma
-      - mesa
-      - mfem
-      - mpich
-      - netlib-lapack
-      - nvhpc
-      - oce
-      - openblas
-      - openfoam
-      - openturns
-      - parallelio
-      - plumed
-      - precice
-      #- py-tensorflow
-      #- qt
-      - raja
-      - relion
-      #- rocblas
-      - rocfft
-      - rocsolver
-      - rocsparse
-      - rust
-      - slate
-      - strumpack
-      - sundials
-      - trilinos
-      - umpire
-      #- visit
-      - vtk
-      - vtk-h
-      - vtk-m
-      - warpx
-      - wrf
-      - wxwidgets
-      build-job:
-        tags: [ "spack", "large" ]
-        variables:
-          CI_JOB_SIZE: large
-          SPACK_BUILD_JOBS: "8"
-
-    - match:
-      - adios2
-      - amrex
-      - archer
-      - ascent
-      - autoconf-archive
-      - axom
-      - binutils
-      - blaspp
-      - blt
-      - boost
-      - butterflypack
-      - cabana
-      - caliper
-      - camp
-      - chai
-      - conduit
-      - curl
-      - datatransferkit
-      - double-conversion
-      - dray
-      - eigen
-      - faodel
-      - ffmpeg
-      - fftw
-      - fortrilinos
-      - gettext
-      - gperftools
-      - gptune
-      - hdf5
-      - heffte
-      - hpctoolkit
-      - hwloc
-      - hydrogen
-      - hypre
-      - kokkos
-      - lammps
-      - lapackpp
-      - legion
-      - libtool
-      - libxml2
-      - libzmq
-      - llvm-openmp-ompt
-      - mbedtls
-      - mfem
-      - mpich
-      - mvapich2
-      - nasm
-      - netlib-scalapack
-      - omega-h
-      - openblas
-      - openjpeg
-      - openmpi
-      - openpmd-api
-      - pagmo2
-      - papyrus
-      - parsec
-      - pdt
-      - pegtl
-      - petsc
-      - pumi
-      - py-beniget
-      - py-cinemasci
-      - pygmo
-      - py-ipython-genutils
-      - py-packaging
-      - py-petsc4py
-      - py-scipy
-      - py-statsmodels
-      - py-warlock
-      - py-warpx
-      - raja
-      - samrai
-      - slepc
-      - slurm
-      - sqlite
-      - strumpack
-      - sundials
-      - superlu-dist
-      - tasmanian
-      - tau
-      - upcxx
-      - vtk
-      - vtk-h
-      - vtk-m
-      - zfp
-      build-job:
-        tags: [ "spack", "medium" ]
-        variables:
-          CI_JOB_SIZE: "medium"
-          SPACK_BUILD_JOBS: "2"
-
-    - match:
-      - alsa-lib
-      - ant
-      - antlr
-      - argobots
-      - autoconf-archive
-      - automake
-      - berkeley-db
-      - bison
-      - blt
-      - bzip2
-      - camp
-      - cmake
-      - curl
-      - czmq
-      - darshan-util
-      - diffutils
-      - docbook-xml
-      - exmcutils
-      - expat
-      - findutils
-      - flit
-      - freetype
-      - gawk
-      - gdbm
-      - gettext
-      - glib
-      - gmake
-      - gotcha
-      - hpcviewer
-      - hwloc
-      - jansson
-      - json-c
-      - libbsd
-      - libedit
-      - libevent
-      - libfabric
-      - libffi
-      - libgcrypt
-      - libiconv
-      - libidn2
-      - libjpeg-turbo
-      - libmd
-      - libnrm
-      - libpciaccess
-      - libpng
-      - libsigsegv
-      - libsodium
-      - libunistring
-      - libunwind
-      - libxml2
-      - libyaml
-      - libzmq
-      - lua
-      - lua-luaposix
-      - lz4
-      - m4
-      - meson
-      - metis
-      - mpfr
-      - ncurses
-      - ninja
-      - numactl
-      - openblas
-      - openjdk
-      - openssh
-      - openssl
-      - papi
-      - parallel-netcdf
-      - pcre
-      - pcre2
-      - pdsh
-      - perl
-      - perl-data-dumper
-      - pkgconf
-      - py-alembic
-      - py-cffi
-      - py-cycler
-      - py-decorator
-      - py-idna
-      - py-jsonschema
-      - py-kiwisolver
-      - py-mistune
-      - py-pycparser
-      - py-setuptools
-      - py-setuptools-scm
-      - py-six
-      - py-testpath
-      - py-wheel
-      - qhull
-      - readline
-      - sed
-      - slurm
-      - snappy
-      - sqlite
-      - superlu
-      - swig
-      - tar
-      - tcl
-      - texinfo
-      - tut
-      - unzip
-      - util-linux-uuid
-      - util-macros
-      - xz
-      - yaml-cpp
-      - zfp
-      - zlib
-      - zstd
-      build-job:
-        tags: [ "spack", "small" ]
-        variables:
-          CI_JOB_SIZE: "small"
-          SPACK_BUILD_JOBS: "1"
diff --git a/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml
new file mode 100644
index 0000000000..8dfb169a3e
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml
@@ -0,0 +1,4 @@
+ci:
+  pipeline-gen:
+  - build-job-remove:
+      image: macos-run-on-metal
diff --git a/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
index b774879329..8634042101 100644
--- a/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
@@ -1,5 +1,10 @@
 ci:
   pipeline-gen:
+  - build-job:
+      before_script-:
+      # Test package relocation on linux using a modified prefix
+      # This is not well supported on MacOS (https://github.com/spack/spack/issues/37162)
+      - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
   - match_behavior: first
     submapping:
     - match:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
index 2407c17bde..aa756a6e6a 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
@@ -1,4 +1,7 @@
 spack:
+  include:
+  - $spack/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
+
   view: false
 
   concretizer:
@@ -62,4 +65,4 @@ spack:
   mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray-rhel" }
 
   cdash:
-    build-group: E4S Cray
\ No newline at end of file
+    build-group: E4S Cray
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
index 9fa4625f08..0c0b3a0179 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
@@ -1,4 +1,7 @@
 spack:
+  include:
+  - $spack/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
+
   view: false
 
   concretizer:
@@ -45,4 +48,4 @@ spack:
   mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray-sles" }
 
   cdash:
-    build-group: E4S Cray SLES
\ No newline at end of file
+    build-group: E4S Cray SLES
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
index 90fc297a0d..7f22e232d1 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
@@ -1,19 +1,6 @@
 spack:
   view: false
 
-  concretizer:
-    unify: false
-    reuse: false
-
-  config:
-    concretizer: clingo
-    db_lock_timeout: 120
-    install_tree:
-      root: $spack/opt/spack
-      padded_length: 256
-      projections:
-        all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
-
   packages:
     all:
       require: target=aarch64
@@ -97,42 +84,11 @@ spack:
   ci:
     pipeline-gen:
     - build-job-remove:
-        image: no-image
-        tags: [spack, public]
+        tags: [ spack, public ]
     - build-job:
+        variables:
+          CI_GPG_KEY_ROOT: /etc/protected-runner
         tags: [ "macos-ventura", "apple-clang-14", "aarch64-macos" ]
-        script::
-        - - spack compiler find
-          - cd ${SPACK_CONCRETE_ENV_DIR}
-          - spack env activate --without-view .
-          - if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; fi
-          - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
-          # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
-          - if [[ -r /etc/protected-runner/e4s.gpg ]]; then spack gpg trust /etc/protected-runner/e4s.gpg; fi
-          # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification)
-          - if [[ -r /etc/protected-runner/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /etc/protected-runner/intermediate_ci_signing_key.gpg; fi
-          - if [[ -r /etc/protected-runner/spack_public_key.gpg ]]; then spack gpg trust /etc/protected-runner/spack_public_key.gpg; fi
-          - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
-        after_script:
-        - - cat /proc/loadavg || true
-    - signing-job:
-        image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
-        tags: ["aws"]
-        script:
-        - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
-          - /sign.sh
-          - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
-          - aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
-    - any-job:
-        image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
-        tags: ["spack"]
-        before_script:
-        - - uname -a || true
-          - grep -E "vendor|model name" /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
-          - nproc || true
-        - - . "./share/spack/setup-env.sh"
-          - spack --version
-          - spack arch
 
   cdash:
     build-group: Machine Learning MPS