Added cloud_pipline for E4S on Amazon Linux (#29522)

Add two new cloud pipelines for E4S on Amazon Linux, include arm and x86 (v3 + v4) stacks.

Notes:
- Updated mpark-variant to remove conflict that no longer exists in Amazon Linux
- Which command on Amazon Linux prefixes on all results when padded_length is too high. In this case, padded_length<=503 works as expected. Chose conservative length of 384.

3 files changed, 985 insertions, 0 deletions

diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
index 321c2b6af6..7d2251f544 100644
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -42,12 +42,22 @@ default:
       - runner_system_failure
       - stuck_or_timeout_failure
 
+.generate-aarch64:
+  extends: [ ".generate" ]
+  tags: ["spack", "public", "medium", "aarch64"]
+
 .pr-generate:
   extends: [ ".pr", ".generate" ]
 
+.pr-generate-aarch64:
+  extends: [ ".pr", ".generate-aarch64" ]
+
 .develop-generate:
   extends: [ ".develop", ".generate" ]
 
+.develop-generate-aarch64:
+  extends: [ ".develop", ".generate-aarch64" ]
+
 .build:
   stage: build
 
@@ -364,6 +374,87 @@ data-vis-sdk-develop-build:
       job: data-vis-sdk-develop-generate
 
 ########################################
+# AWS E4S (x86_64)
+########################################
+
+# Call this AFTER .*-generate
+.aws-e4s-overrides:
+  # This controls image for generate step; build step is controlled by spack.yaml
+  # Note that generator emits OS info for build so these should be the same.
+  image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] }
+
+.aws-e4s:
+  variables:
+    SPACK_CI_STACK_NAME: aws-e4s
+
+aws-e4s-pr-generate:
+  extends: [ ".aws-e4s", ".pr-generate", ".aws-e4s-overrides" ]
+  tags: ["spack", "public", "medium", "x86_64_v4"]
+
+aws-e4s-develop-generate:
+  extends: [ ".aws-e4s", ".develop-generate", ".aws-e4s-overrides" ]
+  tags: ["spack", "public", "medium", "x86_64_v4"]
+
+aws-e4s-pr-build:
+  extends: [ ".aws-e4s", ".pr-build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: aws-e4s-pr-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: aws-e4s-pr-generate
+
+aws-e4s-develop-build:
+  extends: [ ".aws-e4s", ".develop-build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: aws-e4s-develop-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: aws-e4s-develop-generate
+
+
+# Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64)
+.aws-e4s-aarch64:
+  variables:
+    SPACK_CI_STACK_NAME: aws-e4s-aarch64
+
+aws-e4s-aarch64-pr-generate:
+  extends: [ ".aws-e4s-aarch64", ".pr-generate-aarch64", ".aws-e4s-overrides" ]
+  tags: ["spack", "public", "large", "aarch64"]
+
+aws-e4s-aarch64-develop-generate:
+  extends: [ ".aws-e4s-aarch64", ".develop-generate-aarch64", ".aws-e4s-overrides" ]
+  tags: ["spack", "public", "large", "aarch64"]
+
+aws-e4s-aarch64-pr-build:
+  extends: [ ".aws-e4s-aarch64", ".pr-build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: aws-e4s-aarch64-pr-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: aws-e4s-aarch64-pr-generate
+
+aws-e4s-aarch64-develop-build:
+  extends: [ ".aws-e4s-aarch64", ".develop-build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: aws-e4s-aarch64-develop-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: aws-e4s-aarch64-develop-generate
+
+
+########################################
 # Spack Tutorial
 ########################################
 .tutorial:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-e4s-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-e4s-aarch64/spack.yaml
new file mode 100644
index 0000000000..cd6e35de15
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-e4s-aarch64/spack.yaml
@@ -0,0 +1,451 @@
+spack:
+  view: false
+  concretization: separately
+
+  concretizer:
+    reuse: false
+
+  config:
+    concretizer: clingo
+    install_tree:
+      root: /home/software/spack
+      padded_length: 384
+      projections:
+        all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
+
+  packages:
+    all:
+      providers:
+        blas:
+          - openblas
+        mkl:
+          - intel-oneapi-mkl
+        mpi:
+          - openmpi
+          - mpich
+      variants: +mpi
+    binutils:
+      variants: +ld +gold +headers +libiberty ~nls
+      version:
+        - 2.36.1
+    doxygen:
+      version:
+        - 1.8.20
+    elfutils:
+      variants: +bzip2 ~nls +xz
+    hdf5:
+      variants: +fortran +hl +shared
+    libfabric:
+      variants: fabrics=efa,tcp,udp,sockets,verbs,shm,mrail,rxd,rxm
+    libunwind:
+      variants: +pic +xz
+    mesa:
+      variants: ~llvm
+    mesa18:
+      variants: ~llvm
+    mpich:
+      variants: ~wrapperrpath netmod=ofi device=ch4
+    ncurses:
+      variants: +termlib
+    openblas:
+      variants: threads=openmp
+    openmpi:
+      variants: fabrics=ofi +legacylaunchers
+    openturns:
+      version: [1.18]
+    trilinos:
+      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+    xz:
+      variants: +pic
+
+  definitions:
+
+  - cuda_specs:
+    - amrex +cuda cuda_arch=70
+    - caliper +cuda cuda_arch=70
+    - chai ~benchmarks ~tests +cuda cuda_arch=70 ^umpire ~shared
+# Breaks on cuda compilation due to link errors: 
+# libcurand.so.10: undefined reference to `logf@GLIBC_2.27'
+    #- ginkgo +cuda cuda_arch=70
+    - heffte +cuda cuda_arch=70
+# x86-specific
+    #- hpx +cuda cuda_arch=70
+    - hypre +cuda cuda_arch=70
+    - kokkos +wrapper +cuda cuda_arch=70
+    - kokkos-kernels +cuda cuda_arch=70 ^kokkos +wrapper +cuda cuda_arch=70
+    - magma +cuda cuda_arch=70
+    - mfem +cuda cuda_arch=70
+    - parsec +cuda cuda_arch=70
+# Breaks in CICD
+# cuda 11.7.0 did not work
+    #- petsc +cuda cuda_arch=70
+    - raja +cuda cuda_arch=70
+    - slate +cuda cuda_arch=70
+# Breaks in CICD
+# Does not pass cuda_arch for petsc
+    #- slepc +cuda cuda_arch=70
+    - strumpack ~slate +cuda cuda_arch=70
+    - sundials +cuda cuda_arch=70
+    - superlu-dist +cuda cuda_arch=70
+    - tasmanian +cuda cuda_arch=70
+    # Trilinos: enable CUDA, Kokkos, and important Tpetra-era solver packages;
+    # disable Epetra; disable ETI to speed up CI; disable all other TPLs
+    - trilinos@13.2.0 +cuda cuda_arch=70 +wrapper    +amesos2 +belos +ifpack2 +kokkos +muelu +nox +stratimikos +tpetra    ~amesos ~anasazi ~aztec ~epetraext ~ifpack ~isorropia ~ml ~teko ~tempus ~zoltan ~zoltan2     ~explicit_template_instantiation ~adios2~basker~boost~chaco~complex~debug~dtk~epetraextbtf~epetraextexperimental~epetraextgraphreorderings~exodus~float~fortran~gtest~hypre~intrepid~intrepid2~ipo~mesquite~minitensor~mumps~openmp~phalanx~piro~rocm~rol~rythmos~sacado~scorec~shards~shared~shylu~stk~stokhos~strumpack~suite-sparse~superlu~superlu-dist~trilinoscouplings~x11
+    - umpire ~shared +cuda cuda_arch=70
+    - vtk-m +cuda cuda_arch=70
+    - zfp +cuda cuda_arch=70
+   #- ascent ~shared +cuda cuda_arch=70
+   #- axom +cuda cuda_arch=70 ^umpire ~shared
+   #- dealii +cuda cuda_arch=70 # gmsh
+   #- flecsi +cuda cuda_arch=70
+   #- paraview +cuda cuda_arch=70
+
+  - rocm_specs:
+    - kokkos +rocm amdgpu_target=gfx906
+    #- amrex +rocm amdgpu_target=gfx906
+    #- chai +rocm ~benchmarks amdgpu_target=gfx906
+    #- ginkgo +rocm amdgpu_target=gfx906 # needs hip<4.1
+    #- raja +rocm ~openmp amdgpu_target=gfx906 # blt 0.3.6 issue with rocm
+    #- slate +rocm amdgpu_target=gfx906
+    #- strumpack +rocm ~slate amdgpu_target=gfx906
+    #- sundials +rocm amdgpu_target=gfx906
+    #- tasmanian +rocm amdgpu_target=gfx906
+    #- umpire+rocm amdgpu_target=gfx906 # blt 0.3.6 issue with rocm
+
+  - default_specs:
+    - adios
+    - adios2
+    - aml
+    - amrex
+    - arborx
+    - archer
+    - argobots
+    - ascent
+    - axom
+    - bolt
+    - cabana
+    - caliper
+    - chai ~benchmarks ~tests
+    - conduit
+    - darshan-runtime
+    - darshan-util
+    - datatransferkit
+    - dyninst
+    - faodel
+    - flecsi@1.4.2 +external_cinch
+    - flit
+    - flux-core
+    - fortrilinos
+    - gasnet
+    - ginkgo
+    - globalarrays
+    - gmp
+    - gotcha
+    - gptune
+    - h5bench
+    - hdf5
+    - heffte +fftw
+    - hpctoolkit
+# x86 specific
+    #- hpx
+    - hypre
+    - kokkos +openmp
+    - kokkos-kernels +openmp
+    - lammps
+    - legion
+    - libnrm
+    - libquo
+    - libunwind
+# bad build on aarch64, graviton2 (likely amdgpu?)
+    #- llvm targets=amdgpu,nvptx +clang +compiler-rt +libcxx +lld +lldb +llvm_dylib +flang ~cuda
+    - loki
+    - mercury
+    - metall
+    - mfem
+    - mpark-variant
+    - mpifileutils ~xattr
+    - nccmp
+    - nco
+    - ninja
+    - nrm
+    - omega-h
+    - openmpi
+    - openpmd-api
+    - papi
+    - papyrus@1.0.1
+    - parallel-netcdf
+    - parsec ~cuda
+    - pdt
+    - petsc
+# Depends on emmintrin.h (x86)
+    #- phist
+    - plasma
+    - precice
+    - pumi
+    - py-jupyterhub
+    - py-libensemble
+    - py-petsc4py
+    - py-warpx ^warpx dims=2
+    - py-warpx ^warpx dims=3
+    - py-warpx ^warpx dims=rz
+    - qthreads scheduler=distrib
+    - raja
+    - rempi
+    - scr
+    - slate ~cuda
+    - slepc
+    - stc
+    - strumpack ~slate
+    - sundials
+    - superlu
+    - superlu-dist
+    - swig
+    - swig@4.0.2-fortran
+    - sz
+    - tasmanian
+    - tau +mpi +python
+    - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+    - turbine
+    - umap
+    - umpire
+    - unifyfs@0.9.1
+    - upcxx
+# x86 specific ASM
+    #- variorum
+    - veloc
+    - wannier90
+    - zfp
+    #- dealii
+    #- geopm
+    #- qt
+    #- qwt
+    #- umpire # unsatisfiable concretization conflict w/ blt
+
+  - compiler:
+    - '%gcc@7.3.1'
+
+  - target:
+    #- 'target=x86_64'
+    - 'target=aarch64'
+    - 'target=graviton2'
+
+
+  specs:
+
+  - matrix:
+    - - $default_specs
+    - - $compiler
+    - - $target
+
+  - matrix:
+    - - $cuda_specs
+    - - $compiler
+    - - $target
+
+  mirrors: { "mirror": "s3://spack-binaries/aws-e4s-aarch64" }
+
+  gitlab-ci:
+
+    script:
+      - . "./share/spack/setup-env.sh"
+      - spack --version
+      - cd ${SPACK_CONCRETE_ENV_DIR}
+      - spack env activate --without-view .
+      - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
+      - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
+      - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+
+    image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] }
+    mappings:
+      - match:
+          - llvm
+          - llvm-amdgpu
+          - hpx
+          - paraview
+          - petsc
+        runner-attributes:
+          tags: [ "spack", "public", "huge", "aarch64" ]
+          variables:
+            CI_JOB_SIZE: huge
+            KUBERNETES_CPU_REQUEST: 11000m
+            KUBERNETES_MEMORY_REQUEST: 42G
+
+      - match:
+          - ascent
+          - axom
+          - cuda
+          - dyninst
+          - ginkgo
+          - kokkos-kernels
+          - kokkos-nvcc-wrapper
+          - magma
+          - mfem
+          - mpich
+          - openturns
+          - precice
+          - raja
+          - rocblas
+          - rocsolver
+          - rust
+          - slate
+          - strumpack
+          - sundials
+          - trilinos
+          - umpire
+          - vtk-h
+          - vtk-m
+          - warpx
+        runner-attributes:
+          tags: [ "spack", "public", "large", "aarch64" ]
+          variables:
+            CI_JOB_SIZE: large
+            KUBERNETES_CPU_REQUEST: 8000m
+            KUBERNETES_MEMORY_REQUEST: 12G
+
+      - match:
+          - adios2
+          - amrex
+          - archer
+          - ascent
+          - axom
+          - binutils
+          - blaspp
+          - boost
+          - butterflypack
+          - cabana
+          - caliper
+          - camp
+          - chai
+          - conduit
+          - datatransferkit
+          - faodel
+          - ffmpeg
+          - fftw
+          - fortrilinos
+          - gperftools
+          - gptune
+          - hdf5
+          - heffte
+          - hpctoolkit
+          - hwloc
+          - hypre
+          - kokkos
+          - lammps
+          - lapackpp
+          - legion
+          - libzmq
+          - llvm-openmp-ompt
+          - mbedtls
+          - netlib-scalapack
+          - omega-h
+          - openmpi
+          - openpmd-api
+          - pagmo2
+          - papyrus
+          - parsec
+          - pdt
+          - pumi
+          - py-ipython-genutils
+          - py-petsc4py
+          - py-scipy
+          - py-statsmodels
+          - py-warlock
+          - py-warpx
+          - pygmo
+          - slepc
+          - slurm
+          - strumpack
+          - sundials
+          - superlu-dist
+          - tasmanian
+          - tau
+          - upcxx
+          - vtk-h
+          - zfp
+        runner-attributes:
+          tags: [ "spack", "public", "medium", "aarch64" ]
+          variables:
+            CI_JOB_SIZE: "medium"
+            KUBERNETES_CPU_REQUEST: "2000m"
+            KUBERNETES_MEMORY_REQUEST: "4G"
+
+      - match:
+          - alsa-lib
+          - ant
+          - antlr
+          - argobots
+          - automake
+          - berkeley-db
+          - bison
+          - blt
+          - cmake
+          - curl
+          - darshan-util
+          - diffutils
+          - exmcutils
+          - expat
+          - flit
+          - freetype
+          - gdbm
+          - gotcha
+          - hpcviewer
+          - jansson
+          - json-c
+          - libbsd
+          - libevent
+          - libjpeg-turbo
+          - libnrm
+          - libpng
+          - libunistring
+          - lua-luaposix
+          - m4
+          - mpfr
+          - ncurses
+          - openblas
+          - openjdk
+          - papi
+          - parallel-netcdf
+          - pcre2
+          - perl-data-dumper
+          - pkgconf
+          - py-alembic
+          - py-idna
+          - py-testpath
+          - qhull
+          - snappy
+          - swig
+          - tar
+          - tcl
+          - texinfo
+          - unzip
+          - util-linux-uuid
+          - util-macros
+          - yaml-cpp
+          - zlib
+          - zstd
+        runner-attributes:
+          tags: [ "spack", "public", "small", "aarch64" ]
+          variables:
+            CI_JOB_SIZE: "small"
+            KUBERNETES_CPU_REQUEST: "500m"
+            KUBERNETES_MEMORY_REQUEST: "500M"
+
+      - match: ['os=amzn2']
+        runner-attributes:
+          tags: ["spack", "public", "aarch64"]
+          variables:
+            CI_JOB_SIZE: "default"
+
+    broken-specs-url: "s3://spack-binaries-develop/broken-specs"
+    service-job-attributes:
+      before_script:
+        - . "./share/spack/setup-env.sh"
+        - spack --version
+      image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] }
+      tags: ["spack", "public", "aarch64"]
+
+  cdash:
+    build-group: AWS Packages
+    url: https://cdash.spack.io
+    project: Spack Testing
+    site: Cloud Gitlab Infrastructure
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-e4s/spack.yaml
new file mode 100644
index 0000000000..5de67eca10
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-e4s/spack.yaml
@@ -0,0 +1,443 @@
+spack:
+  view: false
+  concretization: separately
+
+  concretizer:
+    reuse: false
+
+  config:
+    concretizer: clingo
+    install_tree:
+      root: /home/software/spack
+      padded_length: 384
+      projections:
+        all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
+
+  packages:
+    all:
+      providers:
+        blas:
+          - openblas
+        mkl:
+          - intel-oneapi-mkl
+        mpi:
+          - openmpi
+          - mpich
+      variants: +mpi
+    binutils:
+      variants: +ld +gold +headers +libiberty ~nls
+      version:
+        - 2.36.1
+    doxygen:
+      version:
+        - 1.8.20
+    elfutils:
+      variants: +bzip2 ~nls +xz
+    hdf5:
+      variants: +fortran +hl +shared
+    libfabric:
+      variants: fabrics=efa,tcp,udp,sockets,verbs,shm,mrail,rxd,rxm
+    libunwind:
+      variants: +pic +xz
+    mesa:
+      variants: ~llvm
+    mesa18:
+      variants: ~llvm
+    mpich:
+      variants: ~wrapperrpath netmod=ofi device=ch4
+    ncurses:
+      variants: +termlib
+    openblas:
+      variants: threads=openmp
+    openmpi:
+      variants: fabrics=ofi +legacylaunchers
+    openturns:
+      version: [1.18]
+    trilinos:
+      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+    xz:
+      variants: +pic
+
+  definitions:
+
+  - cuda_specs:
+    - amrex +cuda cuda_arch=70
+    - caliper +cuda cuda_arch=70
+    - chai ~benchmarks ~tests +cuda cuda_arch=70 ^umpire ~shared
+    - ginkgo +cuda cuda_arch=70
+    - heffte +cuda cuda_arch=70
+    - hpx +cuda cuda_arch=70
+    - hypre +cuda cuda_arch=70
+    - kokkos +wrapper +cuda cuda_arch=70
+    - kokkos-kernels +cuda cuda_arch=70 ^kokkos +wrapper +cuda cuda_arch=70
+    - magma +cuda cuda_arch=70
+    - mfem +cuda cuda_arch=70
+    - parsec +cuda cuda_arch=70
+# Breaks in CICD
+    #- petsc +cuda cuda_arch=70
+    - raja +cuda cuda_arch=70
+    - slate +cuda cuda_arch=70
+# Breaks in CICD
+# Does not pass cuda_arch for petsc
+    #- slepc +cuda cuda_arch=70
+    - strumpack ~slate +cuda cuda_arch=70
+    - sundials +cuda cuda_arch=70
+    - superlu-dist +cuda cuda_arch=70
+    - tasmanian +cuda cuda_arch=70
+    # Trilinos: enable CUDA, Kokkos, and important Tpetra-era solver packages;
+    # disable Epetra; disable ETI to speed up CI; disable all other TPLs
+    - trilinos@13.2.0 +cuda cuda_arch=70 +wrapper    +amesos2 +belos +ifpack2 +kokkos +muelu +nox +stratimikos +tpetra    ~amesos ~anasazi ~aztec ~epetraext ~ifpack ~isorropia ~ml ~teko ~tempus ~zoltan ~zoltan2     ~explicit_template_instantiation ~adios2~basker~boost~chaco~complex~debug~dtk~epetraextbtf~epetraextexperimental~epetraextgraphreorderings~exodus~float~fortran~gtest~hypre~intrepid~intrepid2~ipo~mesquite~minitensor~mumps~openmp~phalanx~piro~rocm~rol~rythmos~sacado~scorec~shards~shared~shylu~stk~stokhos~strumpack~suite-sparse~superlu~superlu-dist~trilinoscouplings~x11
+    - umpire ~shared +cuda cuda_arch=70
+    - vtk-m +cuda cuda_arch=70
+    - zfp +cuda cuda_arch=70
+   #- ascent ~shared +cuda cuda_arch=70
+   #- axom +cuda cuda_arch=70 ^umpire ~shared
+   #- dealii +cuda cuda_arch=70 # gmsh
+   #- flecsi +cuda cuda_arch=70
+   #- paraview +cuda cuda_arch=70
+
+  - rocm_specs:
+    - kokkos +rocm amdgpu_target=gfx906
+    #- amrex +rocm amdgpu_target=gfx906
+    #- chai +rocm ~benchmarks amdgpu_target=gfx906
+    #- ginkgo +rocm amdgpu_target=gfx906 # needs hip<4.1
+    #- raja +rocm ~openmp amdgpu_target=gfx906 # blt 0.3.6 issue with rocm
+    #- slate +rocm amdgpu_target=gfx906
+    #- strumpack +rocm ~slate amdgpu_target=gfx906
+    #- sundials +rocm amdgpu_target=gfx906
+    #- tasmanian +rocm amdgpu_target=gfx906
+    #- umpire+rocm amdgpu_target=gfx906 # blt 0.3.6 issue with rocm
+
+  - default_specs:
+    - adios
+    - adios2
+    - aml
+    - amrex
+    - arborx
+    - archer
+    - argobots
+    - ascent
+    - axom
+    - bolt
+    - cabana
+    - caliper
+    - chai ~benchmarks ~tests
+    - conduit
+    - darshan-runtime
+    - darshan-util
+    - datatransferkit
+    - dyninst
+    - faodel
+    - flecsi@1.4.2 +external_cinch
+    - flit
+    - flux-core
+    - fortrilinos
+    - gasnet
+    - ginkgo
+    - globalarrays
+    - gmp
+    - gotcha
+    - gptune
+    - h5bench
+    - hdf5
+    - heffte +fftw
+    - hpctoolkit
+    - hpx
+    - hypre
+    - kokkos +openmp
+    - kokkos-kernels +openmp
+    - lammps
+    - legion
+    - libnrm
+    - libquo
+    - libunwind
+    - llvm targets=amdgpu,nvptx +clang +compiler-rt +libcxx +lld +lldb +llvm_dylib +flang ~cuda
+    - loki
+    - mercury
+    - metall
+    - mfem
+    - mpark-variant
+    - mpifileutils ~xattr
+    - nccmp
+    - nco
+    - ninja
+    - nrm
+    - omega-h
+    - openmpi
+    - openpmd-api
+    - papi
+    - papyrus@1.0.1
+    - parallel-netcdf
+    - parsec ~cuda
+    - pdt
+    - petsc
+    - phist
+    - plasma
+    - precice
+    - pumi
+    - py-jupyterhub
+    - py-libensemble
+    - py-petsc4py
+    - py-warpx ^warpx dims=2
+    - py-warpx ^warpx dims=3
+    - py-warpx ^warpx dims=rz
+    - qthreads scheduler=distrib
+    - raja
+    - rempi
+    - scr
+    - slate ~cuda
+    - slepc
+    - stc
+    - strumpack ~slate
+    - sundials
+    - superlu
+    - superlu-dist
+    - swig
+    - swig@4.0.2-fortran
+    - sz
+    - tasmanian
+    - tau +mpi +python
+    - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+    - turbine
+    - umap
+    - umpire
+    - unifyfs@0.9.1
+    - upcxx
+    - variorum
+    - veloc
+    - wannier90
+    - zfp
+    #- dealii
+    #- geopm
+    #- qt
+    #- qwt
+
+  - compiler:
+    - '%gcc@7.3.1'
+
+  - target:
+    #- 'target=x86_64'
+    - 'target=x86_64_v3'
+    - 'target=x86_64_v4'
+
+
+  specs:
+
+  - matrix:
+    - - $default_specs
+    - - $compiler
+    - - $target
+
+  - matrix:
+    - - $cuda_specs
+    - - $compiler
+    - - $target
+
+  mirrors: { "mirror": "s3://spack-binaries/aws-e4s" }
+
+  gitlab-ci:
+
+    script:
+      - . "./share/spack/setup-env.sh"
+      - spack --version
+      - cd ${SPACK_CONCRETE_ENV_DIR}
+      - spack env activate --without-view .
+      - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
+      - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
+      - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+
+    image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] }
+    mappings:
+      - match:
+          - llvm
+          - llvm-amdgpu
+          - paraview
+        runner-attributes:
+          tags: [ "spack", "public", "huge", "x86_64_v4" ]
+          variables:
+            CI_JOB_SIZE: huge
+            KUBERNETES_CPU_REQUEST: 11000m
+            KUBERNETES_MEMORY_REQUEST: 42G
+
+
+      - match:
+          - adios2
+          - ascent
+          - axom
+          - boost
+          - cuda
+          - dyninst
+          - ginkgo
+          - hpx
+          - kokkos-kernels
+          - kokkos-nvcc-wrapper
+          - magma
+          - mfem
+          - mpich
+          - openturns
+          - petsc
+          - precice
+          - raja
+          - rocblas
+          - rocsolver
+          - rust
+          - slate
+          - strumpack
+          - sundials
+          - trilinos
+          - umpire
+          - vtk-h
+          - vtk-m
+          - warpx
+        runner-attributes:
+          tags: [ "spack", "public", "large", "x86_64_v4" ]
+          variables:
+            CI_JOB_SIZE: large
+            KUBERNETES_CPU_REQUEST: 8000m
+            KUBERNETES_MEMORY_REQUEST: 12G
+
+      - match:
+          - amrex
+          - archer
+          - ascent
+          - axom
+          - binutils
+          - blaspp
+          - butterflypack
+          - cabana
+          - caliper
+          - camp
+          - chai
+          - conduit
+          - datatransferkit
+          - faodel
+          - ffmpeg
+          - fftw
+          - fortrilinos
+          - gperftools
+          - gptune
+          - hdf5
+          - heffte
+          - hpctoolkit
+          - hwloc
+          - hypre
+          - kokkos
+          - lammps
+          - lapackpp
+          - legion
+          - libzmq
+          - llvm-openmp-ompt
+          - mbedtls
+          - netlib-scalapack
+          - omega-h
+          - openmpi
+          - openpmd-api
+          - pagmo2
+          - papyrus
+          - parsec
+          - pdt
+          - pumi
+          - py-ipython-genutils
+          - py-petsc4py
+          - py-scipy
+          - py-statsmodels
+          - py-warlock
+          - py-warpx
+          - pygmo
+          - slepc
+          - slurm
+          - strumpack
+          - sundials
+          - superlu-dist
+          - tasmanian
+          - tau
+          - upcxx
+          - vtk-h
+          - zfp
+        runner-attributes:
+          tags: [ "spack", "public", "medium", "x86_64_v4" ]
+          variables:
+            CI_JOB_SIZE: "medium"
+            KUBERNETES_CPU_REQUEST: "2000m"
+            KUBERNETES_MEMORY_REQUEST: "4G"
+
+      - match:
+          - alsa-lib
+          - ant
+          - antlr
+          - argobots
+          - automake
+          - berkeley-db
+          - bison
+          - blt
+          - cmake
+          - curl
+          - darshan-util
+          - diffutils
+          - exmcutils
+          - expat
+          - flit
+          - freetype
+          - gdbm
+          - gotcha
+          - hpcviewer
+          - jansson
+          - json-c
+          - libbsd
+          - libevent
+          - libjpeg-turbo
+          - libnrm
+          - libpng
+          - libunistring
+          - lua-luaposix
+          - m4
+          - mpfr
+          - ncurses
+          - openblas
+          - openjdk
+          - papi
+          - parallel-netcdf
+          - pcre2
+          - perl-data-dumper
+          - pkgconf
+          - py-alembic
+          - py-idna
+          - py-testpath
+          - qhull
+          - snappy
+          - swig
+          - tar
+          - tcl
+          - texinfo
+          - unzip
+          - util-linux-uuid
+          - util-macros
+          - yaml-cpp
+          - zlib
+          - zstd
+        runner-attributes:
+          tags: [ "spack", "public", "small", "x86_64_v4" ]
+          variables:
+            CI_JOB_SIZE: "small"
+            KUBERNETES_CPU_REQUEST: "500m"
+            KUBERNETES_MEMORY_REQUEST: "500M"
+
+      - match: ['os=amzn2']
+        runner-attributes:
+          tags: ["spack", "public", "x86_64_v4"]
+          variables:
+            CI_JOB_SIZE: "default"
+
+    broken-specs-url: "s3://spack-binaries-develop/broken-specs"
+    service-job-attributes:
+      before_script:
+        - . "./share/spack/setup-env.sh"
+        - spack --version
+      image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] }
+      tags: ["spack", "public", "x86_64_v4"]
+
+  cdash:
+    build-group: AWS Packages
+    url: https://cdash.spack.io
+    project: Spack Testing
+    site: Cloud Gitlab Infrastructure