e4s ci stacks: sync with e4s-23.08 (#40263)

* e4s amd64 gcc ci stack: sync with e4s-23.08

* e4s amd64 oneapi ci stack: sync with e4s-23.08

* e4s ppc64 gcc ci stack: sync with e4s-23.08

* add new ci stack: e4s amd64 gcc w/ external rocm

* add new ci stack: e4s arm gcc ci

* updates

* py-scipy: -fvisibility issue is resolved in 2023.1.0: #39464

* paraview oneapi fails

* comment out pkgs that fail to build on power

* fix arm stack name

* fix cabana +cuda specification

* comment out failing spces

* visit fails build on arm

* comment out slepc arm builds due to make issue

* comment out failing dealii arm builds

6 files changed, 1212 insertions, 289 deletions

diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
index d7f8a38ef6..fe801e68f0 100644
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -287,7 +287,7 @@ protected-publish:
 
 e4s-generate:
   extends: [ ".e4s", ".generate-x86_64"]
-  image: ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01
+  image: ecpe4s/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01
 
 e4s-build:
   extends: [ ".e4s", ".build" ]
@@ -301,6 +301,52 @@ e4s-build:
       job: e4s-generate
 
 ########################################
+# E4S ARM pipeline
+########################################
+.e4s-arm:
+  extends: [ ".linux_aarch64" ]
+  variables:
+    SPACK_CI_STACK_NAME: e4s-arm
+
+e4s-arm-generate:
+  extends: [ ".e4s-arm", ".generate-aarch64" ]
+  image: ecpe4s/ubuntu20.04-runner-arm64-gcc-11.4:2023.08.01
+
+e4s-arm-build:
+  extends: [ ".e4s-arm", ".build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: e4s-arm-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: e4s-arm-generate
+
+########################################
+# E4S ROCm External pipeline
+########################################
+.e4s-rocm-external:
+  extends: [ ".linux_x86_64_v3" ]
+  variables:
+    SPACK_CI_STACK_NAME: e4s-rocm-external
+
+e4s-rocm-external-generate:
+  extends: [ ".e4s-rocm-external", ".generate-x86_64"]
+  image: ecpe4s/ubuntu20.04-runner-amd64-gcc-11.4-rocm5.4.3:2023.08.01
+
+e4s-rocm-external-build:
+  extends: [ ".e4s-rocm-external", ".build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: e4s-rocm-external-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: e4s-rocm-external-generate
+
+########################################
 # GPU Testing Pipeline
 ########################################
 .gpu-tests:
@@ -333,7 +379,7 @@ gpu-tests-build:
 
 e4s-oneapi-generate:
   extends: [ ".e4s-oneapi", ".generate-x86_64"]
-  image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023.07.21
+  image: ecpe4s/ubuntu20.04-runner-amd64-oneapi-2023.2.1:2023.08.01
 
 e4s-oneapi-build:
   extends: [ ".e4s-oneapi", ".build" ]
@@ -350,7 +396,7 @@ e4s-oneapi-build:
 # E4S on Power
 ########################################
 .e4s-power-generate-tags-and-image:
-  image: { "name": "ecpe4s/ubuntu20.04-runner-ppc64le:2023-01-01", "entrypoint": [""] }
+  image: { "name": "ecpe4s/ubuntu20.04-runner-ppc64-gcc-11.4:2023.08.01", "entrypoint": [""] }
   tags: ["spack", "public", "large", "ppc64le"]
 
 .e4s-power:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-arm/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-arm/spack.yaml
new file mode 100644
index 0000000000..4a15e62b9b
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-arm/spack.yaml
@@ -0,0 +1,351 @@
+spack:
+  view: false
+
+  concretizer:
+    reuse: false
+    unify: false
+
+  packages:
+    all:
+      require: '%gcc@11.4.0 target=aarch64'
+      providers:
+        blas: [openblas]
+        mpi: [mpich]
+      variants: +mpi
+    binutils:
+      variants: +ld +gold +headers +libiberty ~nls
+    elfutils:
+      variants: +bzip2 ~nls +xz
+    hdf5:
+      variants: +fortran +hl +shared
+    libfabric:
+      variants: fabrics=sockets,tcp,udp,rxm
+    libunwind:
+      variants: +pic +xz
+    openblas:
+      variants: threads=openmp
+    trilinos:
+      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
+        +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
+        +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
+        +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+    xz:
+      variants: +pic
+    mesa:
+      version: [21.3.8]
+    mpi:
+      require: mpich
+    mpich:
+      require: '~wrapperrpath ~hwloc'
+    ncurses:
+      require: '@6.3 +termlib'
+    tbb:
+      require: intel-tbb
+    boost:
+      version: [1.79.0]
+      variants: +atomic +chrono +container +date_time +exception +filesystem +graph
+        +iostreams +locale +log +math +mpi +multithreaded +program_options +random
+        +regex +serialization +shared +signals +stacktrace +system +test +thread +timer
+        cxxstd=17 visibility=global
+    libffi:
+      require: "@3.4.4"
+    vtk-m:
+      require: "+examples"
+    cuda:
+      version: [11.8.0]
+
+  compilers:
+  - compiler:
+      spec: gcc@11.4.0
+      paths:
+        cc: /usr/bin/gcc
+        cxx: /usr/bin/g++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      flags: {}
+      operating_system: ubuntu20.04
+      target: aarch64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+
+  specs:
+  # CPU
+  - adios
+  - alquimia
+  - aml
+  - amrex
+  - arborx
+  - argobots
+  - ascent # ecp dav
+  - axom
+  - bolt
+  - boost
+  - bricks ~cuda
+  - butterflypack
+  - cabana
+  - caliper
+  - chai ~benchmarks ~tests
+  - charliecloud
+  - conduit
+  - datatransferkit
+  - dyninst
+  - exaworks
+  - flecsi
+  - flit
+  - flux-core
+  - fortrilinos
+  - gasnet
+  - ginkgo
+  - globalarrays
+  - gmp
+  - gotcha
+  - gptune ~mpispawn
+  - h5bench
+  - hdf5-vol-async
+  - hdf5-vol-cache
+  - hdf5-vol-log
+  - heffte +fftw
+  - hpctoolkit
+  - hpx networking=mpi
+  - hypre
+  - kokkos +openmp
+  - kokkos-kernels +openmp
+  - lammps
+  - lbann
+  - legion
+  - libnrm
+  - libquo
+  - libunwind
+  - loki
+  - mercury
+  - metall
+  - mfem
+  - mgard +serial +openmp +timing +unstructured ~cuda
+  - mpark-variant
+  - mpifileutils ~xattr
+  - nccmp
+  - nco
+  - netlib-scalapack
+  - nrm
+  - nvhpc
+  - omega-h
+  - openfoam
+  - openmpi
+  - openpmd-api
+  - papi
+  - papyrus
+  - parsec ~cuda
+  - pdt
+  - petsc
+  - phist
+  - plasma
+  - plumed
+  - precice
+  - pruners-ninja
+  - pumi
+  - py-h5py
+  - py-jupyterhub
+  - py-libensemble
+  - py-petsc4py
+  - py-warpx
+  - qthreads scheduler=distrib
+  - quantum-espresso
+  - raja
+  - rempi
+  - scr
+  - slate ~cuda
+  - slepc
+  - stc
+  - strumpack ~slate
+  - sundials
+  - superlu
+  - superlu-dist
+  - swig@4.0.2-fortran
+  - sz3
+  - tasmanian
+  - tau +mpi +python
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+  - turbine
+  - umap
+  - umpire
+  - upcxx
+  - wannier90
+  - xyce +mpi +shared +pymi +pymi_static_tpls
+  # INCLUDED IN ECP DAV CPU
+  - adios2
+  - darshan-runtime
+  - darshan-util
+  - faodel
+  - hdf5
+  - libcatalyst
+  - parallel-netcdf
+  - paraview
+  - py-cinemasci
+  - sz
+  - unifyfs
+  - veloc
+  # - visit                         # silo: https://github.com/spack/spack/issues/39538
+  - vtk-m
+  - zfp 
+  # --
+  # - dealii                        # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
+  # - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp # py-numcodecs@0.7.3: gcc: error: unrecognized command-line option '-mno-sse2'
+  # - archer                        # part of llvm +omp_tsan
+  - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 +paraview +pnetcdf +sz +unifyfs +veloc ~visit +vtkm +zfp  # +visit: ?
+  # - geopm                         # geopm: https://github.com/spack/spack/issues/38795
+  # - variorum                      # variorum: https://github.com/spack/spack/issues/38786
+
+  # CUDA NOARCH
+  - bricks +cuda
+  - flux-core +cuda
+  - hpctoolkit +cuda
+  - papi +cuda
+  - tau +mpi +cuda
+  # --
+  # - legion +cuda                    # legion: needs NVIDIA driver
+
+  # CUDA 75  
+  - amrex +cuda cuda_arch=75
+  - arborx +cuda cuda_arch=75 ^kokkos +wrapper
+  - cabana +cuda cuda_arch=75 ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=75
+  - caliper +cuda cuda_arch=75
+  - chai ~benchmarks ~tests +cuda cuda_arch=75 ^umpire ~shared
+  - flecsi +cuda cuda_arch=75
+  - ginkgo +cuda cuda_arch=75
+  - heffte +cuda cuda_arch=75
+  - hpx +cuda cuda_arch=75
+  - hypre +cuda cuda_arch=75
+  - kokkos +wrapper +cuda cuda_arch=75
+  - kokkos-kernels +cuda cuda_arch=75 ^kokkos +wrapper +cuda cuda_arch=75
+  - magma +cuda cuda_arch=75
+  - mfem +cuda cuda_arch=75
+  - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=75
+  - omega-h +cuda cuda_arch=75
+  - parsec +cuda cuda_arch=75
+  - petsc +cuda cuda_arch=75
+  - raja +cuda cuda_arch=75
+  - slate +cuda cuda_arch=75
+  - strumpack ~slate +cuda cuda_arch=75
+  - sundials +cuda cuda_arch=75
+  - superlu-dist +cuda cuda_arch=75
+  - tasmanian +cuda cuda_arch=75
+  - trilinos +cuda cuda_arch=75
+  - umpire ~shared +cuda cuda_arch=75
+  # INCLUDED IN ECP DAV CUDA
+  - adios2 +cuda cuda_arch=75
+  - paraview +cuda cuda_arch=75
+  - vtk-m +cuda cuda_arch=75
+  - zfp +cuda cuda_arch=75
+  # --
+  # - ascent +cuda cuda_arch=75     # ascent: https://github.com/spack/spack/issues/38045
+  # - axom +cuda cuda_arch=75       # axom: https://github.com/spack/spack/issues/29520
+  # - cusz +cuda cuda_arch=75       # cusz: https://github.com/spack/spack/issues/38787
+  # - dealii +cuda cuda_arch=75     # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
+  # - ecp-data-vis-sdk +adios2 +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=75  # embree: https://github.com/spack/spack/issues/39534
+  # - lammps +cuda cuda_arch=75     # lammps: needs NVIDIA driver
+  # - lbann +cuda cuda_arch=75      # lbann: https://github.com/spack/spack/issues/38788
+  # - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf ~cusz +mgard +cuda cuda_arch=75 # libpressio: CMake Error at CMakeLists.txt:498 (find_library): Could not find CUFile_LIBRARY using the following names: cufile ; +cusz: https://github.com/spack/spack/issues/38787 
+  # - py-torch +cuda cuda_arch=75   # skipped, installed by other means
+  # - slepc +cuda cuda_arch=75      # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
+  # - upcxx +cuda cuda_arch=75      # upcxx: needs NVIDIA driver
+
+  # CUDA 80
+  - amrex +cuda cuda_arch=80
+  - arborx +cuda cuda_arch=80 ^kokkos +wrapper
+  - cabana +cuda cuda_arch=80 ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=80
+  - caliper +cuda cuda_arch=80
+  - chai ~benchmarks ~tests +cuda cuda_arch=80 ^umpire ~shared
+  - flecsi +cuda cuda_arch=80
+  - ginkgo +cuda cuda_arch=80
+  - heffte +cuda cuda_arch=80
+  - hpx +cuda cuda_arch=80
+  - hypre +cuda cuda_arch=80
+  - kokkos +wrapper +cuda cuda_arch=80
+  - kokkos-kernels +cuda cuda_arch=80 ^kokkos +wrapper +cuda cuda_arch=80
+  - magma +cuda cuda_arch=80
+  - mfem +cuda cuda_arch=80
+  - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=80
+  - omega-h +cuda cuda_arch=80
+  - parsec +cuda cuda_arch=80
+  - petsc +cuda cuda_arch=80
+  - raja +cuda cuda_arch=80
+  - slate +cuda cuda_arch=80
+  - strumpack ~slate +cuda cuda_arch=80
+  - sundials +cuda cuda_arch=80
+  - superlu-dist +cuda cuda_arch=80
+  - tasmanian +cuda cuda_arch=80
+  - trilinos +cuda cuda_arch=80
+  - umpire ~shared +cuda cuda_arch=80
+  # INCLUDED IN ECP DAV CUDA
+  - adios2 +cuda cuda_arch=80
+  - paraview +cuda cuda_arch=80
+  - vtk-m +cuda cuda_arch=80
+  - zfp +cuda cuda_arch=80
+  # --
+  # - ascent +cuda cuda_arch=80     # ascent: https://github.com/spack/spack/issues/38045
+  # - axom +cuda cuda_arch=80       # axom: https://github.com/spack/spack/issues/29520
+  # - cusz +cuda cuda_arch=80       # cusz: https://github.com/spack/spack/issues/38787
+  # - dealii +cuda cuda_arch=80     # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
+  # - ecp-data-vis-sdk +adios2 +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=80 # embree: https://github.com/spack/spack/issues/39534
+  # - lammps +cuda cuda_arch=80     # lammps: needs NVIDIA driver
+  # - lbann +cuda cuda_arch=80      # lbann: https://github.com/spack/spack/issues/38788
+  # - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf ~cusz +mgard +cuda cuda_arch=80 # libpressio: CMake Error at CMakeLists.txt:498 (find_library): Could not find CUFile_LIBRARY using the following names: cufile ; +cusz: https://github.com/spack/spack/issues/38787 
+  # - py-torch +cuda cuda_arch=80   # skipped, installed by other means
+  # - slepc +cuda cuda_arch=80      # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
+  # - upcxx +cuda cuda_arch=80      # upcxx: needs NVIDIA driver
+
+  # CUDA 90
+  - amrex +cuda cuda_arch=90
+  - arborx +cuda cuda_arch=90 ^kokkos +wrapper
+  - cabana +cuda cuda_arch=90  ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=90
+  - caliper +cuda cuda_arch=90
+  - chai ~benchmarks ~tests +cuda cuda_arch=90 ^umpire ~shared
+  - flecsi +cuda cuda_arch=90
+  - ginkgo +cuda cuda_arch=90
+  - heffte +cuda cuda_arch=90
+  - hpx +cuda cuda_arch=90
+  - kokkos +wrapper +cuda cuda_arch=90
+  - kokkos-kernels +cuda cuda_arch=90 ^kokkos +wrapper +cuda cuda_arch=90
+  - magma +cuda cuda_arch=90
+  - mfem +cuda cuda_arch=90
+  - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=90
+  - parsec +cuda cuda_arch=90
+  - petsc +cuda cuda_arch=90
+  - raja +cuda cuda_arch=90
+  - slate +cuda cuda_arch=90
+  - strumpack ~slate +cuda cuda_arch=90
+  - sundials +cuda cuda_arch=90
+  - superlu-dist +cuda cuda_arch=90
+  - trilinos +cuda cuda_arch=90
+  - umpire ~shared +cuda cuda_arch=90
+  # INCLUDED IN ECP DAV CUDA
+  - adios2 +cuda cuda_arch=90
+  # - paraview +cuda cuda_arch=90   # paraview: InstallError: Incompatible cuda_arch=90
+  - vtk-m +cuda cuda_arch=90
+  - zfp +cuda cuda_arch=90
+  # --
+  # - ascent +cuda cuda_arch=90     # ascent: https://github.com/spack/spack/issues/38045
+  # - axom +cuda cuda_arch=90       # axom: https://github.com/spack/spack/issues/29520
+  # - cusz +cuda cuda_arch=90       # cusz: https://github.com/spack/spack/issues/38787
+  # - dealii +cuda cuda_arch=90     # dealii: https://github.com/spack/spack/issues/39532
+  # - ecp-data-vis-sdk +adios2 +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=90 # embree: https://github.com/spack/spack/issues/39534
+  # - hypre +cuda cuda_arch=90      # concretizer: hypre +cuda requires cuda@:11, but cuda_arch=90 requires cuda@12:
+  # - lammps +cuda cuda_arch=90     # lammps: needs NVIDIA driver
+  # - lbann +cuda cuda_arch=90      # concretizer: Cannot select a single "version" for package "lbann"
+  # - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf ~cusz +mgard +cuda cuda_arch=90 # libpressio: CMake Error at CMakeLists.txt:498 (find_library): Could not find CUFile_LIBRARY using the following names: cufile ; +cusz: https://github.com/spack/spack/issues/38787 
+  # - omega-h +cuda cuda_arch=90    # omega-h: https://github.com/spack/spack/issues/39535
+  # - py-torch +cuda cuda_arch=90   # skipped, installed by other means
+  # - slepc +cuda cuda_arch=90      # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
+  # - tasmanian +cuda cuda_arch=90  # tasmanian: conflicts with cuda@12
+  # - upcxx +cuda cuda_arch=90      # upcxx: needs NVIDIA driver
+
+  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-arm" }
+
+  ci:
+    pipeline-gen:
+    - build-job:
+        image: "ecpe4s/ubuntu20.04-runner-arm64-gcc-11.4:2023.08.01"
+
+  cdash:
+    build-group: E4S ARM
\ No newline at end of file
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
index 798ba974ce..160b079eb5 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
@@ -1,22 +1,51 @@
 spack:
-
   view: false
 
   concretizer:
     reuse: false
     unify: false
 
+  compilers:
+  - compiler:
+      spec: oneapi@2023.2.1
+      paths:
+        cc: /opt/intel/oneapi/compiler/2023.2.1/linux/bin/icx
+        cxx: /opt/intel/oneapi/compiler/2023.2.1/linux/bin/icpx
+        f77: /opt/intel/oneapi/compiler/2023.2.1/linux/bin/ifx
+        fc: /opt/intel/oneapi/compiler/2023.2.1/linux/bin/ifx
+      flags: {}
+      operating_system: ubuntu20.04
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+  - compiler:
+      spec: gcc@=11.4.0
+      paths:
+        cc: /usr/bin/gcc
+        cxx: /usr/bin/g++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      flags: {}
+      operating_system: ubuntu20.04
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+
   packages:
     all:
-      require: '%oneapi'
+      require: '%oneapi target=x86_64_v3'
       providers:
         blas: [openblas]
         mpi: [mpich]
         tbb: [intel-tbb]
-      target: [x86_64]
       variants: +mpi
     elfutils:
       variants: +bzip2 ~nls +xz
+    hdf5:
+      require: "%gcc"
+      variants: +fortran +hl +shared
     libfabric:
       variants: fabrics=sockets,tcp,udp,rxm
     libunwind:
@@ -34,15 +63,12 @@ spack:
       variants: +pic
     mesa:
       version: [21.3.8]
-    hdf5:
-      require: "%gcc"
-      variants: +fortran +hl +shared
     mpi:
-      require: "mpich"
+      require: 'mpich@4:'
     mpich:
-      require: '@4.1.1 ~wrapperrpath ~hwloc'
+      require: '~wrapperrpath ~hwloc'
     py-cryptography:
-      require: '@38.0'
+      require: '@38.0.1'
     unzip:
       require: '%gcc'
     binutils:
@@ -60,40 +86,12 @@ spack:
       require: '%gcc'
     openssh:
       require: '%gcc'
-    bison:
-      require: '%gcc'
     libffi:
       require: "@3.4.4"
     dyninst:
       require: "%gcc"
-
-  compilers:
-  - compiler:
-      spec: oneapi@2023.2.0
-      paths:
-        cc: /opt/intel/oneapi/compiler/2023.2.0/linux/bin/icx
-        cxx: /opt/intel/oneapi/compiler/2023.2.0/linux/bin/icpx
-        f77: /opt/intel/oneapi/compiler/2023.2.0/linux/bin/ifx
-        fc: /opt/intel/oneapi/compiler/2023.2.0/linux/bin/ifx
-      flags: {}
-      operating_system: ubuntu20.04
-      target: x86_64
-      modules: []
-      environment: {}
-      extra_rpaths: []
-  - compiler:
-      spec: gcc@11.4.0
-      paths:
-        cc: /usr/bin/gcc
-        cxx: /usr/bin/g++
-        f77: /usr/bin/gfortran
-        fc: /usr/bin/gfortran
-      flags: {}
-      operating_system: ubuntu20.04
-      target: x86_64
-      modules: []
-      environment: {}
-      extra_rpaths: []
+    bison:
+      require: '%gcc'
 
   specs:
   # CPU
@@ -101,7 +99,6 @@ spack:
   - aml
   - amrex
   - arborx
-  - archer
   - argobots
   - axom
   - bolt
@@ -121,11 +118,14 @@ spack:
   - flux-core
   - fortrilinos
   - gasnet
+  - ginkgo
   - globalarrays
   - gmp
   - gotcha
+  - gptune ~mpispawn
   - h5bench
   - hdf5-vol-async
+  - hdf5-vol-cache
   - hdf5-vol-log
   - heffte +fftw
   - hpx networking=mpi
@@ -136,21 +136,21 @@ spack:
   - lbann
   - legion
   - libnrm 
+  - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp
   - libquo
   - libunwind
   - loki
   - mercury
   - metall
   - mfem
-  - mgard +serial +openmp +timing +unstructured ~cuda
   - mpark-variant
   - mpifileutils ~xattr
   - nccmp
   - nco
   - netlib-scalapack
+  - nrm
   - omega-h
   - openmpi
-  - openpmd-api
   - papi
   - papyrus
   - parsec ~cuda
@@ -160,14 +160,18 @@ spack:
   - plasma
   - plumed
   - precice
+  - pruners-ninja
   - pumi
   - py-h5py
+  - py-jupyterhub
   - py-libensemble
   - py-petsc4py
+  - py-warpx
   - qthreads scheduler=distrib
   - quantum-espresso
   - raja
   - rempi
+  - scr
   - slate ~cuda
   - slepc
   - stc
@@ -175,54 +179,46 @@ spack:
   - sundials
   - superlu
   - superlu-dist
-  - swig@4.0.2-fortran
   - sz3
   - tasmanian
-  - trilinos@13.0.1 +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+  - tau +mpi +python
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
   - turbine
   - umap
   - umpire
   - variorum
   - wannier90
+  - xyce +mpi +shared +pymi +pymi_static_tpls
   # INCLUDED IN ECP DAV CPU
-  # - adios2
-  # - ascent
-  # - darshan-runtime
-  # - darshan-util
-  # - faodel
-  # - hdf5
-  # - libcatalyst
-  # - parallel-netcdf
-  # - paraview
-  # - py-cinemasci
-  # - sz
-  # - unifyfs
-  # - veloc
-  # - visit
-  # - vtk-m ~openmp # https://github.com/spack/spack/issues/31830
-  # - zfp
+  - adios2                # mgard:  mgard.tpp:63:48: error: non-constant-expression cannot be narrowed from type 'int' to 'unsigned long' in initializer list [-Wc++11-narrowing]
+  - ascent
+  - darshan-runtime
+  - darshan-util
+  - faodel
+  - hdf5
+  - libcatalyst
+  - parallel-netcdf
+  # - paraview            # paraview: VTK/ThirdParty/cgns/vtkcgns/src/adfh/ADFH.c:2002:23: error: incompatible function pointer types passing 'herr_t (hid_t, const char *, const H5L_info1_t *, void *)' (aka 'int (long, const char *, const H5L_info1_t *, void *)') to parameter of type 'H5L_iterate2_t' (aka 'int (*)(long, const char *,const H5L_info2_t *, void *)') [-Wincompatible-function-pointer-types]
+  - py-cinemasci
+  - sz
+  - unifyfs
+  - veloc
+  # - visit               # silo: https://github.com/spack/spack/issues/39538
+  - vtk-m ~openmp         # https://github.com/spack/spack/issues/31830
+  - zfp
   # --
-  # - alquimia          # pflotran: pflotran/hdf5_aux.F90(5): error #7013: This module file was not generated by any release of this compiler.   [HDF5]
-  # - dealii            # intel-tbb: icpx: error: unknown argument: '-flifetime-dse=1'
-  # - dxt-explorer      # r: https://github.com/spack/spack/issues/40257
-  # - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 +paraview +pnetcdf +sz +unifyfs +veloc +visit +vtkm +zfp # sz: hdf5-filter/H5Z-SZ/src/H5Z_SZ.c:24:9: error: call to undeclared function 'gettimeofday'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
-  # - geopm             # geopm: In file included from src/ProfileTable.cpp:34: ./src/ProfileTable.hpp:79:45: error: no type named 'string' in namespace 'std'
-  # - ginkgo            # ginkgo: icpx: error: clang frontend command failed with exit code 139 (use -v to see invocation)
-  # - gptune ~mpispawn  # py-scipy: for_main.c:(.text+0x19): undefined reference to `MAIN__'
-  # - hdf5-vol-cache    # /H5VLcache_ext.c:580:9: error: incompatible function pointer types initializing 'herr_t (*)(const void *, uint64_t *)' (aka 'int (*)(const void *, unsigned long *)') with an expression of type 'herr_t (const void *, unsigned int *)' (aka 'int (const void *, unsigned int *)') [-Wincompatible-function-pointer-types]
-  # - hpctoolkit        # intel-tbb: icpx: error: unknown argument: '-flifetime-dse=1'
-  # - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp # py-numcodecs: c-blosc/internal-complibs/zlib-1.2.8/gzread.c:30:15: error: call to undeclared function 'read'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
-  # - nrm               # py-scipy: for_main.c:(.text+0x19): undefined reference to `MAIN__'
-  # - openfoam          # adios2: patch failed
-  # - pruners-ninja     # pruners-ninja: ninja_test_pingpong.c:79:5: error: call to undeclared library function 'memset' with type 'void *(void *, int, unsigned long)'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
-  # - py-jupyterhub     # py-ruamel-yaml-clib: setuptools/dist.py:287: SetuptoolsDeprecationWarning: The namespace_packages parameter is deprecated, consider using implicit namespaces instead (PEP 420). See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages
-  # - py-warpx ^warpx dims=2 # py-scipy: for_main.c:(.text+0x19): undefined reference to `MAIN__'
-  # - py-warpx ^warpx dims=3 # py-scipy: for_main.c:(.text+0x19): undefined reference to `MAIN__'
-  # - py-warpx ^warpx dims=rz # py-scipy: for_main.c:(.text+0x19): undefined reference to `MAIN__'  
-  # - scr               # libyogrt: configure: error: slurm is not in specified location!
-  # - tau +mpi +python  # tau: x86_64/lib/Makefile.tau-icpx-papi-mpi-pthread-python-pdt: No such file or directory
-  # - upcxx             # upcxx: /opt/intel/oneapi/mpi/2021.9.0//libfabric/bin/fi_info: error while loading shared libraries: libfabric.so.1: cannot open shared object file: No such file or directory
-  # - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos~shylu # cmake/tps.cmake:220 (message): Unable to compile against Trilinos.  It is possible Trilinos was not properly configured, or the environment has changed since Trilinos was installed.  See the CMake log files for more information.
+  # - alquimia            # pflotran: https://github.com/spack/spack/issues/39474
+  # - archer              # subsumed under llvm +libomp_tsan
+  # - dealii              # dealii: https://github.com/spack/spack/issues/39482
+  # - dxt-explorer        # r: https://github.com/spack/spack/issues/40257
+  # - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 +paraview +pnetcdf +sz +unifyfs +veloc +visit +vtkm +zfp # embree: CMake Error at CMakeLists.txt:215 (MESSAGE): Unsupported compiler: IntelLLVM; qt: qtbase/src/corelib/global/qendian.h:333:54: error: incomplete type 'std::numeric_limits' used in nested name specifier
+  # - geopm               # geopm issue: https://github.com/spack/spack/issues/38795
+  # - hpctoolkit          # dyninst@12.3.0%gcc: /usr/bin/ld: libiberty/./d-demangle.c:142: undefined reference to `_intel_fast_memcpy'; can't mix intel-tbb@%oneapi with dyninst%gcc
+  # - mgard +serial +openmp +timing +unstructured ~cuda # mgard: mgard.tpp:63:48: error: non-constant-expression cannot be narrowed from type 'int' to 'unsigned long' in initializer list [-Wc++11-narrowing]
+  # - openfoam            # cgal: https://github.com/spack/spack/issues/39481
+  # - openpmd-api         # mgard:  mgard.tpp:63:48: error: non-constant-expression cannot be narrowed from type 'int' to 'unsigned long' in initializer list [-Wc++11-narrowing]
+  # - swig@4.0.2-fortran  # ?
+  # - upcxx               # upcxx: /opt/intel/oneapi/mpi/2021.10.0//libfabric/bin/fi_info: error while loading shared libraries: libfabric.so.1: cannot open shared object file: No such file or directory
 
   # GPU
   - aml +ze
@@ -231,27 +227,21 @@ spack:
   - cabana +sycl ^kokkos +sycl +openmp cxxstd=17 +tests +examples
   - kokkos +sycl +openmp cxxstd=17 +tests +examples
   - kokkos-kernels build_type=Release %oneapi ^kokkos +sycl +openmp cxxstd=17 +tests +examples
+  - tau +mpi +opencl +level_zero ~pdt   # tau: requires libdrm.so to be installed
   - slate +sycl
   # --
   # - ginkgo +oneapi                    # InstallError: Ginkgo's oneAPI backend requires theDPC++ compiler as main CXX compiler.
-  # - hpctoolkit +level_zero            # intel-tbb: icpx: error: unknown argument: '-flifetime-dse=1'
+  # - hpctoolkit +level_zero            # dyninst@12.3.0%gcc: /usr/bin/ld: libiberty/./d-demangle.c:142: undefined reference to `_intel_fast_memcpy'; can't mix intel-tbb@%oneapi with dyninst%gcc
   # - sundials +sycl cxxstd=17          # sundials: include/sunmemory/sunmemory_sycl.h:20:10: fatal error: 'CL/sycl.hpp' file not found
-  # - tau +mpi +opencl +level_zero ~pdt # builds ok in container, but needs libdrm, will update container
 
-  # Not necessarily E4S, but useful for E4S packages
   - py-scipy
 
-  # SKIPPED
-  # - nvhpc
-  # - dyninst  # only %gcc
-
-
   mirrors: { "mirror": "s3://spack-binaries/develop/e4s-oneapi" }
 
   ci:
     pipeline-gen:
     - build-job:
-        image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023.07.21
+        image: ecpe4s/ubuntu20.04-runner-amd64-oneapi-2023.2.1:2023.08.01
 
   cdash:
     build-group: E4S OneAPI
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
index a45fbbb3db..1fcde50ca6 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
@@ -1,19 +1,35 @@
 spack:
   view: false
+
+  concretizer:
+    reuse: false
+    unify: false
+
+  compilers:
+  - compiler:
+      spec: gcc@9.4.0
+      paths:
+        cc: /usr/bin/gcc
+        cxx: /usr/bin/g++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      flags: {}
+      operating_system: ubuntu20.04
+      target: ppc64le
+      modules: []
+      environment: {}
+      extra_rpaths: []
+
   packages:
     all:
-      compiler: [gcc@11.1.0]
+      require: "%gcc@9.4.0 target=ppc64le"
+      compiler: [gcc@9.4.0]
       providers:
         blas: [openblas]
         mpi: [mpich]
-      target: [ppc64le]
       variants: +mpi cuda_arch=70
-    tbb:
-      require: intel-tbb
     binutils:
       variants: +ld +gold +headers +libiberty ~nls
-    cuda:
-      version: [11.7.0]
     elfutils:
       variants: +bzip2 ~nls +xz
     hdf5:
@@ -22,30 +38,34 @@ spack:
       variants: fabrics=sockets,tcp,udp,rxm
     libunwind:
       variants: +pic +xz
-    mpich:
-      variants: ~wrapperrpath
-    ncurses:
-      variants: +termlib
     openblas:
       variants: threads=openmp
-    paraview:
-      require: '@5.11 ~qt+osmesa'
     trilinos:
-      require:
-      - one_of: [+amesos +amesos2 +anasazi +aztec +boost +epetra +epetraext +ifpack
-            +intrepid +intrepid2 +isorropia +kokkos +minitensor +nox +piro +phalanx
-            +rol +rythmos +sacado +stk +shards +stratimikos +tempus +tpetra
-            +trilinoscouplings +zoltan]
-      - one_of: [gotype=long_long, gotype=all]
-      - one_of: [~ml ~muelu ~zoltan2 ~teko, +ml +muelu +zoltan2 +teko]
-      - one_of: [+superlu-dist, ~superlu-dist]
-      - one_of: [+shylu, ~shylu]
+      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
+        +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
+        +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
+        +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
     xz:
       variants: +pic
     mesa:
       version: [21.3.8]
+    mpi:
+      require: mpich
+    mpich:
+      require: '~wrapperrpath ~hwloc'
+    ncurses:
+      require: '@6.3 +termlib'
     faodel:
-      require: ~tcmalloc # needed for ppc64le
+      require: "~tcmalloc"
+    tbb:
+      require: intel-tbb
+    libffi:
+      require: "@3.4.4"
+    vtk-m:
+      require: "+examples"
+    cuda:
+      require: "@11.4.4"
+
 
   specs:
   # CPU
@@ -57,6 +77,8 @@ spack:
   - argobots
   - axom
   - bolt
+  - boost
+  - bricks
   - butterflypack
   - cabana
   - caliper
@@ -67,7 +89,7 @@ spack:
   - drishti
   - dxt-explorer
   - dyninst
-  - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 ~paraview +pnetcdf +sz +unifyfs +veloc ~visit +vtkm +zfp # +paraview fails: FAILED: VTK/Filters/Statistics/CMakeFiles/FiltersStatistics-objects.dir/vtkPCAStatistics.cxx.o: /tmp/ccgvkIk5.s: Assembler messages: /tmp/ccgvkIk5.s:260012: Error: invalid machine `power10'
+  # - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 ~paraview +pnetcdf +sz +unifyfs +veloc ~visit +vtkm +zfp # +visit: libext, libxkbfile, libxrender, libxt, silo (https://github.com/spack/spack/issues/39538), cairo
   - exaworks
   - flecsi
   - flit
@@ -85,15 +107,17 @@ spack:
   - hdf5-vol-log
   - heffte +fftw
   - hpctoolkit
-  - hpx max_cpu_count=512 networking=mpi
+  - hpx networking=mpi
   - hypre
   - kokkos +openmp
   - kokkos-kernels +openmp
   - lammps
+  - lbann
   - legion
   - libnrm
   - libquo
   - libunwind
+  - loki
   - mercury
   - metall
   - mfem
@@ -106,20 +130,23 @@ spack:
   - nrm
   - nvhpc
   - omega-h
+  - openfoam
   - openmpi
   - openpmd-api
   - papi
   - papyrus
+  - paraview ~cuda ~rocm
   - parsec ~cuda
   - pdt
   - petsc
-  - phist
   - plasma
   - plumed
+  - precice
+  - pruners-ninja
   - pumi
   - py-h5py
   - py-jupyterhub
-  - py-libensemble +mpi +nlopt
+  - py-libensemble
   - py-petsc4py
   - py-warpx
   - qthreads scheduler=distrib
@@ -134,84 +161,102 @@ spack:
   - sundials
   - superlu
   - superlu-dist
-  - swig
   - swig@4.0.2-fortran
+  - sz3
   - tasmanian
-  - tau +mpi +python
-  - trilinos +belos +ifpack2 +stokhos
+  - tau +mpi +python                        # tau: has issue with `spack env depfile` build
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
   - turbine
   - umap
   - umpire
   - upcxx
   - wannier90
-  - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos~shylu
+  - xyce +mpi +shared +pymi +pymi_static_tpls
+  # INCLUDED IN ECP DAV CPU
+  - adios2
+  - ascent
+  - darshan-runtime
+  - darshan-util
+  - faodel
+  - hdf5
+  - libcatalyst
+  - parallel-netcdf
+  - paraview
+  - py-cinemasci
+  - sz
+  - unifyfs
+  - veloc
+  # - visit                                 # libext, libxkbfile, libxrender, libxt, silo (https://github.com/spack/spack/issues/39538), cairo
+  - vtk-m
+  - zfp
+  # -- 
+  # - archer                                # part of llvm +omp_tsan
+  # - dealii                                # fltk: https://github.com/spack/spack/issues/38791
+  # - geopm                                 # geopm: https://github.com/spack/spack/issues/38798
+  # - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp # py-numcodecs: gcc: error: unrecognized command line option '-mno-sse2'; did you mean '-mno-isel'? gcc: error: unrecognized command line option '-mno-avx2'
+  # - phist +mpi                            # ghost@develop: gcc-9: error: unrecognized command line option '-march=native'; did you mean '-mcpu=native'?
+  # - variorum                              # variorum: https://github.com/spack/spack/issues/38786
 
-  # CUDA
-  - amrex +cuda
-  - arborx +cuda ^kokkos +wrapper
-  - cabana +cuda ^kokkos +wrapper +cuda_lambda +cuda
-  - caliper +cuda
-  - chai ~benchmarks ~tests +cuda ^umpire ~shared
-  - ecp-data-vis-sdk +cuda cuda_arch=70 +adios2 +hdf5 ~paraview +vtkm +zfp # +paraview fails:  FAILED: VTK/Filters/Statistics/CMakeFiles/FiltersStatistics-objects.dir/vtkPCAStatistics.cxx.o; /tmp/ccjmJhb6.s: Assembler messages: /tmp/ccjmJhb6.s:260012: Error: invalid machine `power10'
-  - flecsi +cuda
+  # CUDA NOARCH
+  - bricks +cuda
+  - cabana +cuda ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=70
   - flux-core +cuda
-  - ginkgo +cuda
-  - heffte +cuda
   - hpctoolkit +cuda
-  - hpx max_cpu_count=512 +cuda
-  - hypre +cuda
-  - kokkos +wrapper +cuda
-  - kokkos-kernels +cuda ^kokkos +wrapper +cuda +cuda_lambda
-  - magma +cuda
-  - mfem +cuda
-  - mgard +serial +openmp +timing +unstructured +cuda
-  - omega-h +cuda
   - papi +cuda
-  - petsc +cuda
-  - py-torch +cuda
-  - raja +cuda
-  - slate +cuda
-  - slepc +cuda
-  - strumpack ~slate +cuda
-  - sundials +cuda
-  - superlu-dist +cuda
-  - tasmanian +cuda
-  - tau +mpi +cuda
-  - "trilinos@13.4.0: +belos +ifpack2 +stokhos +cuda"
-  - umpire ~shared +cuda
-  - parsec +cuda
-
-  # CPU FAILURES
-  # - archer                  # llvm@8
-  # - bricks                  # bricks
-  # - geopm                   # geopm
-  # - hdf5-vol-daos           # hdf5-vol-daos: vhost/vhost_user.c:65:32: error: array size missing in 'vhost_message_handlers'
-  # - loki                    # loki
-  # - precice                 # precice
-  # - pruners-ninja           # pruners-ninja
-  # - variorum                # Intel/variorum_cpuid.c:11:5: error: impossible constraint in 'asm'
+  - tau +mpi +cuda                          # tau: has issue with `spack env depfile` build
   # --
-  # bricks: VSBrick-7pt.py-Scalar-8x8x8-1:30:3: error: 'vfloat512' was not declared in this scope
-  # fltk: /usr/bin/ld: ../lib/libfltk_png.a(pngrutil.o): in function `png_read_filter_row': pngrutil.c:(.text.png_read_filter_row+0x90): undefined reference to `png_init_filter_functions_vsx'
-  # geopm: libtool.m4: error: problem compiling CXX test program
-  # llvm@8: clang/lib/Lex/Lexer.cpp:2547:34: error: ISO C++ forbids declaration of 'type name' with no type [-fpermissive]
-  # loki: include/loki/SmallObj.h:462:57: error: ISO C++17 does not allow dynamic exception specifications
-  # precice: /tmp/ccYNMwgE.s: Assembler messages: /tmp/ccYNMwgE.s:278115: Error: invalid machine `power10'
-  # pruners-ninja: test/ninja_test_util.c:34: multiple definition of `a';
+  # - legion +cuda                          # legion: needs NVIDIA driver
 
-  # CUDA FAILURES
-  # - bricks +cuda              # bricks
-  # - dealii +cuda              # fltk
+  # CUDA 70
+  - amrex +cuda cuda_arch=70
+  - arborx +cuda cuda_arch=70 ^kokkos +wrapper
+  - caliper +cuda cuda_arch=70
+  - chai ~benchmarks ~tests +cuda cuda_arch=70 ^umpire ~shared
+  - ecp-data-vis-sdk ~rocm +adios2 ~ascent +hdf5 +vtkm +zfp ~paraview +cuda cuda_arch=70
+  - flecsi +cuda cuda_arch=70
+  - ginkgo +cuda cuda_arch=70
+  - heffte +cuda cuda_arch=70
+  - hpx +cuda cuda_arch=70
+  - hypre +cuda cuda_arch=70
+  - kokkos +wrapper +cuda cuda_arch=70
+  - kokkos-kernels +cuda cuda_arch=70 ^kokkos +wrapper +cuda cuda_arch=70
+  - magma +cuda cuda_arch=70
+  - mfem +cuda cuda_arch=70
+  - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=70
+  - omega-h +cuda cuda_arch=70
+  - parsec +cuda cuda_arch=70
+  - petsc +cuda cuda_arch=70
+  - raja +cuda cuda_arch=70
+  - slate +cuda cuda_arch=70
+  - slepc +cuda cuda_arch=70
+  - strumpack ~slate +cuda cuda_arch=70
+  - sundials +cuda cuda_arch=70
+  - superlu-dist +cuda cuda_arch=70
+  - tasmanian +cuda cuda_arch=70
+  - umpire ~shared +cuda cuda_arch=70
+  # INCLUDED IN ECP DAV CUDA
+  - adios2 +cuda cuda_arch=70
+  # - ascent +cuda cuda_arch=70             # ascent: https://github.com/spack/spack/issues/38045
+  - paraview +cuda cuda_arch=70
+  - vtk-m +cuda cuda_arch=70
+  - zfp +cuda cuda_arch=70
   # --
-  # bricks: VSBrick-7pt.py-Scalar-8x8x8-1:30:3: error: 'vfloat512' was not declared in this scope
-
+  # - axom +cuda cuda_arch=70               # axom: https://github.com/spack/spack/issues/29520
+  # - cusz +cuda cuda_arch=70               # cusz: https://github.com/spack/spack/issues/38787
+  # - dealii +cuda cuda_arch=70             # fltk: https://github.com/spack/spack/issues/38791
+  # - lammps +cuda cuda_arch=70             # lammps: needs NVIDIA driver
+  # - lbann +cuda cuda_arch=70              # lbann: https://github.com/spack/spack/issues/38788
+  # - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +cusz +mgard +cuda cuda_arch=70 ^cusz +cuda cuda_arch=70 # depends_on("cuda@11.7.1:", when="+cuda")
+  # - py-torch +cuda cuda_arch=70           # skipped
+  # - trilinos +cuda cuda_arch=70           # trilinos: https://github.com/trilinos/Trilinos/issues/11630
+  # - upcxx +cuda cuda_arch=70              # upcxx: needs NVIDIA driver
 
   mirrors: { "mirror": "s3://spack-binaries/develop/e4s-power" }
 
   ci:
     pipeline-gen:
     - build-job:
-        image: ecpe4s/ubuntu20.04-runner-ppc64le:2023-01-01
+        image: ecpe4s/ubuntu20.04-runner-ppc64-gcc-11.4:2023.08.01
 
   cdash:
     build-group: E4S Power
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml
new file mode 100644
index 0000000000..dba404e9b3
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml
@@ -0,0 +1,346 @@
+spack:
+  view: false
+
+  concretizer:
+    reuse: false
+    unify: false
+
+  compilers:
+  - compiler:
+      spec: gcc@=11.4.0
+      paths:
+        cc: /usr/bin/gcc
+        cxx: /usr/bin/g++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      flags: {}
+      operating_system: ubuntu20.04
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+
+  packages:
+    all:
+      require: '%gcc target=x86_64_v3'
+      providers:
+        blas: [openblas]
+        mpi: [mpich]
+      variants: +mpi
+    binutils:
+      variants: +ld +gold +headers +libiberty ~nls
+    elfutils:
+      variants: +bzip2 ~nls +xz
+    hdf5:
+      variants: +fortran +hl +shared
+    libfabric:
+      variants: fabrics=sockets,tcp,udp,rxm
+    libunwind:
+      variants: +pic +xz
+    openblas:
+      variants: threads=openmp
+    trilinos:
+      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
+        +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
+        +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
+        +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+    xz:
+      variants: +pic
+    mesa:
+      version: [21.3.8]
+    mpi:
+      require: mpich
+    mpich:
+      require: '~wrapperrpath ~hwloc'
+    ncurses:
+      require: '@6.3 +termlib'
+    tbb:
+      require: intel-tbb
+    boost:
+      version: [1.79.0]
+      variants: +atomic +chrono +container +date_time +exception +filesystem +graph
+        +iostreams +locale +log +math +mpi +multithreaded +program_options +random
+        +regex +serialization +shared +signals +stacktrace +system +test +thread +timer
+        cxxstd=17 visibility=global
+    libffi:
+      require: "@3.4.4"
+    vtk-m:
+      require: "+examples"
+    cuda:
+      version: [11.8.0]
+    paraview:
+      # Don't build GUI support or GLX rendering for HPC/container deployments
+      require: "@5.11 ~qt+osmesa"
+
+    # ROCm 5.4.3
+    comgr:
+      buildable: false
+      externals:
+      - spec: comgr@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    hip-rocclr:
+      buildable: false
+      externals:
+      - spec: hip-rocclr@5.4.3
+        prefix: /opt/rocm-5.4.3/hip
+    hipblas:
+      buildable: false
+      externals:
+      - spec: hipblas@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    hipcub:
+      buildable: false
+      externals:
+      - spec: hipcub@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    hipfft:
+      buildable: false
+      externals:
+      - spec: hipfft@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    hipsparse:
+      buildable: false
+      externals:
+      - spec: hipsparse@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    miopen-hip:
+      buildable: false
+      externals:
+      - spec: hip-rocclr@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    miopengemm:
+      buildable: false
+      externals:
+      - spec: miopengemm@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rccl:
+      buildable: false
+      externals:
+      - spec: rccl@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocblas:
+      buildable: false
+      externals:
+      - spec: rocblas@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocfft:
+      buildable: false
+      externals:
+      - spec: rocfft@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocm-clang-ocl:
+      buildable: false
+      externals:
+      - spec: rocm-clang-ocl@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocm-cmake:
+      buildable: false
+      externals:
+      - spec: rocm-cmake@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocm-dbgapi:
+      buildable: false
+      externals:
+      - spec: rocm-dbgapi@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocm-debug-agent:
+      buildable: false
+      externals:
+      - spec: rocm-debug-agent@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocm-device-libs:
+      buildable: false
+      externals:
+      - spec: rocm-device-libs@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocm-gdb:
+      buildable: false
+      externals:
+      - spec: rocm-gdb@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocm-opencl:
+      buildable: false
+      externals:
+      - spec: rocm-opencl@5.4.3
+        prefix: /opt/rocm-5.4.3/opencl
+    rocm-smi-lib:
+      buildable: false
+      externals:
+      - spec: rocm-smi-lib@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    hip:
+      buildable: false
+      externals:
+      - spec: hip@5.4.3
+        prefix: /opt/rocm-5.4.3
+        extra_attributes:
+          compilers:
+            c: /opt/rocm-5.4.3/llvm/bin/clang++
+            c++: /opt/rocm-5.4.3/llvm/bin/clang++
+            hip: /opt/rocm-5.4.3/hip/bin/hipcc
+    hipify-clang:
+      buildable: false
+      externals:
+      - spec: hipify-clang@5.4.3
+        prefix: /opt/rocm-5.4.3
+    llvm-amdgpu:
+      buildable: false
+      externals:
+      - spec: llvm-amdgpu@5.4.3
+        prefix: /opt/rocm-5.4.3/llvm
+        extra_attributes:
+          compilers:
+            c: /opt/rocm-5.4.3/llvm/bin/clang++
+            cxx: /opt/rocm-5.4.3/llvm/bin/clang++
+    hsakmt-roct:
+      buildable: false
+      externals:
+      - spec: hsakmt-roct@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    hsa-rocr-dev:
+      buildable: false
+      externals:
+      - spec: hsa-rocr-dev@5.4.3
+        prefix: /opt/rocm-5.4.3/
+        extra_atributes:
+          compilers:
+            c: /opt/rocm-5.4.3/llvm/bin/clang++
+            cxx: /opt/rocm-5.4.3/llvm/bin/clang++
+    roctracer-dev-api:
+      buildable: false
+      externals:
+      - spec: roctracer-dev-api@5.4.3
+        prefix: /opt/rocm-5.4.3
+    roctracer-dev:
+      buildable: false
+      externals:
+      - spec: roctracer-dev@4.5.3
+        prefix: /opt/rocm-5.4.3
+    rocprim:
+      buildable: false
+      externals:
+      - spec: rocprim@5.4.3
+        prefix: /opt/rocm-5.4.3
+    rocrand:
+      buildable: false
+      externals:
+      - spec: rocrand@5.4.3
+        prefix: /opt/rocm-5.4.3
+    hipsolver:
+      buildable: false
+      externals:
+      - spec: hipsolver@5.4.3
+        prefix: /opt/rocm-5.4.3 
+    rocsolver:
+      buildable: false
+      externals:
+      - spec: rocsolver@5.4.3
+        prefix: /opt/rocm-5.4.3
+    rocsparse:
+      buildable: false
+      externals:
+      - spec: rocsparse@5.4.3
+        prefix: /opt/rocm-5.4.3
+    rocthrust:
+      buildable: false
+      externals:
+      - spec: rocthrust@5.4.3
+        prefix: /opt/rocm-5.4.3
+    rocprofiler-dev:
+      buildable: false
+      externals:
+      - spec: rocprofiler-dev@5.4.3
+        prefix: /opt/rocm-5.4.3
+
+  specs:
+  # ROCM NOARCH
+  - hpctoolkit +rocm
+  - tau +mpi +rocm   # tau: has issue with `spack env depfile` build
+
+  # ROCM 908
+  - amrex +rocm amdgpu_target=gfx908
+  - arborx +rocm amdgpu_target=gfx908
+  - cabana +rocm amdgpu_target=gfx908
+  - caliper +rocm amdgpu_target=gfx908
+  - chai ~benchmarks +rocm amdgpu_target=gfx908
+  - ecp-data-vis-sdk +paraview +vtkm +rocm amdgpu_target=gfx908
+  - gasnet +rocm amdgpu_target=gfx908
+  - ginkgo +rocm amdgpu_target=gfx908
+  - heffte +rocm amdgpu_target=gfx908
+  - hpx +rocm amdgpu_target=gfx908
+  - hypre +rocm amdgpu_target=gfx908
+  - kokkos +rocm amdgpu_target=gfx908
+  - legion +rocm amdgpu_target=gfx908
+  - magma ~cuda +rocm amdgpu_target=gfx908
+  - mfem +rocm amdgpu_target=gfx908
+  - petsc +rocm amdgpu_target=gfx908
+  - raja ~openmp +rocm amdgpu_target=gfx908
+  - slate +rocm amdgpu_target=gfx908
+  - slepc +rocm amdgpu_target=gfx908 ^petsc +rocm amdgpu_target=gfx908
+  - strumpack ~slate +rocm amdgpu_target=gfx908
+  - sundials +rocm amdgpu_target=gfx908
+  - superlu-dist +rocm amdgpu_target=gfx908
+  - tasmanian ~openmp +rocm amdgpu_target=gfx908
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack ~ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu ~stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long +rocm amdgpu_target=gfx908
+  - umpire +rocm amdgpu_target=gfx908
+  - upcxx +rocm amdgpu_target=gfx908
+  # INCLUDED IN ECP DAV ROCM
+  # - hdf5
+  # - hdf5-vol-async
+  # - hdf5-vol-cache
+  # - hdf5-vol-log
+  # - libcatalyst
+  - paraview +rocm amdgpu_target=gfx908
+  # - vtk-m ~openmp +rocm amdgpu_target=gfx908  # vtk-m: https://github.com/spack/spack/issues/40268
+  # --
+  # - lbann ~cuda +rocm amdgpu_target=gfx908    # aluminum: https://github.com/spack/spack/issues/38807
+  # - papi +rocm amdgpu_target=gfx908           # papi: https://github.com/spack/spack/issues/27898
+
+  # ROCM 90a
+  - amrex +rocm amdgpu_target=gfx90a
+  - arborx +rocm amdgpu_target=gfx90a
+  - cabana +rocm amdgpu_target=gfx90a
+  - caliper +rocm amdgpu_target=gfx90a
+  - chai ~benchmarks +rocm amdgpu_target=gfx90a
+  - ecp-data-vis-sdk +paraview +vtkm +rocm amdgpu_target=gfx90a
+  - gasnet +rocm amdgpu_target=gfx90a
+  - ginkgo +rocm amdgpu_target=gfx90a
+  - heffte +rocm amdgpu_target=gfx90a
+  - hpx +rocm amdgpu_target=gfx90a
+  - hypre +rocm amdgpu_target=gfx90a
+  - kokkos +rocm amdgpu_target=gfx90a
+  - legion +rocm amdgpu_target=gfx90a
+  - magma ~cuda +rocm amdgpu_target=gfx90a
+  - mfem +rocm amdgpu_target=gfx90a
+  - petsc +rocm amdgpu_target=gfx90a
+  - raja ~openmp +rocm amdgpu_target=gfx90a
+  - slate +rocm amdgpu_target=gfx90a
+  - slepc +rocm amdgpu_target=gfx90a ^petsc +rocm amdgpu_target=gfx90a
+  - strumpack ~slate +rocm amdgpu_target=gfx90a
+  - sundials +rocm amdgpu_target=gfx90a
+  - superlu-dist +rocm amdgpu_target=gfx90a
+  - tasmanian ~openmp +rocm amdgpu_target=gfx90a
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack ~ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu ~stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long +rocm amdgpu_target=gfx90a
+  - umpire +rocm amdgpu_target=gfx90a
+  - upcxx +rocm amdgpu_target=gfx90a
+  # INCLUDED IN ECP DAV ROCM
+  # - hdf5
+  # - hdf5-vol-async
+  # - hdf5-vol-cache
+  # - hdf5-vol-log
+  # - libcatalyst
+  - paraview +rocm amdgpu_target=gfx90a
+  # - vtk-m ~openmp +rocm amdgpu_target=gfx90a  # vtk-m: https://github.com/spack/spack/issues/40268
+  # --
+  # - lbann ~cuda +rocm amdgpu_target=gfx90a    # aluminum: https://github.com/spack/spack/issues/38807
+  # - papi +rocm amdgpu_target=gfx90a           # papi: https://github.com/spack/spack/issues/27898
+
+  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-rocm-external" }
+
+  ci:
+    pipeline-gen:
+    - build-job:
+        image: "ecpe4s/ubuntu20.04-runner-amd64-gcc-11.4-rocm5.4.3:2023.08.01"
+
+  cdash:
+    build-group: E4S ROCm External
\ No newline at end of file
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
index a10b279f53..8b5947da34 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
@@ -1,21 +1,34 @@
 spack:
   view: false
+
+  concretizer:
+    reuse: false
+    unify: false
+
+  compilers:
+  - compiler:
+      spec: gcc@=11.4.0
+      paths:
+        cc: /usr/bin/gcc
+        cxx: /usr/bin/g++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      flags: {}
+      operating_system: ubuntu20.04
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+
   packages:
     all:
-      compiler: [gcc@11.1.0]
+      require: '%gcc target=x86_64_v3'
       providers:
         blas: [openblas]
         mpi: [mpich]
-      require: target=x86_64_v3
-      variants: +mpi amdgpu_target=gfx90a cuda_arch=80
-    tbb:
-      require: "intel-tbb"
+      variants: +mpi
     binutils:
       variants: +ld +gold +headers +libiberty ~nls
-    boost:
-      variants: +python +filesystem +iostreams +system
-    cuda:
-      version: [11.7.0]
     elfutils:
       variants: +bzip2 ~nls +xz
     hdf5:
@@ -24,29 +37,40 @@ spack:
       variants: fabrics=sockets,tcp,udp,rxm
     libunwind:
       variants: +pic +xz
-    mpich:
-      variants: ~wrapperrpath
-    ncurses:
-      variants: +termlib
     openblas:
       variants: threads=openmp
-    paraview:
-      # Don't build GUI support or GLX rendering for HPC/container deployments
-      require: "@5.11 ~qt+osmesa"
     trilinos:
-      require:
-      - one_of: [+amesos +amesos2 +anasazi +aztec +boost +epetra +epetraext +ifpack
-            +intrepid +intrepid2 +isorropia +kokkos +minitensor +nox +piro +phalanx
-            +rol +rythmos +sacado +stk +shards +stratimikos +tempus +tpetra
-            +trilinoscouplings +zoltan]
-      - one_of: [gotype=long_long, gotype=all]
-      - one_of: [~ml ~muelu ~zoltan2 ~teko, +ml +muelu +zoltan2 +teko]
-      - one_of: [+superlu-dist, ~superlu-dist]
-      - one_of: [+shylu, ~shylu]
+      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
+        +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
+        +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
+        +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
     xz:
       variants: +pic
     mesa:
       version: [21.3.8]
+    mpi:
+      require: mpich
+    mpich:
+      require: '~wrapperrpath ~hwloc'
+    ncurses:
+      require: '@6.3 +termlib'
+    tbb:
+      require: intel-tbb
+    boost:
+      version: [1.79.0]
+      variants: +atomic +chrono +container +date_time +exception +filesystem +graph
+        +iostreams +locale +log +math +mpi +multithreaded +program_options +random
+        +regex +serialization +shared +signals +stacktrace +system +test +thread +timer
+        cxxstd=17 visibility=global
+    libffi:
+      require: "@3.4.4"
+    vtk-m:
+      require: "+examples"
+    cuda:
+      version: [11.8.0]
+    paraview:
+      # Don't build GUI support or GLX rendering for HPC/container deployments
+      require: "@5.11 ~qt+osmesa"
 
   specs:
   # CPU
@@ -55,13 +79,12 @@ spack:
   - aml
   - amrex
   - arborx
-  - archer
   - argobots
   - axom
   - bolt
-  - bricks
+  - boost
+  - bricks ~cuda
   - butterflypack
-  - boost +python +filesystem +iostreams +system
   - cabana
   - caliper
   - chai ~benchmarks ~tests
@@ -72,7 +95,7 @@ spack:
   - drishti
   - dxt-explorer
   - dyninst
-  - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 +paraview +pnetcdf +sz +unifyfs +veloc ~visit +vtkm +zfp ^hdf5@1.14
+  - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 +paraview +pnetcdf +sz +unifyfs +veloc +visit +vtkm +zfp                                                                                       # adios2~cuda, ascent~cuda, darshan-runtime, darshan-util, faodel, hdf5, libcatalyst, parallel-netcdf, paraview~cuda, py-cinemasci, sz, unifyfs, veloc, visit, vtk-m, zfp
   - exaworks
   - flecsi
   - flit
@@ -83,24 +106,25 @@ spack:
   - globalarrays
   - gmp
   - gotcha
-  - gptune
+  - gptune ~mpispawn
   - h5bench
   - hdf5-vol-async
   - hdf5-vol-cache
   - hdf5-vol-log
   - heffte +fftw
   - hpctoolkit
-  - hpx max_cpu_count=512 networking=mpi
+  - hpx networking=mpi
   - hypre
   - kokkos +openmp
   - kokkos-kernels +openmp
   - lammps
+  - lbann
   - legion
   - libnrm
-  - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed
-    +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +mgard
+  - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp
   - libquo
   - libunwind
+  - loki
   - mercury
   - metall
   - mfem
@@ -113,6 +137,7 @@ spack:
   - nrm
   - nvhpc
   - omega-h
+  - openfoam
   - openmpi
   - openpmd-api
   - papi
@@ -124,16 +149,17 @@ spack:
   - plasma
   - plumed
   - precice
+  - pruners-ninja
   - pumi
-  - py-h5py +mpi
-  - py-h5py ~mpi
+  - py-h5py
   - py-jupyterhub
-  - py-libensemble +mpi +nlopt
+  - py-libensemble
   - py-petsc4py
   - py-warpx
   - qthreads scheduler=distrib
   - quantum-espresso
   - raja
+  - rempi
   - scr
   - slate ~cuda
   - slepc
@@ -142,107 +168,226 @@ spack:
   - sundials
   - superlu
   - superlu-dist
-  - swig
   - swig@4.0.2-fortran
   - sz3
   - tasmanian
   - tau +mpi +python
-  - trilinos@13.0.1 +belos +ifpack2 +stokhos
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
   - turbine
   - umap
   - umpire
   - upcxx
   - variorum
-  - veloc
   - wannier90
-  - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos +shylu
+  - xyce +mpi +shared +pymi +pymi_static_tpls
+  # INCLUDED IN ECP DAV CPU
+  - adios2
+  - ascent
+  - darshan-runtime
+  - darshan-util
+  - faodel
+  - hdf5
+  - libcatalyst
+  - parallel-netcdf
+  - paraview
+  - py-cinemasci
+  - sz
+  - unifyfs
+  - veloc
+  # - visit                           # silo: https://github.com/spack/spack/issues/39538
+  - vtk-m
+  - zfp
+  # --
+  # - archer                          # submerged into llvm +libomp_tsan
+  # - geopm                           # geopm: https://github.com/spack/spack/issues/38795
 
-  # CUDA
-  - amrex +cuda
-  - arborx +cuda ^kokkos +wrapper
+  # CUDA NOARCH
   - bricks +cuda
-  - cabana +cuda ^kokkos +wrapper +cuda_lambda +cuda
-  - caliper +cuda
-  - chai ~benchmarks ~tests +cuda ^umpire ~shared
-  - cusz +cuda
-  - dealii +cuda
-  - ecp-data-vis-sdk +cuda ~ascent +adios2 +hdf5 +paraview +sz +vtkm +zfp  ^hdf5@1.14 # Removing ascent because RAJA build failure
-  - flecsi +cuda
   - flux-core +cuda
-  - ginkgo +cuda
-  - heffte +cuda
   - hpctoolkit +cuda
-  - hpx max_cpu_count=512 +cuda
-  - hypre +cuda
-  - kokkos +wrapper +cuda
-  - kokkos-kernels +cuda ^kokkos +wrapper +cuda +cuda_lambda
-  - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua
-    +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +cusz
-    +mgard +cuda ^cusz +cuda
-  - magma +cuda
-  - mfem +cuda
-  - mgard +serial +openmp +timing +unstructured +cuda
-  - omega-h +cuda
   - papi +cuda
-  - petsc +cuda
-  - py-torch +cuda
-  - raja +cuda
-  - slate +cuda
-  - slepc +cuda
-  - strumpack ~slate +cuda
-  - sundials +cuda
-  - superlu-dist +cuda
-  - tasmanian +cuda
   - tau +mpi +cuda
-  - "trilinos@13.4.0: +belos +ifpack2 +stokhos +cuda"
-  - umpire ~shared +cuda
+  # --
+  # - legion +cuda                    # legion: needs NVIDIA driver
+
+  # CUDA 80
+  - amrex +cuda cuda_arch=80
+  - arborx +cuda cuda_arch=80 ^kokkos +wrapper
+  - cabana +cuda cuda_arch=80 ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=80
+  - caliper +cuda cuda_arch=80
+  - chai ~benchmarks ~tests +cuda cuda_arch=80 ^umpire ~shared
+  - cusz +cuda cuda_arch=80
+  - dealii +cuda cuda_arch=80
+  - ecp-data-vis-sdk ~rocm +adios2 ~ascent +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=80 # +ascent fails because fides fetch error
+  - flecsi +cuda cuda_arch=80
+  - ginkgo +cuda cuda_arch=80
+  - heffte +cuda cuda_arch=80
+  - hpx +cuda cuda_arch=80
+  - hypre +cuda cuda_arch=80
+  - kokkos +wrapper +cuda cuda_arch=80
+  - kokkos-kernels +cuda cuda_arch=80 ^kokkos +wrapper +cuda cuda_arch=80
+  - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +cusz +mgard +cuda cuda_arch=80 ^cusz +cuda cuda_arch=80
+  - magma +cuda cuda_arch=80
+  - mfem +cuda cuda_arch=80
+  - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=80
+  - omega-h +cuda cuda_arch=80
+  - parsec +cuda cuda_arch=80
+  - petsc +cuda cuda_arch=80
+  - py-torch +cuda cuda_arch=80
+  - raja +cuda cuda_arch=80
+  - slate +cuda cuda_arch=80
+  - slepc +cuda cuda_arch=80
+  - strumpack ~slate +cuda cuda_arch=80
+  - sundials +cuda cuda_arch=80
+  - superlu-dist +cuda cuda_arch=80
+  - tasmanian +cuda cuda_arch=80
+  - trilinos +cuda cuda_arch=80
+  - umpire ~shared +cuda cuda_arch=80
+  # INCLUDED IN ECP DAV CUDA
+  # - adios2 +cuda cuda_arch=80
+  # - ascent +cuda cuda_arch=80       # ascent: https://github.com/spack/spack/issues/38045
+  # - paraview +cuda cuda_arch=80
+  # - vtk-m +cuda cuda_arch=80
+  # - zfp +cuda cuda_arch=80
+  # --
+  # - lammps +cuda cuda_arch=80       # lammps: needs NVIDIA driver
+  # - upcxx +cuda cuda_arch=80        # upcxx: needs NVIDIA driver
+  # - axom +cuda cuda_arch=80         # axom: https://github.com/spack/spack/issues/29520
+  # - lbann +cuda cuda_arch=80        # lbann: https://github.com/spack/spack/issues/38788
+
+  # CUDA 90
+  - amrex +cuda cuda_arch=90
+  - arborx +cuda cuda_arch=90 ^kokkos +wrapper
+  - cabana +cuda cuda_arch=90 ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=90
+  - caliper +cuda cuda_arch=90
+  - chai ~benchmarks ~tests +cuda cuda_arch=90 ^umpire ~shared
+  - cusz +cuda cuda_arch=90
+  - flecsi +cuda cuda_arch=90
+  - ginkgo +cuda cuda_arch=90
+  - heffte +cuda cuda_arch=90
+  - hpx +cuda cuda_arch=90
+  - kokkos +wrapper +cuda cuda_arch=90
+  - kokkos-kernels +cuda cuda_arch=90 ^kokkos +wrapper +cuda cuda_arch=90
+  - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +cusz +mgard +cuda cuda_arch=90 ^cusz +cuda cuda_arch=90
+  - magma +cuda cuda_arch=90
+  - mfem +cuda cuda_arch=90
+  - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=90
+  - parsec +cuda cuda_arch=90
+  - petsc +cuda cuda_arch=90
+  - py-torch +cuda cuda_arch=90
+  - raja +cuda cuda_arch=90
+  - slate +cuda cuda_arch=90
+  - slepc +cuda cuda_arch=90
+  - strumpack ~slate +cuda cuda_arch=90
+  - sundials +cuda cuda_arch=90
+  - superlu-dist +cuda cuda_arch=90
+  - trilinos +cuda cuda_arch=90
+  - umpire ~shared +cuda cuda_arch=90
+  # INCLUDED IN ECP DAV CUDA
+  - adios2 +cuda cuda_arch=90
+  # - ascent +cuda cuda_arch=90       # ascent: https://github.com/spack/spack/issues/38045
+  # - paraview +cuda cuda_arch=90     # paraview: InstallError: Incompatible cuda_arch=90
+  - vtk-m +cuda cuda_arch=90
+  - zfp +cuda cuda_arch=90
+  # --
+  # - axom +cuda cuda_arch=90         # axom: https://github.com/spack/spack/issues/29520
+  # - dealii +cuda cuda_arch=90       # dealii: https://github.com/spack/spack/issues/39532
+  # - ecp-data-vis-sdk ~rocm +adios2 +ascent +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=90 # paraview: incompatible cuda_arch; vtk-m: CMake Error at CMake/VTKmWrappers.cmake:413 (message): vtkm_cont needs to be built STATIC as CUDA doesn't support virtual methods across dynamic library boundaries.  You need to set the CMake opt ion BUILD_SHARED_LIBS to `OFF` or (better) turn VTKm_NO_DEPRECATED_VIRTUAL to `ON`.
+  # - hypre +cuda cuda_arch=90        # concretizer: hypre +cuda requires cuda@:11, but cuda_arch=90 requires cuda@12:
+  # - lammps +cuda cuda_arch=90       # lammps: needs NVIDIA driver
+  # - lbann +cuda cuda_arch=90        # concretizer: Cannot select a single "version" for package "lbann"
+  # - omega-h +cuda cuda_arch=90      # omega-h: https://github.com/spack/spack/issues/39535
+  # - tasmanian +cuda cuda_arch=90    # tasmanian: conflicts with cuda@12
+  # - upcxx +cuda cuda_arch=90        # upcxx: needs NVIDIA driver
 
-  # ROCm
-  - amrex +rocm
-  - arborx +rocm
-  - cabana +rocm
-  - caliper +rocm
-  - chai ~benchmarks +rocm
-  - ecp-data-vis-sdk +adios2 +hdf5 +paraview +pnetcdf +sz +vtkm +zfp +rocm ^hdf5@1.14 # Excludes ascent for now due to C++ standard issues
-  - gasnet +rocm
-  - ginkgo +rocm
-  - heffte +rocm
+  # ROCM NOARCH
   - hpctoolkit +rocm
-  - hpx max_cpu_count=512 +rocm
-  - hypre +rocm
-  - kokkos +rocm
-  - magma ~cuda +rocm
-  - mfem +rocm
-  - papi +rocm
-  - petsc +rocm
-  - raja ~openmp +rocm
-  - slate +rocm
-  - slepc +rocm ^petsc +rocm
-  - strumpack ~slate +rocm
-  - sundials +rocm
-  - superlu-dist +rocm
-  - tasmanian ~openmp +rocm
-  - tau +mpi +rocm
-  - "trilinos@13.4.0: +belos ~ifpack2 ~stokhos +rocm"
-  - umpire +rocm
-  - upcxx +rocm
+  - tau +mpi +rocm   # tau: has issue with `spack env depfile` build
 
-  # CPU failures
-  # - geopm                 # /usr/include/x86_64-linux-gnu/bits/string_fortified.h:95:10: error:'__builtin_strncpy' specified bound 512 equals destination size [-Werror=stringop-truncation]
-  # - hdf5-vol-daos         # hdf5-vol-daos: vhost/vhost_user.c:65:32: error: array size missing in 'vhost_message_handlers'
-  # - loki                  # ../include/loki/Singleton.h:158:14: warning: 'template<class> class std::auto_ptr' is deprecated: use 'std::unique_ptr' instead [-Wdeprecated-declarations]
-  # - pruners-ninja         # test/ninja_test_util.c:34: multiple definition of `a';
-  # - rempi                 # rempi_message_manager.h:53:3: error: 'string' does not name a type
+  # ROCM 908
+  - amrex +rocm amdgpu_target=gfx908
+  - arborx +rocm amdgpu_target=gfx908
+  - cabana +rocm amdgpu_target=gfx908
+  - caliper +rocm amdgpu_target=gfx908
+  - chai ~benchmarks +rocm amdgpu_target=gfx908
+  - ecp-data-vis-sdk +paraview +vtkm +rocm amdgpu_target=gfx908
+  - gasnet +rocm amdgpu_target=gfx908
+  - ginkgo +rocm amdgpu_target=gfx908
+  - heffte +rocm amdgpu_target=gfx908
+  - hpx +rocm amdgpu_target=gfx908
+  - hypre +rocm amdgpu_target=gfx908
+  - kokkos +rocm amdgpu_target=gfx908
+  - legion +rocm amdgpu_target=gfx908
+  - magma ~cuda +rocm amdgpu_target=gfx908
+  - mfem +rocm amdgpu_target=gfx908
+  - petsc +rocm amdgpu_target=gfx908
+  - raja ~openmp +rocm amdgpu_target=gfx908
+  - slate +rocm amdgpu_target=gfx908
+  - slepc +rocm amdgpu_target=gfx908 ^petsc +rocm amdgpu_target=gfx908
+  - strumpack ~slate +rocm amdgpu_target=gfx908
+  - sundials +rocm amdgpu_target=gfx908
+  - superlu-dist +rocm amdgpu_target=gfx908
+  - tasmanian ~openmp +rocm amdgpu_target=gfx908
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack ~ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu ~stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long +rocm amdgpu_target=gfx908
+  - umpire +rocm amdgpu_target=gfx908
+  - upcxx +rocm amdgpu_target=gfx908
+  # INCLUDED IN ECP DAV ROCM
+  # - hdf5
+  # - hdf5-vol-async
+  # - hdf5-vol-cache
+  # - hdf5-vol-log
+  # - libcatalyst
+  - paraview +rocm amdgpu_target=gfx908
+  # - vtk-m ~openmp +rocm amdgpu_target=gfx908  # vtk-m: https://github.com/spack/spack/issues/40268
+  # --
+  # - lbann ~cuda +rocm amdgpu_target=gfx908    # aluminum: https://github.com/spack/spack/issues/38807
+  # - papi +rocm amdgpu_target=gfx908           # papi: https://github.com/spack/spack/issues/27898
 
-  # CUDA failures
-  # - parsec +cuda          # parsec/mca/device/cuda/transfer.c:168: multiple definition of `parsec_CUDA_d2h_max_flows';
+  # ROCM 90a
+  - amrex +rocm amdgpu_target=gfx90a
+  - arborx +rocm amdgpu_target=gfx90a
+  - cabana +rocm amdgpu_target=gfx90a
+  - caliper +rocm amdgpu_target=gfx90a
+  - chai ~benchmarks +rocm amdgpu_target=gfx90a
+  - ecp-data-vis-sdk +paraview +vtkm +rocm amdgpu_target=gfx90a
+  - gasnet +rocm amdgpu_target=gfx90a
+  - ginkgo +rocm amdgpu_target=gfx90a
+  - heffte +rocm amdgpu_target=gfx90a
+  - hpx +rocm amdgpu_target=gfx90a
+  - hypre +rocm amdgpu_target=gfx90a
+  - kokkos +rocm amdgpu_target=gfx90a
+  - legion +rocm amdgpu_target=gfx90a
+  - magma ~cuda +rocm amdgpu_target=gfx90a
+  - mfem +rocm amdgpu_target=gfx90a
+  - petsc +rocm amdgpu_target=gfx90a
+  - raja ~openmp +rocm amdgpu_target=gfx90a
+  - slate +rocm amdgpu_target=gfx90a
+  - slepc +rocm amdgpu_target=gfx90a ^petsc +rocm amdgpu_target=gfx90a
+  - strumpack ~slate +rocm amdgpu_target=gfx90a
+  - sundials +rocm amdgpu_target=gfx90a
+  - superlu-dist +rocm amdgpu_target=gfx90a
+  - tasmanian ~openmp +rocm amdgpu_target=gfx90a
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack ~ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu ~stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long +rocm amdgpu_target=gfx90a
+  - umpire +rocm amdgpu_target=gfx90a
+  - upcxx +rocm amdgpu_target=gfx90a
+  # INCLUDED IN ECP DAV ROCM
+  # - hdf5
+  # - hdf5-vol-async
+  # - hdf5-vol-cache
+  # - hdf5-vol-log
+  # - libcatalyst
+  - paraview +rocm amdgpu_target=gfx90a
+  # - vtk-m ~openmp +rocm amdgpu_target=gfx90a  # vtk-m: https://github.com/spack/spack/issues/40268
+  # --
+  # - lbann ~cuda +rocm amdgpu_target=gfx90a    # aluminum: https://github.com/spack/spack/issues/38807
+  # - papi +rocm amdgpu_target=gfx90a           # papi: https://github.com/spack/spack/issues/27898
 
   mirrors: { "mirror": "s3://spack-binaries/develop/e4s" }
 
   ci:
     pipeline-gen:
     - build-job:
-        image: "ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01"
+        image: "ecpe4s/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01"
 
   cdash:
-    build-group: E4S
+    build-group: E4S
\ No newline at end of file