e4s ci: enable some disabled specs (#44934)

* e4s ci: enable some disabled specs

* comment out cp2k +cuda due to unsupported cuda_arch

* comment out dealii+cuda due to concretize error

* work through concretize errors

* e4s: comment out failing builds

* e4s stack: disabled non-building specs

* comment out failing specs

* comment out failing specs

* cleanup comments

2 files changed, 86 insertions, 127 deletions

diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml
index 36bf8037b6..ecf7772ad4 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml
@@ -215,9 +215,13 @@ spack:
   # CUDA 75
   - amrex +cuda cuda_arch=75
   - arborx +cuda cuda_arch=75 ^kokkos +wrapper
+  - axom +cuda cuda_arch=75
   - cabana +cuda cuda_arch=75 ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=75
   - caliper +cuda cuda_arch=75
   - chai +cuda cuda_arch=75 ^umpire ~shared
+  - cusz +cuda cuda_arch=75
+  - dealii +cuda cuda_arch=75
+  - ecp-data-vis-sdk +adios2 +hdf5 +vtkm +zfp ~paraview +cuda cuda_arch=75  # # +paraview: job killed oom?
   - flecsi +cuda cuda_arch=75
   - ginkgo +cuda cuda_arch=75
   - gromacs +cuda cuda_arch=75
@@ -232,8 +236,10 @@ spack:
   - omega-h +cuda cuda_arch=75
   - parsec +cuda cuda_arch=75
   - petsc +cuda cuda_arch=75
+  - py-torch +cuda cuda_arch=75
   - raja +cuda cuda_arch=75
   - slate +cuda cuda_arch=75
+  - slepc +cuda cuda_arch=75
   - strumpack ~slate +cuda cuda_arch=75
   - sundials +cuda cuda_arch=75
   - superlu-dist +cuda cuda_arch=75
@@ -246,25 +252,23 @@ spack:
   - vtk-m +cuda cuda_arch=75
   - zfp +cuda cuda_arch=75
   # --
-  # - axom +cuda cuda_arch=75         # axom: https://github.com/spack/spack/issues/29520
   # - cp2k +mpi +cuda cuda_arch=75    # cp2k: cp2k only supports cuda_arch ('35', '37', '60', '70', '80')
-  # - cusz +cuda cuda_arch=75         # cusz: https://github.com/spack/spack/issues/38787
-  # - dealii +cuda cuda_arch=75       # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
-  # - ecp-data-vis-sdk +adios2 +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=75  # embree: https://github.com/spack/spack/issues/39534
   # - lammps +cuda cuda_arch=75       # lammps: needs NVIDIA driver
-  # - lbann +cuda cuda_arch=75        # lbann: https://github.com/spack/spack/issues/38788
+  # - lbann +cuda cuda_arch=75        # aluminum: include/aluminum/base.hpp:53: multiple definition of `cub::CUB_200400___CUDA_ARCH_LIST___NS::Debug(cudaError,char const*, int)'; lbann: https://github.com/spack/spack/issues/38788
   # - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf ~cusz +mgard +cuda cuda_arch=75 # libpressio: CMake Error at CMakeLists.txt:498 (find_library): Could not find CUFile_LIBRARY using the following names: cufile ; +cusz: https://github.com/spack/spack/issues/38787
-  # - paraview +cuda cuda_arch=75     # Error building some cuda componets in paraview
-  # - py-torch +cuda cuda_arch=75     # skipped, installed by other means
-  # - slepc +cuda cuda_arch=75        # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
+  # - paraview +cuda cuda_arch=75     # paraview: killed oom?
   # - upcxx +cuda cuda_arch=75        # upcxx: needs NVIDIA driver
 
   # CUDA 80
   - amrex +cuda cuda_arch=80
   - arborx +cuda cuda_arch=80 ^kokkos +wrapper
+  - axom +cuda cuda_arch=80
   - cabana +cuda cuda_arch=80 ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=80
   - caliper +cuda cuda_arch=80
   - chai +cuda cuda_arch=80 ^umpire ~shared
+  - cusz +cuda cuda_arch=80
+  - dealii +cuda cuda_arch=80
+  - ecp-data-vis-sdk +adios2 +hdf5 +vtkm +zfp ~paraview +cuda cuda_arch=80 # +paraview: job killed oom?
   - flecsi +cuda cuda_arch=80
   - ginkgo +cuda cuda_arch=80
   - gromacs +cuda cuda_arch=80
@@ -279,8 +283,10 @@ spack:
   - omega-h +cuda cuda_arch=80
   - parsec +cuda cuda_arch=80
   - petsc +cuda cuda_arch=80
+  - py-torch +cuda cuda_arch=80
   - raja +cuda cuda_arch=80
   - slate +cuda cuda_arch=80
+  - slepc +cuda cuda_arch=80
   - strumpack ~slate +cuda cuda_arch=80
   - sundials +cuda cuda_arch=80
   - superlu-dist +cuda cuda_arch=80
@@ -293,30 +299,27 @@ spack:
   - vtk-m +cuda cuda_arch=80
   - zfp +cuda cuda_arch=80
   # --
-  # - axom +cuda cuda_arch=80         # axom: https://github.com/spack/spack/issues/29520
   # - cp2k +mpi +cuda cuda_arch=80    # cp2k: Error: KeyError: 'Point environment variable LIBSMM_PATH to the absolute path of the libsmm.a file'
-  # - cusz +cuda cuda_arch=80         # cusz: https://github.com/spack/spack/issues/38787
-  # - dealii +cuda cuda_arch=80       # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
-  # - ecp-data-vis-sdk +adios2 +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=80 # embree: https://github.com/spack/spack/issues/39534
   # - lammps +cuda cuda_arch=80       # lammps: needs NVIDIA driver
-  # - lbann +cuda cuda_arch=80        # lbann: https://github.com/spack/spack/issues/38788
+  # - lbann +cuda cuda_arch=80        # aluminum: include/aluminum/base.hpp:53: multiple definition of `cub::CUB_200400___CUDA_ARCH_LIST___NS::Debug(cudaError,char const*, int)'; lbann: https://github.com/spack/spack/issues/38788
   # - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf ~cusz +mgard +cuda cuda_arch=80 # libpressio: CMake Error at CMakeLists.txt:498 (find_library): Could not find CUFile_LIBRARY using the following names: cufile ; +cusz: https://github.com/spack/spack/issues/38787
-  # - paraview +cuda cuda_arch=80     # Error building some cuda componets in paraview
-  # - py-torch +cuda cuda_arch=80     # skipped, installed by other means
-  # - slepc +cuda cuda_arch=80        # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
+  # - paraview +cuda cuda_arch=80     # paraview: killed oom?
   # - upcxx +cuda cuda_arch=80        # upcxx: needs NVIDIA driver
 
   # CUDA 90
   - amrex +cuda cuda_arch=90
   - arborx +cuda cuda_arch=90 ^kokkos +wrapper
+  - axom +cuda cuda_arch=90
   - cabana +cuda cuda_arch=90  ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=90
   - caliper +cuda cuda_arch=90
   - chai +cuda cuda_arch=90 ^umpire ~shared
+  - ecp-data-vis-sdk +adios2 +hdf5 +vtkm +zfp ~paraview +cuda cuda_arch=90 # +paraview: vtkm/exec/cuda/internal/ThrustPatches.h(213): error: this declaration has no storage class or type specifier
   - flecsi +cuda cuda_arch=90
   - ginkgo +cuda cuda_arch=90
   - gromacs +cuda cuda_arch=90
   - heffte +cuda cuda_arch=90
   - hpx +cuda cuda_arch=90
+  - hypre +cuda cuda_arch=90
   - kokkos +wrapper +cuda cuda_arch=90
   - kokkos-kernels +cuda cuda_arch=90 ^kokkos +wrapper +cuda cuda_arch=90
   - magma +cuda cuda_arch=90
@@ -324,11 +327,14 @@ spack:
   - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=90
   - parsec +cuda cuda_arch=90
   - petsc +cuda cuda_arch=90
+  - py-torch +cuda cuda_arch=90
   - raja +cuda cuda_arch=90
   - slate +cuda cuda_arch=90
+  - slepc +cuda cuda_arch=90
   - strumpack ~slate +cuda cuda_arch=90
   - sundials +cuda cuda_arch=90
   - superlu-dist +cuda cuda_arch=90
+  - tasmanian +cuda cuda_arch=90
   - trilinos +cuda cuda_arch=90
   - umpire ~shared +cuda cuda_arch=90
   # INCLUDED IN ECP DAV CUDA
@@ -338,19 +344,13 @@ spack:
   - zfp +cuda cuda_arch=90
   # --
   # - ascent +cuda cuda_arch=90       # ascent: https://github.com/spack/spack/issues/38045
-  # - axom +cuda cuda_arch=90         # axom: https://github.com/spack/spack/issues/29520
   # - cp2k +mpi +cuda cuda_arch=90    # cp2k: cp2k only supports cuda_arch ('35', '37', '60', '70', '80')
-  # - cusz +cuda cuda_arch=90         # cusz: https://github.com/spack/spack/issues/38787
-  # - dealii +cuda cuda_arch=90       # dealii: https://github.com/spack/spack/issues/39532
-  # - ecp-data-vis-sdk +adios2 +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=90 # embree: https://github.com/spack/spack/issues/39534
-  # - hypre +cuda cuda_arch=90        # concretizer: hypre +cuda requires cuda@:11, but cuda_arch=90 requires cuda@12:
+  # - cusz +cuda cuda_arch=90         # cusz: cuda-12.5.1-pil77yk7gsseyqitybr47qmhdtszbcwa/targets/sbsa-linux/include/cub/util_device.cuh:160:63: error: 'blockIdx' was not declared in this scope
+  # - dealii +cuda cuda_arch=90       # dealii: concretize conflict + https://github.com/spack/spack/issues/39532
   # - lammps +cuda cuda_arch=90       # lammps: needs NVIDIA driver
-  # - lbann +cuda cuda_arch=90        # concretizer: Cannot select a single "version" for package "lbann"
+  # - lbann +cuda cuda_arch=90        # aluminum: include/aluminum/base.hpp:53: multiple definition of `cub::CUB_200400___CUDA_ARCH_LIST___NS::Debug(cudaError,char const*, int)'; 
   # - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf ~cusz +mgard +cuda cuda_arch=90 # libpressio: CMake Error at CMakeLists.txt:498 (find_library): Could not find CUFile_LIBRARY using the following names: cufile ; +cusz: https://github.com/spack/spack/issues/38787
   # - omega-h +cuda cuda_arch=90      # omega-h: https://github.com/spack/spack/issues/39535
-  # - py-torch +cuda cuda_arch=90     # skipped, installed by other means
-  # - slepc +cuda cuda_arch=90        # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
-  # - tasmanian +cuda cuda_arch=90    # tasmanian: conflicts with cuda@12
   # - upcxx +cuda cuda_arch=90        # upcxx: needs NVIDIA driver
 
   ci:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
index 84666f5beb..5d25864371 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
@@ -185,7 +185,7 @@ spack:
   - sz
   - unifyfs
   - veloc
-  # - visit                           # silo: https://github.com/spack/spack/issues/39538
+  - visit                           # silo: https://github.com/spack/spack/issues/39538
   - vtk-m
   - zfp
   # --
@@ -227,11 +227,11 @@ spack:
   # CUDA 80
   - amrex +cuda cuda_arch=80
   - arborx +cuda cuda_arch=80 ^kokkos +wrapper
+  - axom +cuda cuda_arch=80
   - cabana +cuda cuda_arch=80 ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=80
   - caliper +cuda cuda_arch=80
   - chai +cuda cuda_arch=80 ^umpire ~shared
   - cusz +cuda cuda_arch=80
-  - dealii +cuda cuda_arch=80
   - ecp-data-vis-sdk ~rocm +adios2 ~ascent +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=80 # +ascent fails because fides fetch error
   - exago +mpi +python +raja +hiop ~rocm +cuda cuda_arch=80 ~ipopt ^hiop@1.0.0 ~sparse +mpi +raja ~rocm +cuda cuda_arch=80 #^raja@0.14.0
   - flecsi +cuda cuda_arch=80
@@ -259,109 +259,68 @@ spack:
   - trilinos +cuda cuda_arch=80
   - umpire ~shared +cuda cuda_arch=80
   # INCLUDED IN ECP DAV CUDA
-  # - adios2 +cuda cuda_arch=80
-  # - ascent +cuda cuda_arch=80       # ascent: https://github.com/spack/spack/issues/38045
-  # - paraview +cuda cuda_arch=80
-  # - vtk-m +cuda cuda_arch=80
-  # - zfp +cuda cuda_arch=80
+  - adios2 +cuda cuda_arch=80
+  # - ascent +cuda cuda_arch=80     # ascent: https://github.com/spack/spack/issues/38045
+  - paraview +cuda cuda_arch=80
+  - vtk-m +cuda cuda_arch=80
+  - zfp +cuda cuda_arch=80
   # --
-  # - axom +cuda cuda_arch=80         # axom: https://github.com/spack/spack/issues/29520
-  # - lammps +cuda cuda_arch=80       # lammps: needs NVIDIA driver
-  # - lbann +cuda cuda_arch=80        # lbann: https://github.com/spack/spack/issues/38788
+  # - dealii +cuda cuda_arch=80     # dealii: conflicts with '+cuda ^cuda@12:'
+  # - lammps +cuda cuda_arch=80     # lammps: needs NVIDIA driver
+  # - lbann +cuda cuda_arch=80      # lbann: layers/transform/cereal_registration/../permute/cutensor_support.hpp:95:18: error: 'cutensorInit' was not declared in this scope; did you mean 'cutensorPlan_t'?
   # - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=80 # mgard: https://github.com/spack/spack/issues/44833
-  # - upcxx +cuda cuda_arch=80        # upcxx: needs NVIDIA driver
+  # - upcxx +cuda cuda_arch=80      # upcxx: needs NVIDIA driver
 
-  # # CUDA 90
-  # - amrex +cuda cuda_arch=90
-  # - arborx +cuda cuda_arch=90 ^kokkos +wrapper
-  # - cabana +cuda cuda_arch=90 ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=90
-  # - caliper +cuda cuda_arch=90
-  # - chai +cuda cuda_arch=90 ^umpire ~shared
-  # - cusz +cuda cuda_arch=90
-  # - flecsi +cuda cuda_arch=90
-  # - ginkgo +cuda cuda_arch=90
-  # - gromacs +cuda cuda_arch=90
-  # - heffte +cuda cuda_arch=90
-  # - hpx +cuda cuda_arch=90
-  # - kokkos +wrapper +cuda cuda_arch=90
-  # - kokkos-kernels +cuda cuda_arch=90 ^kokkos +wrapper +cuda cuda_arch=90
-  # - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +cusz +mgard +cuda cuda_arch=90 ^cusz +cuda cuda_arch=90
-  # - magma +cuda cuda_arch=90
-  # - mfem +cuda cuda_arch=90
-  # - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=90
-  # - parsec +cuda cuda_arch=90
-  # - petsc +cuda cuda_arch=90
-  # - py-torch +cuda cuda_arch=90
-  # - raja +cuda cuda_arch=90
-  # - slate +cuda cuda_arch=90
-  # - slepc +cuda cuda_arch=90
-  # - strumpack ~slate +cuda cuda_arch=90
-  # - sundials +cuda cuda_arch=90
-  # - superlu-dist +cuda cuda_arch=90
-  # - trilinos +cuda cuda_arch=90
-  # - umpire ~shared +cuda cuda_arch=90
-  # # INCLUDED IN ECP DAV CUDA
-  # - adios2 +cuda cuda_arch=90
-  # # - ascent +cuda cuda_arch=90       # ascent: https://github.com/spack/spack/issues/38045
-  # # - paraview +cuda cuda_arch=90     # paraview: InstallError: Incompatible cuda_arch=90
-  # - vtk-m +cuda cuda_arch=90
-  # - zfp +cuda cuda_arch=90
-  # # --
-  # # - axom +cuda cuda_arch=90         # axom: https://github.com/spack/spack/issues/29520
-  # # - dealii +cuda cuda_arch=90       # dealii: https://github.com/spack/spack/issues/39532
-  # # - ecp-data-vis-sdk ~rocm +adios2 +ascent +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=90 # paraview: incompatible cuda_arch; vtk-m: CMake Error at CMake/VTKmWrappers.cmake:413 (message): vtkm_cont needs to be built STATIC as CUDA doesn't support virtual methods across dynamic library boundaries.  You need to set the CMake opt ion BUILD_SHARED_LIBS to `OFF` or (better) turn VTKm_NO_DEPRECATED_VIRTUAL to `ON`.
-  # # - hypre +cuda cuda_arch=90        # concretizer: hypre +cuda requires cuda@:11, but cuda_arch=90 requires cuda@12:
-  # # - lammps +cuda cuda_arch=90       # lammps: needs NVIDIA driver
-  # # - lbann +cuda cuda_arch=90        # concretizer: Cannot select a single "version" for package "lbann"
-  # # - omega-h +cuda cuda_arch=90      # omega-h: https://github.com/spack/spack/issues/39535
-  # # - tasmanian +cuda cuda_arch=90    # tasmanian: conflicts with cuda@12
-  # # - upcxx +cuda cuda_arch=90        # upcxx: needs NVIDIA driver
+  # CUDA 90
+  - amrex +cuda cuda_arch=90
+  - arborx +cuda cuda_arch=90 ^kokkos +wrapper
+  - axom +cuda cuda_arch=90
+  - cabana +cuda cuda_arch=90 ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=90
+  - caliper +cuda cuda_arch=90
+  - chai +cuda cuda_arch=90 ^umpire ~shared
+  - flecsi +cuda cuda_arch=90
+  - ginkgo +cuda cuda_arch=90
+  - gromacs +cuda cuda_arch=90
+  - heffte +cuda cuda_arch=90
+  - hpx +cuda cuda_arch=90
+  - hypre +cuda cuda_arch=90
+  - kokkos +wrapper +cuda cuda_arch=90
+  - kokkos-kernels +cuda cuda_arch=90 ^kokkos +wrapper +cuda cuda_arch=90
+  - magma +cuda cuda_arch=90
+  - mfem +cuda cuda_arch=90
+  - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=90
+  - parsec +cuda cuda_arch=90
+  - petsc +cuda cuda_arch=90
+  - raja +cuda cuda_arch=90
+  - slate +cuda cuda_arch=90
+  - slepc +cuda cuda_arch=90
+  - strumpack ~slate +cuda cuda_arch=90
+  - sundials +cuda cuda_arch=90
+  - superlu-dist +cuda cuda_arch=90
+  - tasmanian +cuda cuda_arch=90
+  - trilinos +cuda cuda_arch=90
+  - umpire ~shared +cuda cuda_arch=90
+  # INCLUDED IN ECP DAV CUDA
+  - adios2 +cuda cuda_arch=90
+  # - ascent +cuda cuda_arch=90       # ascent: https://github.com/spack/spack/issues/38045
+  # - paraview +cuda cuda_arch=90     # paraview: InstallError: Incompatible cuda_arch=90
+  - vtk-m +cuda cuda_arch=90
+  - zfp +cuda cuda_arch=90
+  # --
+  # - cusz +cuda cuda_arch=90         # cusz: cuda-12.5.0-ndrzb7undvancjdj3fi6bhthdxdo7gr5/targets/x86_64-linux/include/cub/util_device.cuh:202:50: error: 'blockDim' was not declared in this scope
+  # - dealii +cuda cuda_arch=90       # dealii: conflicts with '+cuda ^cuda@12:'; dealii: https://github.com/spack/spack/issues/39532
+  # - ecp-data-vis-sdk ~rocm +adios2 +ascent +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=90 # +ascent: # ascent: https://github.com/spack/spack/issues/38045; +paraview: VTK/ThirdParty/vtkm/vtkvtkm/vtk-m/vtkm/exec/cuda/internal/ExecutionPolicy.h(121): error: namespace "thrust" has no member "sort"
+  # - lammps +cuda cuda_arch=90       # lammps: needs NVIDIA driver
+  # - lbann +cuda cuda_arch=90        # aluminum: /usr/include/c++/11/bits/basic_string.h:1260: multiple definition of `cub::CUB_200400___CUDA_ARCH_LIST___NS::Debug(cudaError, char const*, int)'; 
+  # - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +cusz +mgard +cuda cuda_arch=90 ^cusz +cuda cuda_arch=90 # cusz: cuda-12.5.0-e3rny44pq5z5x3nnoljynbsq5on5fnl3/targets/x86_64-linux/include/cub/util_device.cuh:202:50: error: 'blockDim' was not declared in this scope
+  # - omega-h +cuda cuda_arch=90      # omega-h: https://github.com/spack/spack/issues/39535
+  # - py-torch +cuda cuda_arch=90     # py-torch: FAILED: caffe2/CMakeFiles/torch_cuda.dir/__/aten/src/ATen/native/transformers/cuda/flash_attn/kernels/flash_bwd_hdim256_bf16_sm80.cu.o
+  # - upcxx +cuda cuda_arch=90        # upcxx: needs NVIDIA driver
 
   # ROCM NOARCH
   - hpctoolkit +rocm
   - tau +mpi +rocm +syscall           # tau: has issue with `spack env depfile` build
 
-  # # ROCM 908
-  # - adios2 +kokkos +rocm amdgpu_target=gfx908
-  # - amrex +rocm amdgpu_target=gfx908
-  # - arborx +rocm amdgpu_target=gfx908
-  # - cabana +rocm amdgpu_target=gfx908
-  # - caliper +rocm amdgpu_target=gfx908
-  # - chai +rocm amdgpu_target=gfx908
-  # - ecp-data-vis-sdk +paraview +vtkm +rocm amdgpu_target=gfx908
-  # - gasnet +rocm amdgpu_target=gfx908
-  # - ginkgo +rocm amdgpu_target=gfx908
-  # - heffte +rocm amdgpu_target=gfx908
-  # - hpx +rocm amdgpu_target=gfx908
-  # - hypre +rocm amdgpu_target=gfx908
-  # - kokkos +rocm amdgpu_target=gfx908
-  # - legion +rocm amdgpu_target=gfx908
-  # - magma ~cuda +rocm amdgpu_target=gfx908
-  # - mfem +rocm amdgpu_target=gfx908
-  # - petsc +rocm amdgpu_target=gfx908
-  # - raja ~openmp +rocm amdgpu_target=gfx908
-  # - slate +rocm amdgpu_target=gfx908
-  # - slepc +rocm amdgpu_target=gfx908 ^petsc +rocm amdgpu_target=gfx908
-  # - strumpack ~slate +rocm amdgpu_target=gfx908
-  # - sundials +rocm amdgpu_target=gfx908
-  # - superlu-dist +rocm amdgpu_target=gfx908
-  # - tasmanian ~openmp +rocm amdgpu_target=gfx908
-  # - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack ~ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu ~stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long +rocm amdgpu_target=gfx908
-  # - umpire +rocm amdgpu_target=gfx908
-  # - upcxx +rocm amdgpu_target=gfx908
-  # # INCLUDED IN ECP DAV ROCM
-  # # - hdf5
-  # # - hdf5-vol-async
-  # # - hdf5-vol-cache
-  # # - hdf5-vol-log
-  # # - libcatalyst
-  # - paraview +rocm amdgpu_target=gfx908
-  # # - vtk-m ~openmp +rocm amdgpu_target=gfx908  # vtk-m: https://github.com/spack/spack/issues/40268
-  # # --
-  # # - exago +mpi +python +raja +hiop +rocm amdgpu_target=gfx908 ~ipopt cxxflags="-Wno-error=non-pod-varargs" ^hiop@1.0.0 ~sparse +mpi +raja +rocm amdgpu_target=gfx908 # hiop: CMake Error at cmake/FindHiopHipLibraries.cmake:23 (find_package)
-  # # - lbann ~cuda +rocm amdgpu_target=gfx908    # aluminum: https://github.com/spack/spack/issues/38807
-  # # - papi +rocm amdgpu_target=gfx908           # papi: https://github.com/spack/spack/issues/27898
-
   # ROCM 90a
   - amrex +rocm amdgpu_target=gfx90a
   - caliper +rocm amdgpu_target=gfx90a
@@ -396,17 +355,17 @@ spack:
   # - arborx +rocm amdgpu_target=gfx90a         # kokkos: https://github.com/spack/spack/issues/44832
   # - cabana +rocm amdgpu_target=gfx90a         # kokkos: https://github.com/spack/spack/issues/44832
   # - exago +mpi +python +raja +hiop +rocm amdgpu_target=gfx90a ~ipopt cxxflags="-Wno-error=non-pod-varargs" ^hiop@1.0.0 ~sparse +mpi +raja +rocm amdgpu_target=gfx90a # hiop: CMake Error at cmake/FindHiopHipLibraries.cmake:23 (find_package)
-  # - kokkos +rocm amdgpu_target=gfx90a # kokkos: https://github.com/spack/spack/issues/44832
+  # - kokkos +rocm amdgpu_target=gfx90a         # kokkos: https://github.com/spack/spack/issues/44832
   # - lbann ~cuda +rocm amdgpu_target=gfx90a    # aluminum: https://github.com/spack/spack/issues/38807
   # - legion +rocm amdgpu_target=gfx90a         # kokkos: https://github.com/spack/spack/issues/44832
   # - papi +rocm amdgpu_target=gfx90a           # papi: https://github.com/spack/spack/issues/27898
   # - petsc +rocm amdgpu_target=gfx90a          # petsc: https://github.com/spack/spack/issues/44600
   # - slepc +rocm amdgpu_target=gfx90a ^petsc +rocm amdgpu_target=gfx90a # petsc: https://github.com/spack/spack/issues/44600
 
-  # - cp2k +mpi +cuda cuda_arch=80    # cp2k: spack-stage-libxsmm-1.17-ml4asr3rsb5biw34x6m764cbrp3zrwyk/spack-src/./src/libxsmm_gemm.c:223: undefined reference to `dgemv_'
-  # - cp2k +mpi +cuda cuda_arch=90 # cp2k: cp2k only supports cuda_arch ('35', '37', '60', '70', '80')
-  # - cp2k +mpi +rocm amdgpu_target=gfx908 # cp2k: Error: KeyError: 'No spec with name rocm in... "-L{}".format(spec["rocm"].libs.directories[0]),
-  # - cp2k +mpi +rocm amdgpu_target=gfx90a # cp2k: Error: KeyError: 'No spec with name rocm in... "-L{}".format(spec["rocm"].libs.directories[0]),
+  # - cp2k +mpi +cuda cuda_arch=80              # cp2k: spack-stage-libxsmm-1.17-r2zqxa24bhufaj5i3ili5se25cw7tioo/spack-src/./src/libxsmm_gemm.c:238: undefined reference to `sgemv_'
+  # - cp2k +mpi +cuda cuda_arch=90              # cp2k: cp2k only supports cuda_arch ('35', '37', '60', '70', '80')
+  # - cp2k +mpi +rocm amdgpu_target=gfx908      # cp2k: "-L{}".format(spec["rocm"].libs.directories[0]),
+  # - cp2k +mpi +rocm amdgpu_target=gfx90a      # cp2k: "-L{}".format(spec["rocm"].libs.directories[0]),
 
   ci:
     pipeline-gen: