CI: E4S: enable full E4S (#24011)

* e4s ci: enable full e4s
* add llvm-amdgpu to list of specs needing an xlarge tagged runner
* comment out qt and qwt because of intermittent build failures
* remove +rocm specs because rocblas job consistently fails due to infrastructure

1 files changed, 295 insertions, 78 deletions

diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
index 96041479ec..5d888bddf4 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
@@ -11,113 +11,316 @@ spack:
 
   packages:
     all:
-      target: [x86_64]
+      compiler:
+        - gcc@7.5.0
       providers:
         blas:
-        - openblas
+          - openblas
         mpi:
-        - mpich
+          - mpich
+      target:
+        - x86_64
       variants: +mpi
+    autoconf:
+      version:
+        - '2.69'
+    automake:
+      version:
+        - 1.16.3
+    berkeley-db:
+      version:
+        - 18.1.40
     binutils:
-      variants: +gold+headers+libiberty~nls
+      variants: +ld +gold +headers +libiberty ~nls
+      version:
+        - 2.33.1
+    boost:
       version:
-      - 2.33.1
+        - 1.75.0
+    bzip2:
+      version:
+        - 1.0.8
+    c-blosc:
+      version:
+        - 1.21.0
     cmake:
-      version: [3.18.4]
+      version:
+        - 3.20.2
+    curl:
+      version:
+        - 7.76.0
+    diffutils:
+      version:
+        - 3.7
+    elfutils:
+      version:
+        - 0.182
+      variants: +bzip2 ~nls +xz
+    expat:
+      version:
+        - 2.2.10
+    findutils:
+      version:
+        - 4.8.0
+    gdbm:
+      version:
+        - 1.18.1
+    gettext:
+      version:
+        - 0.21
+    git:
+      version:
+        - 2.31.0
     hdf5:
-      variants: +fortran
+      variants: +fortran +hl +shared
+      version:
+        - 1.10.7
+    help2man:
+      version:
+        - 1.47.16
+    hwloc:
+      version:
+        - 2.4.1
+    json-c:
+      version:
+        - 0.13.1
+    libbsd:
+      version:
+        - 0.10.0
+    libfabric:
+      version:
+        - 1.12.1
+      variants: fabrics=sockets,tcp,udp,rxm
+    libiconv:
+      version:
+        - 1.16
+    libsigsegv:
+      version:
+        - 2.12
+    libpciaccess:
+      version:
+        - 0.16
+    libtool:
+      version:
+        - 2.4.6
+    libunwind:
+      version:
+        - 1.5.0
+      variants: +pic +xz
+    libxml2:
+      version:
+        - 2.9.10
+    lz4:
+      version:
+        - 1.9.3
+    m4:
+      version:
+        - 1.4.18
+    mesa:
+      variants: ~llvm
+    mesa18:
+      variants: ~llvm
     mpich:
       variants: ~wrapperrpath
+      version:
+        - 3.4.1
+    ncurses:
+      version:
+        - 6.2
+      variants: +termlib
+    numactl:
+      version:
+        - 2.0.14
     openblas:
-      version: [0.3.10]
-    slepc:
-      version: [3.14.0]
+      version:
+        - 0.3.10
+      variants: threads=openmp
+    perl:
+      version:
+        - 5.32.1
+    pkgconf:
+      version:
+        - 1.7.3
+    python:
+      version:
+        - 3.8.10
+    readline:
+      version:
+        - 8
+    sqlite:
+      version:
+        - 3.34.0
+    tar:
+      version:
+        - 1.32
+    texinfo:
+      version:
+        - 6.5
+    xz:
+      version:
+        - 5.2.5
+      variants: +pic
+    zlib:
+      version:
+        - 1.2.11
+    zstd:
+      version:
+        - 1.4.9
 
   definitions:
-  - e4s:
-    # - adios
-    # - adios2
-    # - aml
-    # - amrex
-    # - arborx
+
+  - cuda_specs:
+    - amrex +cuda cuda_arch=70
+    - axom +cuda cuda_arch=70 ^umpire@4.1.2 ~shared
+    - caliper +cuda cuda_arch=70
+    - chai +cuda ~benchmarks ~tests cuda_arch=70 ^umpire@4.1.2 ~shared
+    - ginkgo +cuda cuda_arch=70
+    - hpx +cuda cuda_arch=70
+    - kokkos +cuda +wrapper cuda_arch=70
+    - kokkos-kernels +cuda cuda_arch=70 ^kokkos +cuda +wrapper cuda_arch=70
+    - magma cuda_arch=70
+    - raja +cuda cuda_arch=70
+    - slate +cuda cuda_arch=70
+    - strumpack +cuda ~slate cuda_arch=70
+    - sundials +cuda cuda_arch=70
+    - superlu-dist +cuda cuda_arch=70
+    - tasmanian +cuda cuda_arch=70
+    - zfp +cuda cuda_arch=70
+    #- ascent +cuda ~shared cuda_arch=70
+    #- hypre +cuda cuda_arch=70
+    #- mfem +cuda cuda_arch=70
+    #- umpire +cuda ~shared cuda_arch=70 # unsatisfiable concretization conflict w/ blt
+
+  - rocm_specs:
+    - kokkos +rocm amdgpu_target=gfx906
+    - strumpack +rocm ~slate amdgpu_target=gfx906
+    #- amrex +rocm amdgpu_target=gfx906
+    #- chai +rocm ~benchmarks amdgpu_target=gfx906
+    #- ginkgo +rocm amdgpu_target=gfx906 # needs hip<4.1
+    #- raja +rocm ~openmp amdgpu_target=gfx906 # blt 0.3.6 issue with rocm
+    #- slate +rocm amdgpu_target=gfx906
+    #- sundials +rocm amdgpu_target=gfx906
+    #- tasmanian +rocm amdgpu_target=gfx906
+    #- umpire+rocm amdgpu_target=gfx906 # blt 0.3.6 issue with rocm
+
+  - default_specs:
+    - adios
+    - adios2
+    - aml
+    - amrex
+    - arborx
+    - archer
     - argobots
-    # - ascent
-    # - axom
+    - ascent
+    - axom ^umpire@4.1.2
     - bolt
-    # - caliper
-    # - darshan-runtime
+    - cabana
+    - caliper
+    - chai ~benchmarks ~tests ^umpire@4.1.2
+    - conduit
+    - darshan-runtime
     - darshan-util
-    # - dyninst
+    - dyninst
     - faodel
-    # - flecsi+cinch
-    # - flit
-    # - gasnet
+    - flecsi +cinch
+    - flit
+    - fortrilinos ^trilinos +nox +superlu-dist +stratimikos
+    - gasnet
     - ginkgo
-    # - globalarrays
-    # - gotcha
-    # - hdf5
-    # - hpctoolkit
-    # - hpx
-    # - hypre
-    # - kokkos-kernels+openmp
-    # - kokkos+openmp
-    # - legion
-    # - libnrm
-    # - libquo
-    # - magma cuda_arch=70 ^cuda@10.2.89
-    # - mercury
-    # - mfem
-    # - mpifileutils@develop~xattr
+    - globalarrays
+    - gmp
+    - gotcha
+    - hdf5
+    - heffte +fftw
+    - hpctoolkit
+    - hpx
+    - hypre
+    - kokkos +openmp
+    - kokkos-kernels +openmp
+    - legion
+    - libnrm
+    - libquo
+    - libunwind
+    - loki
+    - mercury
+    - metall
+    - mfem
+    - mpark-variant
+    - mpifileutils ~xattr
     - ninja
-    # - omega-h
-    # - openmpi
-    # - openpmd-api
-    # - papi
-    # - papyrus@1.0.1
-    # - parallel-netcdf
-    # - pdt
-    # - petsc
-    # - phist
-    # - plasma
-    # - precice
-    # - pumi
-    # - py-jupyterhub
-    # - py-libensemble
-    # - py-petsc4py
-    # - qthreads scheduler=distrib
-    # - raja
-    # - rempi
-    # - scr
-    # - slate ^openblas@0.3.6 threads=openmp ^cuda@10.2.89
-    # - slepc
-    # - stc
-    # - strumpack ~slate ^openblas@0.3.6 threads=openmp
-    # - sundials
-    # - superlu
-    # - superlu-dist
+    - nrm
+    - omega-h
+    - openmpi
+    - openpmd-api
+    - papi
+    - papyrus@1.0.1
+    - parallel-netcdf
+    - pdt
+    - petsc
+    - phist
+    - plasma
+    - precice
+    - pumi
+    - py-jupyterhub
+    - py-libensemble
+    - py-petsc4py
+    - py-warpx ^warpx dims=2
+    - py-warpx ^warpx dims=3
+    - py-warpx ^warpx dims=rz
+    - qthreads scheduler=distrib
+    - raja
+    - rempi
+    - scr
+    - slate ~cuda
+    - slepc
+    - stc
+    - strumpack ~slate
+    - sundials
+    - superlu
+    - superlu-dist
     - swig
-    # - sz
-    # - tasmanian
-    # - tau
-    # - trilinos
-    # - turbine
-    # - umap
-    # - umpire
-    # - unifyfs
-    # - upcxx
-    # - veloc
-    # - zfp
+    - swig@4.0.2-fortran
+    - sz
+    - tasmanian
+    - tau
+    - trilinos
+    - trilinos +nox +superlu-dist
+    - turbine
+    - umap
+    - unifyfs@0.9.1
+    - upcxx
+    - zfp
+    #- dealii
+    #- geopm
+    #- llvm-doe@doe +clang +compiler-rt +libcxx +lld +lldb +llvm_dylib +flang
+    #- qt
+    #- qwt
+    #- umpire # unsatisfiable concretization conflict w/ blt
+    #- variorum # root fails
+    #- veloc # issue filed
+
   - arch:
     - '%gcc@7.5.0 arch=linux-ubuntu18.04-x86_64'
 
+
   specs:
+
   - matrix:
-    - - $e4s
+    - - $default_specs
     - - $arch
 
+  - matrix:
+    - - $cuda_specs
+    - - $arch
+
+  # - matrix:
+  #   - - $rocm_specs
+  #   - - $arch
+
   mirrors: { "mirror": "s3://spack-binaries-develop/e4s" }
 
   gitlab-ci:
+
     script:
       - . "./share/spack/setup-env.sh"
       - spack --version
@@ -125,8 +328,22 @@ spack:
       - spack env activate --without-view .
       - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
       - spack -d ci rebuild
+
     mappings:
-      - match: [cuda, dyninst, hpx, precice, strumpack, sundials, trilinos, vtk-h, vtk-m]
+      - match:
+        - cuda
+        - dyninst
+        - hpx
+        - llvm
+        - llvm-amdgpu
+        - precice
+        - rocblas
+        - rocsolver
+        - strumpack
+        - sundials
+        - trilinos
+        - vtk-h
+        - vtk-m
         runner-attributes:
           image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] }
           tags: ["spack", "public", "xlarge", "x86_64"]