e4s: add power stack (#32710)

* e4s: restore builds builds

* gitlab ci: allow UO to build protected binaries for signing

* use newer image; comment out failing builds

* gitlab-ci: Some tweaks for e4s power builds

- fix tags (no longer require generate jobs to run on aws)
- fix resource requests for generation jobs resource requests
- remove SPACK_SIGNING_KEY from protected power build jobs
- update UO signing key path
- change the CDash build group to reflect stack name
- retry pipeline generation jobs *always*

* correct double packages: section

* gitlab-ci:script: modernize

* remove new gnu make, not for ppc64le

---------

Co-authored-by: Scott Wittenburg <scott.wittenburg@kitware.com>

3 files changed, 477 insertions, 317 deletions

diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
index a85393f893..e6bbedb5a1 100644
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -7,10 +7,22 @@ default:
 # Job templates
 ########################################
 
-.pr:
+.aws-pr-creds:
+  variables:
+    AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
+    AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
+
+.aws-protected-creds:
+  variables:
+    AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
+    AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
+
+.pr-refs:
   only:
   - /^pr[\d]+_.*$/
-  - /^github\/pr[\d]+_.*$/
+
+.pr:
+  extends: [ ".pr-refs" ]
   variables:
     SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
     SPACK_PIPELINE_TYPE: "spack_pull_request"
@@ -21,7 +33,6 @@ default:
   - /^develop$/
   - /^releases\/v.*/
   - /^v.*/
-  - /^github\/develop$/
 
 .protected:
   extends: [ ".protected-refs" ]
@@ -30,7 +41,7 @@ default:
     SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
     SPACK_PIPELINE_TYPE: "spack_protected_branch"
 
-.generate:
+.generate-base:
   stage: generate
   script:
     - uname -a || true
@@ -49,7 +60,6 @@ default:
   artifacts:
     paths:
       - "${CI_PROJECT_DIR}/jobs_scratch_dir"
-  tags: ["spack", "aws", "public", "medium", "x86_64"]
   variables:
     KUBERNETES_CPU_REQUEST: 4000m
     KUBERNETES_MEMORY_REQUEST: 16G
@@ -58,12 +68,15 @@ default:
   retry:
     max: 2
     when:
-      - runner_system_failure
-      - stuck_or_timeout_failure
+      - always
+
+.generate:
+  extends: [ ".generate-base" ]
+  tags: ["spack", "public", "medium", "x86_64"]
 
 .generate-aarch64:
   extends: [ ".generate" ]
-  tags: ["spack", "aws", "public", "medium", "aarch64"]
+  tags: ["spack", "public", "medium", "aarch64"]
 
 .pr-generate:
   extends: [ ".pr", ".generate" ]
@@ -81,20 +94,14 @@ default:
   stage: build
 
 .pr-build:
-  extends: [ ".pr", ".build" ]
-  variables:
-    AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
-    AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
+  extends: [ ".pr", ".build", ".aws-pr-creds" ]
 
 .protected-build:
-  extends: [ ".protected", ".build" ]
-  variables:
-    AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
-    AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
+  extends: [ ".protected", ".build", ".aws-protected-creds" ]
 
 protected-publish:
   stage: publish
-  extends: [ ".protected" ]
+  extends: [ ".protected", ".aws-protected-creds" ]
   image: "ghcr.io/spack/python-aws-bash:0.0.1"
   tags: ["spack", "public", "medium", "aws", "x86_64"]
   retry:
@@ -199,7 +206,6 @@ protected-publish:
 #   variables:
 #     AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
 #     AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
-
 # .mac-protected-build:
 #   extends: [ ".mac-protected", ".build" ]
 #   variables:
@@ -357,41 +363,41 @@ e4s-oneapi-protected-build:
 ########################################
 # E4S on Power
 ########################################
-# .power-e4s-generate-tags-and-image:
-#   image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu20.04-runner-ppc64le:2021-07-01", "entrypoint": [""] }
-#   tags: ["spack", "public", "medium", "ppc64le"]
+.e4s-power-generate-tags-and-image:
+  image: { "name": "ecpe4s/ubuntu20.04-runner-ppc64le:2023-01-01", "entrypoint": [""] }
+  tags: ["spack", "public", "large", "ppc64le"]
 
-# .e4s-on-power:
-#   variables:
-#     SPACK_CI_STACK_NAME: e4s-on-power
+.e4s-power:
+  variables:
+    SPACK_CI_STACK_NAME: e4s-power
 
-# e4s-on-power-pr-generate:
-#   extends: [ ".e4s-on-power", ".pr-generate", ".power-e4s-generate-tags-and-image"]
+e4s-power-pr-generate:
+  extends: [ ".e4s-power", ".pr-generate", ".e4s-power-generate-tags-and-image"]
 
-# e4s-on-power-protected-generate:
-#   extends: [ ".e4s-on-power", ".protected-generate", ".power-e4s-generate-tags-and-image"]
+e4s-power-protected-generate:
+  extends: [ ".e4s-power", ".protected-generate", ".e4s-power-generate-tags-and-image"]
 
-# e4s-on-power-pr-build:
-#   extends: [ ".e4s-on-power", ".pr-build" ]
-#   trigger:
-#     include:
-#       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-#         job: e4s-on-power-pr-generate
-#     strategy: depend
-#   needs:
-#     - artifacts: True
-#       job: e4s-on-power-pr-generate
+e4s-power-pr-build:
+  extends: [ ".e4s-power", ".pr-build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: e4s-power-pr-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: e4s-power-pr-generate
 
-# e4s-on-power-protected-build:
-#   extends: [ ".e4s-on-power", ".protected-build" ]
-#   trigger:
-#     include:
-#       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-#         job: e4s-on-power-protected-generate
-#     strategy: depend
-#   needs:
-#     - artifacts: True
-#       job: e4s-on-power-protected-generate
+e4s-power-protected-build:
+  extends: [ ".e4s-power", ".protected-build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: e4s-power-protected-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: e4s-power-protected-generate
 
 #########################################
 # Build tests for different build-systems
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-on-power/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-on-power/spack.yaml
deleted file mode 100644
index 4ae341d0bf..0000000000
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-on-power/spack.yaml
+++ /dev/null
@@ -1,269 +0,0 @@
-spack:
-  view: false
-
-  concretizer:
-    reuse: false
-    unify: false
-
-  config:
-    concretizer: clingo
-    install_tree:
-      root: /home/software/spack
-      padded_length: 512
-      projections:
-        all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
-
-  packages:
-    all:
-      compiler:
-        - gcc@9.3.0
-      providers:
-        blas:
-          - openblas
-        mpi:
-          - mpich
-      target:
-        - ppc64le
-      variants: +mpi
-    binutils:
-      variants: +ld +gold +headers +libiberty ~nls +plugins
-      version:
-      - 2.36.1
-    doxygen:
-      version:
-      - 1.8.20
-    elfutils:
-      variants: +bzip2 ~nls +xz
-    hdf5:
-      variants: +fortran +hl +shared api=v18
-      version:
-      - 1.12.0
-    libfabric:
-      variants: fabrics=sockets,tcp,udp,rxm
-    libunwind:
-      variants: +pic +xz
-    mesa:
-      variants: ~llvm
-    mesa18:
-      variants: ~llvm
-    mpich:
-      variants: ~wrapperrpath
-    ncurses:
-      variants: +termlib
-    openblas:
-      variants: threads=openmp
-    trilinos:
-      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
-    xz:
-      variants: +pic
-
-  definitions:
-
-  - cuda_specs:
-    - amrex +cuda cuda_arch=70
-    - caliper +cuda cuda_arch=70
-    - chai ~benchmarks ~tests +cuda cuda_arch=70 ^umpire ~shared
-    - ginkgo +cuda cuda_arch=70
-    - heffte +cuda cuda_arch=70
-    - hpx +cuda cuda_arch=70
-    - hypre +cuda cuda_arch=70
-    - kokkos +wrapper +cuda cuda_arch=70
-    - kokkos-kernels +cuda cuda_arch=70 ^kokkos +wrapper +cuda cuda_arch=70
-    - magma +cuda cuda_arch=70
-    - mfem +cuda cuda_arch=70
-    - parsec +cuda cuda_arch=70
-    - petsc +cuda cuda_arch=70
-    - raja +cuda cuda_arch=70
-    - slate +cuda cuda_arch=70
-    - slepc +cuda cuda_arch=70
-    - strumpack ~slate +cuda cuda_arch=70
-    - sundials +cuda cuda_arch=70
-    - superlu-dist +cuda cuda_arch=70
-    - tasmanian +cuda cuda_arch=70
-    - trilinos@13.2.0 +cuda cuda_arch=70
-    - umpire ~shared +cuda cuda_arch=70
-    - vtk-m +cuda cuda_arch=70
-    - zfp +cuda cuda_arch=70
-   #- ascent ~shared +cuda cuda_arch=70
-   #- axom +cuda cuda_arch=70 ^umpire ~shared
-   #- dealii +cuda cuda_arch=70 # gmsh
-   #- flecsi +cuda cuda_arch=70
-   #- paraview +cuda cuda_arch=70
-
-  - default_specs:
-    - adios
-    - adios2
-    - aml
-    - amrex
-    - arborx
-    - archer
-    - argobots
-    - ascent
-    - axom ^umpire@4.1.2
-    - bolt
-    - cabana
-    - caliper
-    - chai ~benchmarks ~tests ^umpire@4.1.2
-    - charliecloud
-    - conduit
-    - darshan-runtime
-    - darshan-util
-    - datatransferkit
-    - dyninst
-    - faodel ~tcmalloc
-    - flecsi
-    - flit
-    - flux-core
-    - fortrilinos
-    - gasnet
-    - ginkgo
-    - globalarrays
-    - gmp
-    - gotcha
-    - gptune
-    - hdf5
-    - heffte +fftw
-    - hpctoolkit
-    - hpx
-    - hypre
-    - kokkos +openmp
-    - kokkos-kernels +openmp
-    - legion
-    - libnrm
-    - libquo
-    - libunwind
-    - llvm targets=amdgpu,nvptx +clang +compiler-rt +libcxx +lld +lldb +llvm_dylib +flang ~cuda
-    - loki
-    - mercury
-    - metall
-    - mfem
-    - mpark-variant
-    - mpifileutils ~xattr
-    - netlib-scalapack
-    - ninja
-    - nrm
-    - nvhpc
-    - omega-h
-    - openmpi
-    - openpmd-api ^hdf5@1.12.0 +fortran +shared +hl api=default
-    - papi
-    - papyrus@1.0.1
-    - parallel-netcdf
-    - paraview
-    - parsec ~cuda
-    - pdt
-    - petsc
-    - plasma
-    - precice
-    - pumi
-    - py-jupyterhub
-    - py-libensemble
-    - py-petsc4py
-    - py-warpx ^warpx dims=2 ^hdf5@1.12.0 +fortran +shared +hl api=default
-    - py-warpx ^warpx dims=3 ^hdf5@1.12.0 +fortran +shared +hl api=default
-    - py-warpx ^warpx dims=rz ^hdf5@1.12.0 +fortran +shared +hl api=default
-    - qthreads scheduler=distrib
-    - raja
-    - rempi
-    - scr
-    - slate ~cuda
-    - slepc
-    - stc
-    - strumpack ~slate
-    - sundials
-    - superlu
-    - superlu-dist
-    - swig
-    - swig@4.0.2-fortran
-    - sz
-    - tasmanian
-    - tau +mpi +python
-    - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
-    - turbine
-    - umap
-    - unifyfs@0.9.1
-    - upcxx
-    - variorum
-    - veloc
-    - vtk-m
-    - zfp
-    #- dealii
-    #- geopm
-    #- phist
-    #- qt
-    #- qwt
-    #- stat
-    #- umpire
-
-  - arch:
-    - '%gcc target=ppc64le'
-
-
-  specs:
-
-  - matrix:
-    - - $default_specs
-    - - $arch
-
-  - matrix:
-    - - $cuda_specs
-    - - $arch
-
-  mirrors: { "mirror": "s3://spack-binaries/e4s" }
-
-  gitlab-ci:
-
-    script:
-      - uname -a || true
-      - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
-      - nproc
-      - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.powerpc64le-linux-gnu.tar.gz' -o gmake.tar.gz
-      - printf '8096d202fe0a0c400b8c0573c4b9e009f2f10d2fa850a3f495340f16e9c42454 gmake.tar.gz' | sha256sum --check --strict --quiet
-      - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
-      - . "./share/spack/setup-env.sh"
-      - spack --version
-      - spack arch
-      - cd ${SPACK_CONCRETE_ENV_DIR}
-      - spack env activate --without-view .
-      - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
-      - spack --color=always --backtrace ci rebuild
-    after_script:
-      - cat /proc/loadavg || true
-
-    match_behavior: first
-    mappings:
-      - match:
-        - cuda
-        - dyninst
-        - hpx
-        - llvm
-        - llvm-amdgpu
-        - precice
-        - rocblas
-        - rocsolver
-        - strumpack
-        - sundials
-        - trilinos
-        - vtk-h
-        - vtk-m
-        - warpx
-        runner-attributes:
-          image: { "name": "ghcr.io/spack/e4s-ubuntu-20.04:v2021-10-18", "entrypoint": [""] }
-          tags: ["spack", "public", "xlarge", "ppc64le"]
-      - match: ['os=ubuntu20.04']
-        runner-attributes:
-          image: { "name": "ghcr.io/spack/e4s-ubuntu-20.04:v2021-10-18", "entrypoint": [""] }
-          tags: ["spack", "public", "large", "ppc64le"]
-    broken-specs-url: "s3://spack-binaries-develop/broken-specs"
-    service-job-attributes:
-      before_script:
-        - . "./share/spack/setup-env.sh"
-        - spack --version
-      image: { "name": "ghcr.io/spack/e4s-ubuntu-20.04:v2021-10-18", "entrypoint": [""] }
-      tags: ["spack", "public", "medium", "ppc64le"]
-
-  cdash:
-    build-group: New PR testing workflow
-    url: https://cdash.spack.io
-    project: Spack Testing
-    site: Cloud Gitlab Infrastructure
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
new file mode 100644
index 0000000000..e4b15c8a37
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
@@ -0,0 +1,423 @@
+spack:
+  view: false
+
+  concretizer:
+    reuse: false
+    unify: false
+
+  config:
+    concretizer: clingo
+    install_tree:
+      root: /home/software/spack
+      padded_length: 512
+      projections:
+        all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
+
+  packages:
+    all:
+      compiler: [gcc@11.1.0]
+      providers:
+        blas: [openblas]
+        mpi: [mpich]
+      target: [ppc64le]
+      variants: +mpi cuda_arch=70
+    tbb:
+      require: intel-tbb
+    binutils:
+      variants: +ld +gold +headers +libiberty ~nls
+    cuda:
+      version: [11.7.0]
+    elfutils:
+      variants: +bzip2 ~nls +xz
+    hdf5:
+      variants: +fortran +hl +shared
+    libfabric:
+      variants: fabrics=sockets,tcp,udp,rxm
+    libunwind:
+      variants: +pic +xz
+    mpich:
+      variants: ~wrapperrpath
+    ncurses:
+      variants: +termlib
+    openblas:
+      variants: threads=openmp
+    paraview:
+      require: '@5.11 ~qt+osmesa'
+    python:
+      version: [3.7.15]
+    trilinos:
+      require: +amesos +amesos2 +anasazi +aztec +boost +epetra +epetraext +ifpack
+        +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro
+        +phalanx +rol +rythmos +sacado +stk +shards +shylu +stratimikos +teko +tempus
+        +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+    xz:
+      variants: +pic
+    mesa:
+      version: [21.3.8]
+    faodel:
+      require: ~tcmalloc # needed for ppc64le
+
+  specs:
+  # CPU
+  - adios
+  - alquimia
+  - aml
+  - amrex
+  - arborx
+  - argobots
+  - axom
+  - bolt
+  - butterflypack
+  - cabana
+  - caliper
+  - chai ~benchmarks ~tests
+  - charliecloud
+  - conduit
+  - datatransferkit
+  - dyninst
+  - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 ~paraview +pnetcdf +sz +unifyfs +veloc +visit +vtkm +zfp # +paraview fails: FAILED: VTK/Filters/Statistics/CMakeFiles/FiltersStatistics-objects.dir/vtkPCAStatistics.cxx.o: /tmp/ccgvkIk5.s: Assembler messages: /tmp/ccgvkIk5.s:260012: Error: invalid machine `power10'
+  - exaworks
+  - flecsi
+  - flit
+  - flux-core
+  - fortrilinos
+  - gasnet
+  - ginkgo
+  - globalarrays
+  - gmp
+  - gotcha
+  - gptune
+  - h5bench
+  - hdf5-vol-async
+  - heffte +fftw
+  - hpctoolkit
+  - hpx max_cpu_count=512 networking=mpi
+  - hypre
+  - kokkos +openmp
+  - kokkos-kernels +openmp
+  - lammps
+  - legion
+  - libnrm
+  - libquo
+  - libunwind
+  - mercury
+  - metall
+  - mfem
+  - mpark-variant
+  - mpifileutils ~xattr
+  - nccmp
+  - nco
+  - netlib-scalapack
+  - nrm
+  - nvhpc
+  - omega-h
+  - openmpi
+  - openpmd-api
+  - papi
+  - papyrus
+  - parsec ~cuda
+  - pdt
+  - petsc
+  - phist
+  - plasma
+  - plumed
+  - pumi
+  - py-h5py
+  - py-jupyterhub
+  - py-libensemble +mpi +nlopt
+  - py-petsc4py
+  - py-warpx ^warpx dims=2
+  - py-warpx ^warpx dims=3
+  - py-warpx ^warpx dims=rz
+  - qthreads scheduler=distrib
+  - quantum-espresso
+  - raja
+  - rempi
+  - scr
+  - slate ~cuda
+  - slepc
+  - stc
+  - strumpack ~slate
+  - sundials
+  - superlu
+  - superlu-dist
+  - swig
+  - swig@4.0.2-fortran
+  - tasmanian
+  - tau +mpi +python
+  - trilinos@13.0.1 +belos +ifpack2 +stokhos
+  - turbine
+  - umap
+  - umpire
+  - upcxx
+  - wannier90
+
+  # CUDA
+  - amrex +cuda
+  - arborx +cuda ^kokkos +wrapper
+  - cabana +cuda ^kokkos +wrapper +cuda_lambda +cuda
+  - caliper +cuda
+  - chai ~benchmarks ~tests +cuda ^umpire ~shared
+  - ecp-data-vis-sdk +cuda cuda_arch=70 +adios2 +hdf5 ~paraview +vtkm +zfp # +paraview fails:  FAILED: VTK/Filters/Statistics/CMakeFiles/FiltersStatistics-objects.dir/vtkPCAStatistics.cxx.o; /tmp/ccjmJhb6.s: Assembler messages: /tmp/ccjmJhb6.s:260012: Error: invalid machine `power10'
+  - flecsi +cuda
+  - flux-core +cuda
+  - ginkgo +cuda
+  - heffte +cuda
+  - hpctoolkit +cuda
+  - hpx max_cpu_count=512 +cuda
+  - hypre +cuda
+  - kokkos +wrapper +cuda
+  - kokkos-kernels +cuda ^kokkos +wrapper +cuda
+  - magma +cuda
+  - mfem +cuda
+  - omega-h +cuda
+  - papi +cuda
+  - petsc +cuda
+  - py-torch +cuda
+  - raja +cuda
+  - slate +cuda
+  - slepc +cuda
+  - strumpack ~slate +cuda
+  - sundials +cuda
+  - superlu-dist +cuda
+  - tasmanian +cuda
+  - tau +mpi +cuda
+  - trilinos@13.4.0 +belos +ifpack2 +stokhos +cuda
+  - umpire ~shared +cuda
+  - parsec +cuda
+
+  # CPU FAILURES
+  # - archer                  # llvm@8
+  # - bricks                  # bricks
+  # - geopm                   # geopm
+  # - loki                    # loki
+  # - precice                 # precice
+  # - pruners-ninja           # pruners-ninja
+  # - variorum                # Intel/variorum_cpuid.c:11:5: error: impossible constraint in 'asm'
+  # --
+  # bricks: VSBrick-7pt.py-Scalar-8x8x8-1:30:3: error: 'vfloat512' was not declared in this scope
+  # fltk: /usr/bin/ld: ../lib/libfltk_png.a(pngrutil.o): in function `png_read_filter_row': pngrutil.c:(.text.png_read_filter_row+0x90): undefined reference to `png_init_filter_functions_vsx'
+  # geopm: libtool.m4: error: problem compiling CXX test program
+  # llvm@8: clang/lib/Lex/Lexer.cpp:2547:34: error: ISO C++ forbids declaration of 'type name' with no type [-fpermissive]
+  # loki: include/loki/SmallObj.h:462:57: error: ISO C++17 does not allow dynamic exception specifications
+  # precice: /tmp/ccYNMwgE.s: Assembler messages: /tmp/ccYNMwgE.s:278115: Error: invalid machine `power10'
+  # pruners-ninja: test/ninja_test_util.c:34: multiple definition of `a';
+
+  # CUDA FAILURES
+  # - bricks +cuda              # bricks
+  # - dealii +cuda              # fltk
+  # --
+  # bricks: VSBrick-7pt.py-Scalar-8x8x8-1:30:3: error: 'vfloat512' was not declared in this scope
+
+
+  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-power" }
+
+  gitlab-ci:
+
+    script:
+      - uname -a || true
+      - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
+      - nproc
+      - . "./share/spack/setup-env.sh"
+      - spack --version
+      - spack arch
+      - cd ${SPACK_CONCRETE_ENV_DIR}
+      - spack env activate --without-view .
+      - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
+      - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
+      - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi
+      - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
+      - spack --color=always --backtrace ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+    after_script:
+      - cat /proc/loadavg || true
+
+    image: ecpe4s/ubuntu20.04-runner-ppc64le:2023-01-01
+
+    match_behavior: first
+    mappings:
+      - match:
+          - hipblas
+          - llvm
+          - llvm-amdgpu
+          - rocblas
+        runner-attributes:
+          tags: [ "spack", "huge", "ppc64le" ]
+          variables:
+            CI_JOB_SIZE: huge
+
+      - match:
+          - cuda
+          - dyninst
+          - ginkgo
+          - hpx
+          - kokkos-kernels
+          - kokkos-nvcc-wrapper
+          - magma
+          - mfem
+          - mpich
+          - openturns
+          - precice
+          - raja
+          - rust
+          - slate
+          - trilinos
+          - vtk-m
+          - warpx
+        runner-attributes:
+          tags: [ "spack", "large", "ppc64le" ]
+          variables:
+            CI_JOB_SIZE: large
+
+      - match:
+          - adios2
+          - amrex
+          - archer
+          - ascent
+          - axom
+          - binutils
+          - blaspp
+          - boost
+          - butterflypack
+          - cabana
+          - caliper
+          - camp
+          - chai
+          - conduit
+          - datatransferkit
+          - faodel
+          - ffmpeg
+          - fftw
+          - fortrilinos
+          - gperftools
+          - gptune
+          - hdf5
+          - heffte
+          - hpctoolkit
+          - hwloc
+          - hypre
+          - kokkos
+          - lammps
+          - lapackpp
+          - legion
+          - libzmq
+          - llvm-openmp-ompt
+          - mbedtls
+          - netlib-scalapack
+          - omega-h
+          - openmpi
+          - openpmd-api
+          - pagmo2
+          - papyrus
+          - parsec
+          - pdt
+          - petsc
+          - pumi
+          - py-ipython-genutils
+          - py-petsc4py
+          - py-scipy
+          - py-statsmodels
+          - py-warlock
+          - py-warpx
+          - pygmo
+          - slepc
+          - slurm
+          - strumpack
+          - sundials
+          - superlu-dist
+          - tasmanian
+          - tau
+          - upcxx
+          - vtk-h
+          - zfp
+        runner-attributes:
+          tags: [ "spack", "medium", "ppc64le" ]
+          variables:
+            CI_JOB_SIZE: "medium"
+
+      - match:
+          - alsa-lib
+          - ant
+          - antlr
+          - argobots
+          - automake
+          - berkeley-db
+          - bison
+          - blt
+          - cmake
+          - curl
+          - darshan-util
+          - diffutils
+          - exmcutils
+          - expat
+          - flit
+          - freetype
+          - gdbm
+          - gotcha
+          - hpcviewer
+          - jansson
+          - json-c
+          - libbsd
+          - libevent
+          - libjpeg-turbo
+          - libnrm
+          - libpng
+          - libunistring
+          - lua-luaposix
+          - m4
+          - mpfr
+          - ncurses
+          - openblas
+          - openjdk
+          - papi
+          - parallel-netcdf
+          - pcre2
+          - perl-data-dumper
+          - pkgconf
+          - py-alembic
+          - py-idna
+          - py-testpath
+          - qhull
+          - snappy
+          - swig
+          - tar
+          - tcl
+          - texinfo
+          - unzip
+          - util-linux-uuid
+          - util-macros
+          - yaml-cpp
+          - zlib
+          - zstd
+        runner-attributes:
+          tags: [ "spack", "small", "ppc64le" ]
+          variables:
+            CI_JOB_SIZE: "small"
+
+      - match: ['os=ubuntu20.04']
+        runner-attributes:
+          tags: ["spack", "ppc64le"]
+          variables:
+            CI_JOB_SIZE: "default"
+
+    broken-specs-url: "s3://spack-binaries/broken-specs"
+
+    service-job-attributes:
+      before_script:
+        - . "./share/spack/setup-env.sh"
+        - spack --version
+      image: ecpe4s/ubuntu20.04-runner-ppc64le:2023-01-01
+      tags: ["spack", "public", "ppc64le"]
+
+    signing-job-attributes:
+      image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
+      tags: ["spack", "aws"]
+      script:
+        - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
+        - /sign.sh
+        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
+
+  cdash:
+    build-group: E4S Power
+    url: https://cdash.spack.io
+    project: Spack Testing
+    site: Cloud Gitlab Infrastructure