E4S Cray CI Stack (#37837)

* e4s cray ci stack

* e4s ci: add cray

* add zen4 tag

* WIP: new defintions just for cray

* updates

* remove ci signing job overrride, not necessary

* echo $PATH and show modules loaded

* add mirror

* add external def for cray-libsci

* comment out quantum-espresso

* use /etc/protected-runner as key path

* cray ci stack: do not remove tags: [spack, public]

* make cray stack composable

* generate job should run on public tagged runner, override default config:install_tree:root

* CI: Use relative path in default script

* CI: Use relative includes paths for shell runners

* Use concrete_env_dir for relpath

* ml-darwin-aarch64-mps: jax has bazel codesign issue

---------

Co-authored-by: Scott Wittenburg <scott.wittenburg@kitware.com>
Co-authored-by: Ryan Krattiger <ryan.krattiger@kitware.com>

6 files changed, 502 insertions, 1 deletions

diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
index 50f51d9eeb..24bad76570 100644
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -4,6 +4,11 @@ default:
   image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
 
 # CI Platform-Arch
+.cray_zen4:
+  variables:
+    SPACK_TARGET_PLATFORM: "cray"
+    SPACK_TARGET_ARCH: "zen4"
+
 .darwin_x86_64:
   variables:
     SPACK_TARGET_PLATFORM: "darwin"
@@ -815,3 +820,89 @@ aws-pcluster-build-neoverse_v1:
   needs:
     - artifacts: True
       job: aws-pcluster-generate-neoverse_v1
+
+# Cray definitions
+.base-cray-job:
+  variables:
+    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-cray/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+    AWS_ACCESS_KEY_ID: ${CRAY_MIRRORS_AWS_ACCESS_KEY_ID}
+    AWS_SECRET_ACCESS_KEY: ${CRAY_MIRRORS_AWS_SECRET_ACCESS_KEY}
+  rules:
+    - if: $CI_COMMIT_REF_NAME == "develop"
+      # Pipelines on develop only rebuild what is missing from the mirror
+      when: always
+      variables:
+        SPACK_PIPELINE_TYPE: "spack_protected_branch"
+    - if: $CI_COMMIT_REF_NAME =~ /^pr[\d]+_.*$/
+      # Pipelines on PR branches rebuild only what's missing, and do extra pruning
+      when: always
+      variables:
+        SPACK_PIPELINE_TYPE: "spack_pull_request"
+        SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-cray/prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+        SPACK_PRUNE_UNTOUCHED: "True"
+        SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
+
+.generate-cray:
+  tags: [ "cce@15.0.1", "cray-zen4", "public" ]
+  extends: [ ".base-cray-job" ]
+  stage: generate
+  script:
+    - echo $PATH
+    - module avail
+    - module list
+    - export SPACK_DISABLE_LOCAL_CONFIG=1
+    - export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache
+    - uname -a || true
+    - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
+    - nproc || true
+    - . "./share/spack/setup-env.sh"
+    - spack --version
+    - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
+    - spack env activate --without-view .
+    - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
+    - spack
+      --config-scope "${SPACK_CI_CONFIG_ROOT}"
+      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
+      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
+      ${CI_STACK_CONFIG_SCOPES}
+      ci generate --check-index-only
+      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
+      --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
+      --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
+  after_script:
+    - cat /proc/loadavg || true
+  artifacts:
+    paths:
+      - "${CI_PROJECT_DIR}/jobs_scratch_dir"
+  interruptible: true
+  timeout: 60 minutes
+  retry:
+    max: 2
+    when:
+      - always
+
+.build-cray:
+  extends: [ ".base-cray-job" ]
+  stage: build
+
+#######################################
+# E4S - Cray
+#######################################
+.e4s-cray:
+  extends: [ ".cray_zen4" ]
+  variables:
+    SPACK_CI_STACK_NAME: e4s-cray
+
+e4s-cray-generate:
+  extends: [ ".generate-cray", ".e4s-cray" ]
+
+e4s-cray-build:
+  extends: [ ".build-cray", ".e4s-cray" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: e4s-cray-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: e4s-cray-generate
\ No newline at end of file
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray/ci.yaml
new file mode 100644
index 0000000000..114dcc81dc
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray/ci.yaml
@@ -0,0 +1,295 @@
+ci:
+  pipeline-gen:
+  - build-job-remove:
+      image: no-image
+  - build-job:
+      script+:
+      # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
+      - if [[ -r /etc/protected-runner/e4s.gpg ]]; then spack gpg trust /etc/protected-runner/e4s.gpg; fi
+      # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification)
+      - if [[ -r /etc/protected-runner/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /etc/protected-runner/intermediate_ci_signing_key.gpg; fi
+      - if [[ -r /etc/protected-runner/spack_public_key.gpg ]]; then spack gpg trust /etc/protected-runner/spack_public_key.gpg; fi
+  - match_behavior: first
+    submapping:
+    - match:
+      - hipblas
+      - llvm
+      - llvm-amdgpu
+      - pango
+      - paraview
+      - py-tensorflow
+      - py-torch
+      - qt
+      - rocblas
+      - visit
+      build-job:
+        tags: [ "spack", "huge" ]
+        variables:
+          CI_JOB_SIZE: huge
+          SPACK_BUILD_JOBS: "12"
+
+    - match:
+      - ascent
+      - atk
+      - axom
+      - cistem
+      - cmake
+      - ctffind
+      - cuda
+      - dealii
+      - dray
+      - dyninst
+      - ecp-data-vis-sdk
+      - gcc
+      - ginkgo
+      - hdf5
+      - hpx
+      - kokkos-kernels
+      - kokkos-nvcc-wrapper
+      - lbann
+      - magma
+      - mesa
+      - mfem
+      - mpich
+      - netlib-lapack
+      - nvhpc
+      - oce
+      - openblas
+      - openfoam
+      - openturns
+      - parallelio
+      - plumed
+      - precice
+      #- py-tensorflow
+      #- qt
+      - raja
+      - relion
+      #- rocblas
+      - rocfft
+      - rocsolver
+      - rocsparse
+      - rust
+      - slate
+      - strumpack
+      - sundials
+      - trilinos
+      - umpire
+      #- visit
+      - vtk
+      - vtk-h
+      - vtk-m
+      - warpx
+      - wrf
+      - wxwidgets
+      build-job:
+        tags: [ "spack", "large" ]
+        variables:
+          CI_JOB_SIZE: large
+          SPACK_BUILD_JOBS: "8"
+
+    - match:
+      - adios2
+      - amrex
+      - archer
+      - ascent
+      - autoconf-archive
+      - axom
+      - binutils
+      - blaspp
+      - blt
+      - boost
+      - butterflypack
+      - cabana
+      - caliper
+      - camp
+      - chai
+      - conduit
+      - curl
+      - datatransferkit
+      - double-conversion
+      - dray
+      - eigen
+      - faodel
+      - ffmpeg
+      - fftw
+      - fortrilinos
+      - gettext
+      - gperftools
+      - gptune
+      - hdf5
+      - heffte
+      - hpctoolkit
+      - hwloc
+      - hydrogen
+      - hypre
+      - kokkos
+      - lammps
+      - lapackpp
+      - legion
+      - libtool
+      - libxml2
+      - libzmq
+      - llvm-openmp-ompt
+      - mbedtls
+      - mfem
+      - mpich
+      - mvapich2
+      - nasm
+      - netlib-scalapack
+      - omega-h
+      - openblas
+      - openjpeg
+      - openmpi
+      - openpmd-api
+      - pagmo2
+      - papyrus
+      - parsec
+      - pdt
+      - pegtl
+      - petsc
+      - pumi
+      - py-beniget
+      - py-cinemasci
+      - pygmo
+      - py-ipython-genutils
+      - py-packaging
+      - py-petsc4py
+      - py-scipy
+      - py-statsmodels
+      - py-warlock
+      - py-warpx
+      - raja
+      - samrai
+      - slepc
+      - slurm
+      - sqlite
+      - strumpack
+      - sundials
+      - superlu-dist
+      - tasmanian
+      - tau
+      - upcxx
+      - vtk
+      - vtk-h
+      - vtk-m
+      - zfp
+      build-job:
+        tags: [ "spack", "medium" ]
+        variables:
+          CI_JOB_SIZE: "medium"
+          SPACK_BUILD_JOBS: "2"
+
+    - match:
+      - alsa-lib
+      - ant
+      - antlr
+      - argobots
+      - autoconf-archive
+      - automake
+      - berkeley-db
+      - bison
+      - blt
+      - bzip2
+      - camp
+      - cmake
+      - curl
+      - czmq
+      - darshan-util
+      - diffutils
+      - docbook-xml
+      - exmcutils
+      - expat
+      - findutils
+      - flit
+      - freetype
+      - gawk
+      - gdbm
+      - gettext
+      - glib
+      - gmake
+      - gotcha
+      - hpcviewer
+      - hwloc
+      - jansson
+      - json-c
+      - libbsd
+      - libedit
+      - libevent
+      - libfabric
+      - libffi
+      - libgcrypt
+      - libiconv
+      - libidn2
+      - libjpeg-turbo
+      - libmd
+      - libnrm
+      - libpciaccess
+      - libpng
+      - libsigsegv
+      - libsodium
+      - libunistring
+      - libunwind
+      - libxml2
+      - libyaml
+      - libzmq
+      - lua
+      - lua-luaposix
+      - lz4
+      - m4
+      - meson
+      - metis
+      - mpfr
+      - ncurses
+      - ninja
+      - numactl
+      - openblas
+      - openjdk
+      - openssh
+      - openssl
+      - papi
+      - parallel-netcdf
+      - pcre
+      - pcre2
+      - pdsh
+      - perl
+      - perl-data-dumper
+      - pkgconf
+      - py-alembic
+      - py-cffi
+      - py-cycler
+      - py-decorator
+      - py-idna
+      - py-jsonschema
+      - py-kiwisolver
+      - py-mistune
+      - py-pycparser
+      - py-setuptools
+      - py-setuptools-scm
+      - py-six
+      - py-testpath
+      - py-wheel
+      - qhull
+      - readline
+      - sed
+      - slurm
+      - snappy
+      - sqlite
+      - superlu
+      - swig
+      - tar
+      - tcl
+      - texinfo
+      - tut
+      - unzip
+      - util-linux-uuid
+      - util-macros
+      - xz
+      - yaml-cpp
+      - zfp
+      - zlib
+      - zstd
+      build-job:
+        tags: [ "spack", "small" ]
+        variables:
+          CI_JOB_SIZE: "small"
+          SPACK_BUILD_JOBS: "1"
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray/config.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray/config.yaml
new file mode 100644
index 0000000000..6cf3846bdd
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray/config.yaml
@@ -0,0 +1,9 @@
+config:
+  concretizer: clingo
+  db_lock_timeout: 120
+  install_tree:
+    root: $spack/opt/spack
+    padded_length: 256
+    projections:
+      all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
+
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray/zen4/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray/zen4/ci.yaml
new file mode 100644
index 0000000000..3bee4c4000
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray/zen4/ci.yaml
@@ -0,0 +1,4 @@
+ci:
+  pipeline-gen:
+  - build-job:
+      tags: ["cce@15.0.1", "cray-zen4"]
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray/spack.yaml
new file mode 100644
index 0000000000..27d823a510
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray/spack.yaml
@@ -0,0 +1,102 @@
+spack:
+  view: false
+
+  concretizer:
+    reuse: false
+    unify: false
+
+  compilers:
+  - compiler:
+      spec: cce@15.0.1
+      paths:
+        cc: cc
+        cxx: CC
+        f77: ftn
+        fc: ftn
+      flags: {}
+      operating_system: rhel8
+      target: any
+      modules:
+      - PrgEnv-cray/8.3.3
+      - cce/15.0.1
+      environment:
+        set:
+          MACHTYPE: x86_64
+  - compiler:
+      spec: gcc@11.2.0
+      paths:
+        cc: gcc
+        cxx: g++
+        f77: gfortran
+        fc: gfortran
+      flags: {}
+      operating_system: rhel8
+      target: any
+      modules:
+      - PrgEnv-gnu
+      - gcc/11.2.0
+      environment: {}
+
+  packages:
+    all:
+      require: '%cce@15.0.1'
+      compiler: [cce@15.0.1]
+      providers:
+        blas: [cray-libsci]
+        lapack: [cray-libsci]
+        mpi: [cray-mpich]
+        tbb: [intel-tbb]
+        scalapack: [netlib-scalapack]
+      target: [zen4]
+      variants: +mpi
+
+    binutils:
+      variants: +ld +gold +headers +libiberty ~nls
+    hdf5:
+      variants: +fortran +hl +shared
+    libunwind:
+      variants: +pic +xz
+    ncurses:
+      require: '@6.3 +termlib'
+    openblas:
+      require: '@0.3.20'
+      variants: threads=openmp
+    python:
+      require: '@3.7.15'
+    xz:
+      variants: +pic
+    elfutils:
+      variants: +bzip2 ~nls +xz
+      require: '%gcc'
+
+    # EXTERNALS
+    cray-mpich:
+      buildable: false
+      externals:
+      - spec: cray-mpich@8.1.25 %cce@15.0.1
+        prefix: /opt/cray/pe/mpich/8.1.25/ofi/cray/10.0
+        modules:
+        - cray-mpich/8.1.25
+    cray-libsci:
+      buildable: false
+      externals:
+      - spec: cray-libsci@23.02.1.1 %cce@15.0.1
+        prefix: /opt/cray/pe/libsci/23.02.1.1/CRAY/9.0/x86_64/
+        modules:
+        - cray-libsci/23.02.1.1
+
+  specs:
+  - butterflypack
+  - hypre
+  - kokkos
+  - kokkos-kernels
+  - petsc
+  - raja
+  - slepc
+  - superlu-dist
+  - tau
+
+  mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray" }
+
+  cdash:
+    build-group: E4S Cray
\ No newline at end of file
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
index c74c12f865..efb29e8617 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
@@ -26,7 +26,7 @@ spack:
   - py-transformers
 
   # JAX
-  - py-jax
+  # - py-jax        # bazel codesign
   # - py-jaxlib     # bazel codesign
 
   # Keras