From 5f1bc15e80deb15efde63632cfcba6d2a3c1f363 Mon Sep 17 00:00:00 2001 From: eugeneswalker <38933153+eugeneswalker@users.noreply.github.com> Date: Tue, 6 Jun 2023 20:41:32 -0500 Subject: E4S Cray CI Stack (#37837) * e4s cray ci stack * e4s ci: add cray * add zen4 tag * WIP: new defintions just for cray * updates * remove ci signing job overrride, not necessary * echo $PATH and show modules loaded * add mirror * add external def for cray-libsci * comment out quantum-espresso * use /etc/protected-runner as key path * cray ci stack: do not remove tags: [spack, public] * make cray stack composable * generate job should run on public tagged runner, override default config:install_tree:root * CI: Use relative path in default script * CI: Use relative includes paths for shell runners * Use concrete_env_dir for relpath * ml-darwin-aarch64-mps: jax has bazel codesign issue --------- Co-authored-by: Scott Wittenburg Co-authored-by: Ryan Krattiger --- share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml | 91 +++++++ .../gitlab/cloud_pipelines/configs/cray/ci.yaml | 295 +++++++++++++++++++++ .../cloud_pipelines/configs/cray/config.yaml | 9 + .../cloud_pipelines/configs/cray/zen4/ci.yaml | 4 + .../cloud_pipelines/stacks/e4s-cray/spack.yaml | 102 +++++++ .../stacks/ml-darwin-aarch64-mps/spack.yaml | 2 +- 6 files changed, 502 insertions(+), 1 deletion(-) create mode 100644 share/spack/gitlab/cloud_pipelines/configs/cray/ci.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/configs/cray/config.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/configs/cray/zen4/ci.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/stacks/e4s-cray/spack.yaml (limited to 'share') diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml index 50f51d9eeb..24bad76570 100644 --- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml +++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml @@ -4,6 +4,11 @@ default: image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } # CI Platform-Arch +.cray_zen4: + variables: + SPACK_TARGET_PLATFORM: "cray" + SPACK_TARGET_ARCH: "zen4" + .darwin_x86_64: variables: SPACK_TARGET_PLATFORM: "darwin" @@ -815,3 +820,89 @@ aws-pcluster-build-neoverse_v1: needs: - artifacts: True job: aws-pcluster-generate-neoverse_v1 + +# Cray definitions +.base-cray-job: + variables: + SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-cray/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}" + AWS_ACCESS_KEY_ID: ${CRAY_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${CRAY_MIRRORS_AWS_SECRET_ACCESS_KEY} + rules: + - if: $CI_COMMIT_REF_NAME == "develop" + # Pipelines on develop only rebuild what is missing from the mirror + when: always + variables: + SPACK_PIPELINE_TYPE: "spack_protected_branch" + - if: $CI_COMMIT_REF_NAME =~ /^pr[\d]+_.*$/ + # Pipelines on PR branches rebuild only what's missing, and do extra pruning + when: always + variables: + SPACK_PIPELINE_TYPE: "spack_pull_request" + SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-cray/prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}" + SPACK_PRUNE_UNTOUCHED: "True" + SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1" + +.generate-cray: + tags: [ "cce@15.0.1", "cray-zen4", "public" ] + extends: [ ".base-cray-job" ] + stage: generate + script: + - echo $PATH + - module avail + - module list + - export SPACK_DISABLE_LOCAL_CONFIG=1 + - export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache + - uname -a || true + - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true + - nproc || true + - . "./share/spack/setup-env.sh" + - spack --version + - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME} + - spack env activate --without-view . + - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs" + - spack + --config-scope "${SPACK_CI_CONFIG_ROOT}" + --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}" + --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}" + ${CI_STACK_CONFIG_SCOPES} + ci generate --check-index-only + --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}" + --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir" + --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml" + after_script: + - cat /proc/loadavg || true + artifacts: + paths: + - "${CI_PROJECT_DIR}/jobs_scratch_dir" + interruptible: true + timeout: 60 minutes + retry: + max: 2 + when: + - always + +.build-cray: + extends: [ ".base-cray-job" ] + stage: build + +####################################### +# E4S - Cray +####################################### +.e4s-cray: + extends: [ ".cray_zen4" ] + variables: + SPACK_CI_STACK_NAME: e4s-cray + +e4s-cray-generate: + extends: [ ".generate-cray", ".e4s-cray" ] + +e4s-cray-build: + extends: [ ".build-cray", ".e4s-cray" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: e4s-cray-generate + strategy: depend + needs: + - artifacts: True + job: e4s-cray-generate \ No newline at end of file diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray/ci.yaml new file mode 100644 index 0000000000..114dcc81dc --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/cray/ci.yaml @@ -0,0 +1,295 @@ +ci: + pipeline-gen: + - build-job-remove: + image: no-image + - build-job: + script+: + # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) + - if [[ -r /etc/protected-runner/e4s.gpg ]]; then spack gpg trust /etc/protected-runner/e4s.gpg; fi + # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) + - if [[ -r /etc/protected-runner/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /etc/protected-runner/intermediate_ci_signing_key.gpg; fi + - if [[ -r /etc/protected-runner/spack_public_key.gpg ]]; then spack gpg trust /etc/protected-runner/spack_public_key.gpg; fi + - match_behavior: first + submapping: + - match: + - hipblas + - llvm + - llvm-amdgpu + - pango + - paraview + - py-tensorflow + - py-torch + - qt + - rocblas + - visit + build-job: + tags: [ "spack", "huge" ] + variables: + CI_JOB_SIZE: huge + SPACK_BUILD_JOBS: "12" + + - match: + - ascent + - atk + - axom + - cistem + - cmake + - ctffind + - cuda + - dealii + - dray + - dyninst + - ecp-data-vis-sdk + - gcc + - ginkgo + - hdf5 + - hpx + - kokkos-kernels + - kokkos-nvcc-wrapper + - lbann + - magma + - mesa + - mfem + - mpich + - netlib-lapack + - nvhpc + - oce + - openblas + - openfoam + - openturns + - parallelio + - plumed + - precice + #- py-tensorflow + #- qt + - raja + - relion + #- rocblas + - rocfft + - rocsolver + - rocsparse + - rust + - slate + - strumpack + - sundials + - trilinos + - umpire + #- visit + - vtk + - vtk-h + - vtk-m + - warpx + - wrf + - wxwidgets + build-job: + tags: [ "spack", "large" ] + variables: + CI_JOB_SIZE: large + SPACK_BUILD_JOBS: "8" + + - match: + - adios2 + - amrex + - archer + - ascent + - autoconf-archive + - axom + - binutils + - blaspp + - blt + - boost + - butterflypack + - cabana + - caliper + - camp + - chai + - conduit + - curl + - datatransferkit + - double-conversion + - dray + - eigen + - faodel + - ffmpeg + - fftw + - fortrilinos + - gettext + - gperftools + - gptune + - hdf5 + - heffte + - hpctoolkit + - hwloc + - hydrogen + - hypre + - kokkos + - lammps + - lapackpp + - legion + - libtool + - libxml2 + - libzmq + - llvm-openmp-ompt + - mbedtls + - mfem + - mpich + - mvapich2 + - nasm + - netlib-scalapack + - omega-h + - openblas + - openjpeg + - openmpi + - openpmd-api + - pagmo2 + - papyrus + - parsec + - pdt + - pegtl + - petsc + - pumi + - py-beniget + - py-cinemasci + - pygmo + - py-ipython-genutils + - py-packaging + - py-petsc4py + - py-scipy + - py-statsmodels + - py-warlock + - py-warpx + - raja + - samrai + - slepc + - slurm + - sqlite + - strumpack + - sundials + - superlu-dist + - tasmanian + - tau + - upcxx + - vtk + - vtk-h + - vtk-m + - zfp + build-job: + tags: [ "spack", "medium" ] + variables: + CI_JOB_SIZE: "medium" + SPACK_BUILD_JOBS: "2" + + - match: + - alsa-lib + - ant + - antlr + - argobots + - autoconf-archive + - automake + - berkeley-db + - bison + - blt + - bzip2 + - camp + - cmake + - curl + - czmq + - darshan-util + - diffutils + - docbook-xml + - exmcutils + - expat + - findutils + - flit + - freetype + - gawk + - gdbm + - gettext + - glib + - gmake + - gotcha + - hpcviewer + - hwloc + - jansson + - json-c + - libbsd + - libedit + - libevent + - libfabric + - libffi + - libgcrypt + - libiconv + - libidn2 + - libjpeg-turbo + - libmd + - libnrm + - libpciaccess + - libpng + - libsigsegv + - libsodium + - libunistring + - libunwind + - libxml2 + - libyaml + - libzmq + - lua + - lua-luaposix + - lz4 + - m4 + - meson + - metis + - mpfr + - ncurses + - ninja + - numactl + - openblas + - openjdk + - openssh + - openssl + - papi + - parallel-netcdf + - pcre + - pcre2 + - pdsh + - perl + - perl-data-dumper + - pkgconf + - py-alembic + - py-cffi + - py-cycler + - py-decorator + - py-idna + - py-jsonschema + - py-kiwisolver + - py-mistune + - py-pycparser + - py-setuptools + - py-setuptools-scm + - py-six + - py-testpath + - py-wheel + - qhull + - readline + - sed + - slurm + - snappy + - sqlite + - superlu + - swig + - tar + - tcl + - texinfo + - tut + - unzip + - util-linux-uuid + - util-macros + - xz + - yaml-cpp + - zfp + - zlib + - zstd + build-job: + tags: [ "spack", "small" ] + variables: + CI_JOB_SIZE: "small" + SPACK_BUILD_JOBS: "1" diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray/config.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray/config.yaml new file mode 100644 index 0000000000..6cf3846bdd --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/cray/config.yaml @@ -0,0 +1,9 @@ +config: + concretizer: clingo + db_lock_timeout: 120 + install_tree: + root: $spack/opt/spack + padded_length: 256 + projections: + all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray/zen4/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray/zen4/ci.yaml new file mode 100644 index 0000000000..3bee4c4000 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/cray/zen4/ci.yaml @@ -0,0 +1,4 @@ +ci: + pipeline-gen: + - build-job: + tags: ["cce@15.0.1", "cray-zen4"] diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray/spack.yaml new file mode 100644 index 0000000000..27d823a510 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray/spack.yaml @@ -0,0 +1,102 @@ +spack: + view: false + + concretizer: + reuse: false + unify: false + + compilers: + - compiler: + spec: cce@15.0.1 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: rhel8 + target: any + modules: + - PrgEnv-cray/8.3.3 + - cce/15.0.1 + environment: + set: + MACHTYPE: x86_64 + - compiler: + spec: gcc@11.2.0 + paths: + cc: gcc + cxx: g++ + f77: gfortran + fc: gfortran + flags: {} + operating_system: rhel8 + target: any + modules: + - PrgEnv-gnu + - gcc/11.2.0 + environment: {} + + packages: + all: + require: '%cce@15.0.1' + compiler: [cce@15.0.1] + providers: + blas: [cray-libsci] + lapack: [cray-libsci] + mpi: [cray-mpich] + tbb: [intel-tbb] + scalapack: [netlib-scalapack] + target: [zen4] + variants: +mpi + + binutils: + variants: +ld +gold +headers +libiberty ~nls + hdf5: + variants: +fortran +hl +shared + libunwind: + variants: +pic +xz + ncurses: + require: '@6.3 +termlib' + openblas: + require: '@0.3.20' + variants: threads=openmp + python: + require: '@3.7.15' + xz: + variants: +pic + elfutils: + variants: +bzip2 ~nls +xz + require: '%gcc' + + # EXTERNALS + cray-mpich: + buildable: false + externals: + - spec: cray-mpich@8.1.25 %cce@15.0.1 + prefix: /opt/cray/pe/mpich/8.1.25/ofi/cray/10.0 + modules: + - cray-mpich/8.1.25 + cray-libsci: + buildable: false + externals: + - spec: cray-libsci@23.02.1.1 %cce@15.0.1 + prefix: /opt/cray/pe/libsci/23.02.1.1/CRAY/9.0/x86_64/ + modules: + - cray-libsci/23.02.1.1 + + specs: + - butterflypack + - hypre + - kokkos + - kokkos-kernels + - petsc + - raja + - slepc + - superlu-dist + - tau + + mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray" } + + cdash: + build-group: E4S Cray \ No newline at end of file diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml index c74c12f865..efb29e8617 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml @@ -26,7 +26,7 @@ spack: - py-transformers # JAX - - py-jax + # - py-jax # bazel codesign # - py-jaxlib # bazel codesign # Keras -- cgit v1.2.3-60-g2f50