diff options
author | eugeneswalker <38933153+eugeneswalker@users.noreply.github.com> | 2023-06-06 20:41:32 -0500 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-06-06 18:41:32 -0700 |
commit | 5f1bc15e80deb15efde63632cfcba6d2a3c1f363 (patch) | |
tree | 8961c51b8b5c5ed9659fd46a628389de8e44ead8 /share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml | |
parent | fa9fb60df332a430ed10ce007d3a07ec90aefcb5 (diff) | |
download | spack-5f1bc15e80deb15efde63632cfcba6d2a3c1f363.tar.gz spack-5f1bc15e80deb15efde63632cfcba6d2a3c1f363.tar.bz2 spack-5f1bc15e80deb15efde63632cfcba6d2a3c1f363.tar.xz spack-5f1bc15e80deb15efde63632cfcba6d2a3c1f363.zip |
E4S Cray CI Stack (#37837)
* e4s cray ci stack
* e4s ci: add cray
* add zen4 tag
* WIP: new defintions just for cray
* updates
* remove ci signing job overrride, not necessary
* echo $PATH and show modules loaded
* add mirror
* add external def for cray-libsci
* comment out quantum-espresso
* use /etc/protected-runner as key path
* cray ci stack: do not remove tags: [spack, public]
* make cray stack composable
* generate job should run on public tagged runner, override default config:install_tree:root
* CI: Use relative path in default script
* CI: Use relative includes paths for shell runners
* Use concrete_env_dir for relpath
* ml-darwin-aarch64-mps: jax has bazel codesign issue
---------
Co-authored-by: Scott Wittenburg <scott.wittenburg@kitware.com>
Co-authored-by: Ryan Krattiger <ryan.krattiger@kitware.com>
Diffstat (limited to 'share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml')
-rw-r--r-- | share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml | 91 |
1 files changed, 91 insertions, 0 deletions
diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml index 50f51d9eeb..24bad76570 100644 --- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml +++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml @@ -4,6 +4,11 @@ default: image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } # CI Platform-Arch +.cray_zen4: + variables: + SPACK_TARGET_PLATFORM: "cray" + SPACK_TARGET_ARCH: "zen4" + .darwin_x86_64: variables: SPACK_TARGET_PLATFORM: "darwin" @@ -815,3 +820,89 @@ aws-pcluster-build-neoverse_v1: needs: - artifacts: True job: aws-pcluster-generate-neoverse_v1 + +# Cray definitions +.base-cray-job: + variables: + SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-cray/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}" + AWS_ACCESS_KEY_ID: ${CRAY_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${CRAY_MIRRORS_AWS_SECRET_ACCESS_KEY} + rules: + - if: $CI_COMMIT_REF_NAME == "develop" + # Pipelines on develop only rebuild what is missing from the mirror + when: always + variables: + SPACK_PIPELINE_TYPE: "spack_protected_branch" + - if: $CI_COMMIT_REF_NAME =~ /^pr[\d]+_.*$/ + # Pipelines on PR branches rebuild only what's missing, and do extra pruning + when: always + variables: + SPACK_PIPELINE_TYPE: "spack_pull_request" + SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-cray/prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}" + SPACK_PRUNE_UNTOUCHED: "True" + SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1" + +.generate-cray: + tags: [ "cce@15.0.1", "cray-zen4", "public" ] + extends: [ ".base-cray-job" ] + stage: generate + script: + - echo $PATH + - module avail + - module list + - export SPACK_DISABLE_LOCAL_CONFIG=1 + - export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache + - uname -a || true + - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true + - nproc || true + - . "./share/spack/setup-env.sh" + - spack --version + - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME} + - spack env activate --without-view . + - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs" + - spack + --config-scope "${SPACK_CI_CONFIG_ROOT}" + --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}" + --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}" + ${CI_STACK_CONFIG_SCOPES} + ci generate --check-index-only + --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}" + --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir" + --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml" + after_script: + - cat /proc/loadavg || true + artifacts: + paths: + - "${CI_PROJECT_DIR}/jobs_scratch_dir" + interruptible: true + timeout: 60 minutes + retry: + max: 2 + when: + - always + +.build-cray: + extends: [ ".base-cray-job" ] + stage: build + +####################################### +# E4S - Cray +####################################### +.e4s-cray: + extends: [ ".cray_zen4" ] + variables: + SPACK_CI_STACK_NAME: e4s-cray + +e4s-cray-generate: + extends: [ ".generate-cray", ".e4s-cray" ] + +e4s-cray-build: + extends: [ ".build-cray", ".e4s-cray" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: e4s-cray-generate + strategy: depend + needs: + - artifacts: True + job: e4s-cray-generate
\ No newline at end of file |