summaryrefslogtreecommitdiff
path: root/share
diff options
context:
space:
mode:
authorkwryankrattiger <80296582+kwryankrattiger@users.noreply.github.com>2023-08-07 11:23:16 -0500
committerGitHub <noreply@github.com>2023-08-07 10:23:16 -0600
commitb54d286b4a341667f3620f43f7c13a5ba777fb63 (patch)
tree22ac216f984364a9745c765738caae98b99c3b78 /share
parentea9c488897a8729e31461731e9df934db8c27205 (diff)
downloadspack-b54d286b4a341667f3620f43f7c13a5ba777fb63.tar.gz
spack-b54d286b4a341667f3620f43f7c13a5ba777fb63.tar.bz2
spack-b54d286b4a341667f3620f43f7c13a5ba777fb63.tar.xz
spack-b54d286b4a341667f3620f43f7c13a5ba777fb63.zip
CI: remove redundant sections (#38514)
Refactor gitlab ci configs so that mac and cray jobs can reuse as much higher level configuration as possible. * CI: remove redundant sections * CI: Include base linux CI configs in cray stacks Relocation and runner mapping is consistent between cray and linux runners. * Export user cache path in before script * CI: add GPG root for mac runners * Disable user configs Metal runners share a ~ directory * Disable user config and add configs in activate env
Diffstat (limited to 'share')
-rw-r--r--share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml124
-rw-r--r--share/spack/gitlab/cloud_pipelines/configs/ci.yaml25
-rw-r--r--share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml284
-rw-r--r--share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml284
-rw-r--r--share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml4
-rw-r--r--share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml5
-rw-r--r--share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml5
-rw-r--r--share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml5
-rw-r--r--share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml50
9 files changed, 70 insertions, 716 deletions
diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
index 4b927a04ae..358d3af121 100644
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -19,6 +19,11 @@ default:
SPACK_TARGET_PLATFORM: "darwin"
SPACK_TARGET_ARCH: "x86_64"
+.darwin_aarch64:
+ variables:
+ SPACK_TARGET_PLATFORM: "darwin"
+ SPACK_TARGET_ARCH: "aarch64"
+
.linux_x86_64_v3:
variables:
SPACK_TARGET_PLATFORM: "linux"
@@ -60,6 +65,8 @@ default:
.base-job:
variables:
SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+ SPACK_DISABLE_LOCAL_CONFIG: "1"
+ SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/tmp/_user_cache/"
rules:
- if: $CI_COMMIT_REF_NAME == "develop"
# Pipelines on develop only rebuild what is missing from the mirror
@@ -101,7 +108,7 @@ default:
AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
-.generate-base:
+.generate-common:
stage: generate
script:
- uname -a || true
@@ -136,41 +143,17 @@ default:
when:
- always
-.generate:
- extends: [ ".base-job", ".generate-base" ]
- tags: ["spack", "public", "medium", "x86_64"]
-
-.darwin-generate-base:
- stage: generate
- script:
- - export SPACK_DISABLE_LOCAL_CONFIG=1
- - export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache
- - uname -a || true
- - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
- - nproc || true
- - . "./share/spack/setup-env.sh"
- - spack --version
- - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
- - spack env activate --without-view .
- - spack -d ci generate --check-index-only
- --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
- --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
- --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
- after_script:
- - cat /proc/loadavg || true
- artifacts:
- paths:
- - "${CI_PROJECT_DIR}/jobs_scratch_dir"
- interruptible: true
- timeout: 60 minutes
- retry:
- max: 2
- when:
- - always
+# Generate without tags for cases using external runners
+.generate-base:
+ extends: [ ".base-job", ".generate-common" ]
-.darwin-generate:
- extends: [ ".base-job", ".darwin-generate-base" ]
+.generate-x86_64:
+ extends: [ ".generate-base" ]
+ tags: ["spack", "public", "medium", "x86_64"]
+.generate-aarch64:
+ extends: [ ".generate-base" ]
+ tags: ["spack", "public", "medium", "aarch64"]
.generate-deprecated:
extends: [ ".base-job" ]
@@ -204,10 +187,6 @@ default:
- always
tags: ["spack", "public", "medium", "x86_64"]
-.generate-aarch64:
- extends: [ ".base-job", ".generate" ]
- tags: ["spack", "public", "medium", "aarch64"]
-
.build:
extends: [ ".base-job" ]
stage: build
@@ -294,7 +273,7 @@ protected-publish:
SPACK_CI_STACK_NAME: e4s
e4s-generate:
- extends: [ ".e4s", ".generate"]
+ extends: [ ".e4s", ".generate-x86_64"]
image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
e4s-build:
@@ -317,7 +296,7 @@ e4s-build:
SPACK_CI_STACK_NAME: gpu-tests
gpu-tests-generate:
- extends: [ ".gpu-tests", ".generate"]
+ extends: [ ".gpu-tests", ".generate-x86_64"]
image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
gpu-tests-build:
@@ -340,7 +319,7 @@ gpu-tests-build:
SPACK_CI_STACK_NAME: e4s-oneapi
e4s-oneapi-generate:
- extends: [ ".e4s-oneapi", ".generate"]
+ extends: [ ".e4s-oneapi", ".generate-x86_64"]
image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023.07.21
e4s-oneapi-build:
@@ -367,7 +346,7 @@ e4s-oneapi-build:
SPACK_CI_STACK_NAME: e4s-power
e4s-power-generate:
- extends: [ ".e4s-power", ".generate", ".e4s-power-generate-tags-and-image"]
+ extends: [ ".e4s-power", ".generate-x86_64", ".e4s-power-generate-tags-and-image"]
e4s-power-build:
extends: [ ".e4s-power", ".build" ]
@@ -389,7 +368,7 @@ e4s-power-build:
SPACK_CI_STACK_NAME: build_systems
build_systems-generate:
- extends: [ ".build_systems", ".generate"]
+ extends: [ ".build_systems", ".generate-x86_64"]
build_systems-build:
extends: [ ".build_systems", ".build" ]
@@ -411,7 +390,7 @@ build_systems-build:
SPACK_CI_STACK_NAME: radiuss
radiuss-generate:
- extends: [ ".radiuss", ".generate" ]
+ extends: [ ".radiuss", ".generate-x86_64" ]
radiuss-build:
extends: [ ".radiuss", ".build" ]
@@ -443,7 +422,7 @@ radiuss-build:
SPACK_CI_STACK_NAME: radiuss-aws
radiuss-aws-generate:
- extends: [ ".radiuss-aws", ".generate", ".radiuss-aws-overrides", ".tags-x86_64_v4" ]
+ extends: [ ".radiuss-aws", ".generate-x86_64", ".radiuss-aws-overrides", ".tags-x86_64_v4" ]
radiuss-aws-build:
extends: [ ".radiuss-aws", ".build" ]
@@ -487,7 +466,7 @@ radiuss-aws-aarch64-build:
SPACK_CI_STACK_NAME: data-vis-sdk
data-vis-sdk-generate:
- extends: [ ".data-vis-sdk", ".generate"]
+ extends: [ ".data-vis-sdk", ".generate-x86_64"]
image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
data-vis-sdk-build:
@@ -517,7 +496,7 @@ data-vis-sdk-build:
SPACK_CI_STACK_NAME: aws-isc
aws-isc-generate:
- extends: [ ".aws-isc", ".generate", ".aws-isc-overrides", ".tags-x86_64_v4" ]
+ extends: [ ".aws-isc", ".generate-x86_64", ".aws-isc-overrides", ".tags-x86_64_v4" ]
aws-isc-build:
extends: [ ".aws-isc", ".build" ]
@@ -561,7 +540,7 @@ aws-isc-aarch64-build:
SPACK_CI_STACK_NAME: tutorial
tutorial-generate:
- extends: [ ".tutorial", ".generate"]
+ extends: [ ".tutorial", ".generate-x86_64"]
image: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-05-07
tutorial-build:
@@ -584,7 +563,7 @@ tutorial-build:
SPACK_CI_STACK_NAME: ml-linux-x86_64-cpu
ml-linux-x86_64-cpu-generate:
- extends: [ ".generate", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ]
+ extends: [ ".generate-x86_64", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ]
image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
ml-linux-x86_64-cpu-build:
@@ -607,7 +586,7 @@ ml-linux-x86_64-cpu-build:
SPACK_CI_STACK_NAME: ml-linux-x86_64-cuda
ml-linux-x86_64-cuda-generate:
- extends: [ ".generate", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ]
+ extends: [ ".generate-x86_64", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ]
image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
ml-linux-x86_64-cuda-build:
@@ -630,7 +609,7 @@ ml-linux-x86_64-cuda-build:
SPACK_CI_STACK_NAME: ml-linux-x86_64-rocm
ml-linux-x86_64-rocm-generate:
- extends: [ ".generate", .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ]
+ extends: [ ".generate-x86_64", .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ]
image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
ml-linux-x86_64-rocm-build:
@@ -648,12 +627,13 @@ ml-linux-x86_64-rocm-build:
# Machine Learning - Darwin aarch64 (MPS)
########################################
.ml-darwin-aarch64-mps:
+ extends: [".darwin_aarch64"]
variables:
SPACK_CI_STACK_NAME: ml-darwin-aarch64-mps
ml-darwin-aarch64-mps-generate:
tags: [ "macos-ventura", "apple-clang-14", "aarch64-macos" ]
- extends: [ ".ml-darwin-aarch64-mps", ".darwin-generate"]
+ extends: [ ".ml-darwin-aarch64-mps", ".generate-base"]
ml-darwin-aarch64-mps-build:
extends: [ ".ml-darwin-aarch64-mps", ".build" ]
@@ -717,7 +697,7 @@ deprecated-ci-build:
SPACK_CI_STACK_NAME: aws-pcluster-icelake
# aws-pcluster-generate-icelake:
-# extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".generate", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
+# extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".generate-x86_64", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
# aws-pcluster-build-icelake:
# extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".build" ]
@@ -736,7 +716,7 @@ deprecated-ci-build:
SPACK_CI_STACK_NAME: aws-pcluster-skylake
# aws-pcluster-generate-skylake:
-# extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".generate", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
+# extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".generate-x86_64", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
# aws-pcluster-build-skylake:
# extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".build" ]
@@ -810,42 +790,12 @@ deprecated-ci-build:
SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
.generate-cray:
- extends: [ ".base-cray-job" ]
+ extends: [ ".generate-common", ".base-cray-job" ]
stage: generate
- script:
+ before_script:
- echo $PATH
- module avail
- module list
- - export SPACK_DISABLE_LOCAL_CONFIG=1
- - export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache
- - uname -a || true
- - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
- - nproc || true
- - . "./share/spack/setup-env.sh"
- - spack --version
- - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
- - spack env activate --without-view .
- - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
- - spack
- --config-scope "${SPACK_CI_CONFIG_ROOT}"
- --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
- --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
- ${CI_STACK_CONFIG_SCOPES}
- ci generate --check-index-only
- --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
- --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
- --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
- after_script:
- - cat /proc/loadavg || true
- artifacts:
- paths:
- - "${CI_PROJECT_DIR}/jobs_scratch_dir"
- interruptible: true
- timeout: 60 minutes
- retry:
- max: 2
- when:
- - always
.generate-cray-rhel:
tags: [ "cray-rhel-zen4", "public" ]
@@ -903,4 +853,4 @@ e4s-cray-sles-build:
strategy: depend
needs:
- artifacts: True
- job: e4s-cray-sles-generate \ No newline at end of file
+ job: e4s-cray-sles-generate
diff --git a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
index 53bc5f63d7..1b3e723d5a 100644
--- a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
@@ -10,19 +10,17 @@ ci:
- build-job:
before_script-:
- - spack list --count # ensure that spack's cache is populated
- script::
- - - spack compiler find
- - cd ${SPACK_CONCRETE_ENV_DIR}
- - spack env activate --without-view .
+ - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
+ - spack compiler find
- if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; fi
- - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
+ - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
# AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
- - k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust $k
+ - - k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust $k
# UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification)
- k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust $k
- k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k
- - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+ script::
+ - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
after_script:
- - cat /proc/loadavg || true
variables:
@@ -47,8 +45,7 @@ ci:
- if [[ $CI_COMMIT_TAG == "develop-"* ]]; then export SPACK_REPLACE_VERSION=develop; fi
- export SPACK_BUILDCACHE_SOURCE=${SPACK_SOURCE_MIRROR//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}}
script:
- - - cd ${SPACK_CONCRETE_ENV_DIR}
- - spack env activate --without-view .
+ - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
- echo Copying environment specs from ${SRC_MIRROR} to ${SPACK_BUILDCACHE_DESTINATION}
- spack buildcache sync "${SPACK_BUILDCACHE_SOURCE}" "${SPACK_BUILDCACHE_DESTINATION}"
- curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub
@@ -89,12 +86,16 @@ ci:
KUBERNETES_MEMORY_REQUEST: "500M"
- any-job:
- image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
tags: ["spack"]
+ image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
+ variables:
+ # Disable local configs to avoid issues on shell runners
+ SPACK_DISABLE_LOCAL_CONFIG: "1"
before_script:
+ - - export SPACK_USER_CACHE_PATH="${CI_PROJECT_DIR}/_user_cache/"
- - uname -a || true
- grep -E "vendor|model name" /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
- - nproc
+ - nproc || true
- - . "./share/spack/setup-env.sh"
- spack --version
- spack arch
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml
index 3df3e1cc7d..a74ef3f98f 100644
--- a/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml
@@ -5,287 +5,3 @@ ci:
- build-job:
variables:
CI_GPG_KEY_ROOT: /etc/protected-runner
- - match_behavior: first
- submapping:
- - match:
- - hipblas
- - llvm
- - llvm-amdgpu
- - pango
- - paraview
- - py-tensorflow
- - py-torch
- - qt
- - rocblas
- - visit
- build-job:
- tags: [ "spack", "huge" ]
- variables:
- CI_JOB_SIZE: huge
- SPACK_BUILD_JOBS: "12"
-
- - match:
- - ascent
- - atk
- - axom
- - cistem
- - cmake
- - ctffind
- - cuda
- - dealii
- - dray
- - dyninst
- - ecp-data-vis-sdk
- - gcc
- - ginkgo
- - hdf5
- - hpx
- - kokkos-kernels
- - kokkos-nvcc-wrapper
- - lbann
- - magma
- - mesa
- - mfem
- - mpich
- - netlib-lapack
- - nvhpc
- - oce
- - openblas
- - openfoam
- - openturns
- - parallelio
- - plumed
- - precice
- #- py-tensorflow
- #- qt
- - raja
- - relion
- #- rocblas
- - rocfft
- - rocsolver
- - rocsparse
- - rust
- - slate
- - strumpack
- - sundials
- - trilinos
- - umpire
- #- visit
- - vtk
- - vtk-h
- - vtk-m
- - warpx
- - wrf
- - wxwidgets
- build-job:
- tags: [ "spack", "large" ]
- variables:
- CI_JOB_SIZE: large
- SPACK_BUILD_JOBS: "8"
-
- - match:
- - adios2
- - amrex
- - archer
- - ascent
- - autoconf-archive
- - axom
- - binutils
- - blaspp
- - blt
- - boost
- - butterflypack
- - cabana
- - caliper
- - camp
- - chai
- - conduit
- - curl
- - datatransferkit
- - double-conversion
- - dray
- - eigen
- - faodel
- - ffmpeg
- - fftw
- - fortrilinos
- - gettext
- - gperftools
- - gptune
- - hdf5
- - heffte
- - hpctoolkit
- - hwloc
- - hydrogen
- - hypre
- - kokkos
- - lammps
- - lapackpp
- - legion
- - libtool
- - libxml2
- - libzmq
- - llvm-openmp-ompt
- - mbedtls
- - mfem
- - mpich
- - mvapich2
- - nasm
- - netlib-scalapack
- - omega-h
- - openblas
- - openjpeg
- - openmpi
- - openpmd-api
- - pagmo2
- - papyrus
- - parsec
- - pdt
- - pegtl
- - petsc
- - pumi
- - py-beniget
- - py-cinemasci
- - pygmo
- - py-ipython-genutils
- - py-packaging
- - py-petsc4py
- - py-scipy
- - py-statsmodels
- - py-warlock
- - py-warpx
- - raja
- - samrai
- - slepc
- - slurm
- - sqlite
- - strumpack
- - sundials
- - superlu-dist
- - tasmanian
- - tau
- - upcxx
- - vtk
- - vtk-h
- - vtk-m
- - zfp
- build-job:
- tags: [ "spack", "medium" ]
- variables:
- CI_JOB_SIZE: "medium"
- SPACK_BUILD_JOBS: "2"
-
- - match:
- - alsa-lib
- - ant
- - antlr
- - argobots
- - autoconf-archive
- - automake
- - berkeley-db
- - bison
- - blt
- - bzip2
- - camp
- - cmake
- - curl
- - czmq
- - darshan-util
- - diffutils
- - docbook-xml
- - exmcutils
- - expat
- - findutils
- - flit
- - freetype
- - gawk
- - gdbm
- - gettext
- - glib
- - gmake
- - gotcha
- - hpcviewer
- - hwloc
- - jansson
- - json-c
- - libbsd
- - libedit
- - libevent
- - libfabric
- - libffi
- - libgcrypt
- - libiconv
- - libidn2
- - libjpeg-turbo
- - libmd
- - libnrm
- - libpciaccess
- - libpng
- - libsigsegv
- - libsodium
- - libunistring
- - libunwind
- - libxml2
- - libyaml
- - libzmq
- - lua
- - lua-luaposix
- - lz4
- - m4
- - meson
- - metis
- - mpfr
- - ncurses
- - ninja
- - numactl
- - openblas
- - openjdk
- - openssh
- - openssl
- - papi
- - parallel-netcdf
- - pcre
- - pcre2
- - pdsh
- - perl
- - perl-data-dumper
- - pkgconf
- - py-alembic
- - py-cffi
- - py-cycler
- - py-decorator
- - py-idna
- - py-jsonschema
- - py-kiwisolver
- - py-mistune
- - py-pycparser
- - py-setuptools
- - py-setuptools-scm
- - py-six
- - py-testpath
- - py-wheel
- - qhull
- - readline
- - sed
- - slurm
- - snappy
- - sqlite
- - superlu
- - swig
- - tar
- - tcl
- - texinfo
- - tut
- - unzip
- - util-linux-uuid
- - util-macros
- - xz
- - yaml-cpp
- - zfp
- - zlib
- - zstd
- build-job:
- tags: [ "spack", "small" ]
- variables:
- CI_JOB_SIZE: "small"
- SPACK_BUILD_JOBS: "1"
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml
index 3df3e1cc7d..a74ef3f98f 100644
--- a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml
@@ -5,287 +5,3 @@ ci:
- build-job:
variables:
CI_GPG_KEY_ROOT: /etc/protected-runner
- - match_behavior: first
- submapping:
- - match:
- - hipblas
- - llvm
- - llvm-amdgpu
- - pango
- - paraview
- - py-tensorflow
- - py-torch
- - qt
- - rocblas
- - visit
- build-job:
- tags: [ "spack", "huge" ]
- variables:
- CI_JOB_SIZE: huge
- SPACK_BUILD_JOBS: "12"
-
- - match:
- - ascent
- - atk
- - axom
- - cistem
- - cmake
- - ctffind
- - cuda
- - dealii
- - dray
- - dyninst
- - ecp-data-vis-sdk
- - gcc
- - ginkgo
- - hdf5
- - hpx
- - kokkos-kernels
- - kokkos-nvcc-wrapper
- - lbann
- - magma
- - mesa
- - mfem
- - mpich
- - netlib-lapack
- - nvhpc
- - oce
- - openblas
- - openfoam
- - openturns
- - parallelio
- - plumed
- - precice
- #- py-tensorflow
- #- qt
- - raja
- - relion
- #- rocblas
- - rocfft
- - rocsolver
- - rocsparse
- - rust
- - slate
- - strumpack
- - sundials
- - trilinos
- - umpire
- #- visit
- - vtk
- - vtk-h
- - vtk-m
- - warpx
- - wrf
- - wxwidgets
- build-job:
- tags: [ "spack", "large" ]
- variables:
- CI_JOB_SIZE: large
- SPACK_BUILD_JOBS: "8"
-
- - match:
- - adios2
- - amrex
- - archer
- - ascent
- - autoconf-archive
- - axom
- - binutils
- - blaspp
- - blt
- - boost
- - butterflypack
- - cabana
- - caliper
- - camp
- - chai
- - conduit
- - curl
- - datatransferkit
- - double-conversion
- - dray
- - eigen
- - faodel
- - ffmpeg
- - fftw
- - fortrilinos
- - gettext
- - gperftools
- - gptune
- - hdf5
- - heffte
- - hpctoolkit
- - hwloc
- - hydrogen
- - hypre
- - kokkos
- - lammps
- - lapackpp
- - legion
- - libtool
- - libxml2
- - libzmq
- - llvm-openmp-ompt
- - mbedtls
- - mfem
- - mpich
- - mvapich2
- - nasm
- - netlib-scalapack
- - omega-h
- - openblas
- - openjpeg
- - openmpi
- - openpmd-api
- - pagmo2
- - papyrus
- - parsec
- - pdt
- - pegtl
- - petsc
- - pumi
- - py-beniget
- - py-cinemasci
- - pygmo
- - py-ipython-genutils
- - py-packaging
- - py-petsc4py
- - py-scipy
- - py-statsmodels
- - py-warlock
- - py-warpx
- - raja
- - samrai
- - slepc
- - slurm
- - sqlite
- - strumpack
- - sundials
- - superlu-dist
- - tasmanian
- - tau
- - upcxx
- - vtk
- - vtk-h
- - vtk-m
- - zfp
- build-job:
- tags: [ "spack", "medium" ]
- variables:
- CI_JOB_SIZE: "medium"
- SPACK_BUILD_JOBS: "2"
-
- - match:
- - alsa-lib
- - ant
- - antlr
- - argobots
- - autoconf-archive
- - automake
- - berkeley-db
- - bison
- - blt
- - bzip2
- - camp
- - cmake
- - curl
- - czmq
- - darshan-util
- - diffutils
- - docbook-xml
- - exmcutils
- - expat
- - findutils
- - flit
- - freetype
- - gawk
- - gdbm
- - gettext
- - glib
- - gmake
- - gotcha
- - hpcviewer
- - hwloc
- - jansson
- - json-c
- - libbsd
- - libedit
- - libevent
- - libfabric
- - libffi
- - libgcrypt
- - libiconv
- - libidn2
- - libjpeg-turbo
- - libmd
- - libnrm
- - libpciaccess
- - libpng
- - libsigsegv
- - libsodium
- - libunistring
- - libunwind
- - libxml2
- - libyaml
- - libzmq
- - lua
- - lua-luaposix
- - lz4
- - m4
- - meson
- - metis
- - mpfr
- - ncurses
- - ninja
- - numactl
- - openblas
- - openjdk
- - openssh
- - openssl
- - papi
- - parallel-netcdf
- - pcre
- - pcre2
- - pdsh
- - perl
- - perl-data-dumper
- - pkgconf
- - py-alembic
- - py-cffi
- - py-cycler
- - py-decorator
- - py-idna
- - py-jsonschema
- - py-kiwisolver
- - py-mistune
- - py-pycparser
- - py-setuptools
- - py-setuptools-scm
- - py-six
- - py-testpath
- - py-wheel
- - qhull
- - readline
- - sed
- - slurm
- - snappy
- - sqlite
- - superlu
- - swig
- - tar
- - tcl
- - texinfo
- - tut
- - unzip
- - util-linux-uuid
- - util-macros
- - xz
- - yaml-cpp
- - zfp
- - zlib
- - zstd
- build-job:
- tags: [ "spack", "small" ]
- variables:
- CI_JOB_SIZE: "small"
- SPACK_BUILD_JOBS: "1"
diff --git a/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml
new file mode 100644
index 0000000000..8dfb169a3e
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml
@@ -0,0 +1,4 @@
+ci:
+ pipeline-gen:
+ - build-job-remove:
+ image: macos-run-on-metal
diff --git a/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
index b774879329..8634042101 100644
--- a/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
@@ -1,5 +1,10 @@
ci:
pipeline-gen:
+ - build-job:
+ before_script-:
+ # Test package relocation on linux using a modified prefix
+ # This is not well supported on MacOS (https://github.com/spack/spack/issues/37162)
+ - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- match_behavior: first
submapping:
- match:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
index 2407c17bde..aa756a6e6a 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
@@ -1,4 +1,7 @@
spack:
+ include:
+ - $spack/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
+
view: false
concretizer:
@@ -62,4 +65,4 @@ spack:
mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray-rhel" }
cdash:
- build-group: E4S Cray \ No newline at end of file
+ build-group: E4S Cray
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
index 9fa4625f08..0c0b3a0179 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
@@ -1,4 +1,7 @@
spack:
+ include:
+ - $spack/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
+
view: false
concretizer:
@@ -45,4 +48,4 @@ spack:
mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray-sles" }
cdash:
- build-group: E4S Cray SLES \ No newline at end of file
+ build-group: E4S Cray SLES
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
index 90fc297a0d..7f22e232d1 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
@@ -1,19 +1,6 @@
spack:
view: false
- concretizer:
- unify: false
- reuse: false
-
- config:
- concretizer: clingo
- db_lock_timeout: 120
- install_tree:
- root: $spack/opt/spack
- padded_length: 256
- projections:
- all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
-
packages:
all:
require: target=aarch64
@@ -97,42 +84,11 @@ spack:
ci:
pipeline-gen:
- build-job-remove:
- image: no-image
- tags: [spack, public]
+ tags: [ spack, public ]
- build-job:
+ variables:
+ CI_GPG_KEY_ROOT: /etc/protected-runner
tags: [ "macos-ventura", "apple-clang-14", "aarch64-macos" ]
- script::
- - - spack compiler find
- - cd ${SPACK_CONCRETE_ENV_DIR}
- - spack env activate --without-view .
- - if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; fi
- - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
- # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
- - if [[ -r /etc/protected-runner/e4s.gpg ]]; then spack gpg trust /etc/protected-runner/e4s.gpg; fi
- # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification)
- - if [[ -r /etc/protected-runner/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /etc/protected-runner/intermediate_ci_signing_key.gpg; fi
- - if [[ -r /etc/protected-runner/spack_public_key.gpg ]]; then spack gpg trust /etc/protected-runner/spack_public_key.gpg; fi
- - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
- after_script:
- - - cat /proc/loadavg || true
- - signing-job:
- image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
- tags: ["aws"]
- script:
- - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
- - /sign.sh
- - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
- - aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
- - any-job:
- image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
- tags: ["spack"]
- before_script:
- - - uname -a || true
- - grep -E "vendor|model name" /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
- - nproc || true
- - - . "./share/spack/setup-env.sh"
- - spack --version
- - spack arch
cdash:
build-group: Machine Learning MPS