From b54d286b4a341667f3620f43f7c13a5ba777fb63 Mon Sep 17 00:00:00 2001 From: kwryankrattiger <80296582+kwryankrattiger@users.noreply.github.com> Date: Mon, 7 Aug 2023 11:23:16 -0500 Subject: CI: remove redundant sections (#38514) Refactor gitlab ci configs so that mac and cray jobs can reuse as much higher level configuration as possible. * CI: remove redundant sections * CI: Include base linux CI configs in cray stacks Relocation and runner mapping is consistent between cray and linux runners. * Export user cache path in before script * CI: add GPG root for mac runners * Disable user configs Metal runners share a ~ directory * Disable user config and add configs in activate env --- share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml | 124 +++------ share/spack/gitlab/cloud_pipelines/configs/ci.yaml | 25 +- .../cloud_pipelines/configs/cray-rhel/ci.yaml | 284 --------------------- .../cloud_pipelines/configs/cray-sles/ci.yaml | 284 --------------------- .../gitlab/cloud_pipelines/configs/darwin/ci.yaml | 4 + .../gitlab/cloud_pipelines/configs/linux/ci.yaml | 5 + .../stacks/e4s-cray-rhel/spack.yaml | 5 +- .../stacks/e4s-cray-sles/spack.yaml | 5 +- .../stacks/ml-darwin-aarch64-mps/spack.yaml | 50 +--- 9 files changed, 70 insertions(+), 716 deletions(-) create mode 100644 share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml index 4b927a04ae..358d3af121 100644 --- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml +++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml @@ -19,6 +19,11 @@ default: SPACK_TARGET_PLATFORM: "darwin" SPACK_TARGET_ARCH: "x86_64" +.darwin_aarch64: + variables: + SPACK_TARGET_PLATFORM: "darwin" + SPACK_TARGET_ARCH: "aarch64" + .linux_x86_64_v3: variables: SPACK_TARGET_PLATFORM: "linux" @@ -60,6 +65,8 @@ default: .base-job: variables: SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}" + SPACK_DISABLE_LOCAL_CONFIG: "1" + SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/tmp/_user_cache/" rules: - if: $CI_COMMIT_REF_NAME == "develop" # Pipelines on develop only rebuild what is missing from the mirror @@ -101,7 +108,7 @@ default: AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY} -.generate-base: +.generate-common: stage: generate script: - uname -a || true @@ -136,41 +143,17 @@ default: when: - always -.generate: - extends: [ ".base-job", ".generate-base" ] - tags: ["spack", "public", "medium", "x86_64"] - -.darwin-generate-base: - stage: generate - script: - - export SPACK_DISABLE_LOCAL_CONFIG=1 - - export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc || true - - . "./share/spack/setup-env.sh" - - spack --version - - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME} - - spack env activate --without-view . - - spack -d ci generate --check-index-only - --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}" - --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir" - --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml" - after_script: - - cat /proc/loadavg || true - artifacts: - paths: - - "${CI_PROJECT_DIR}/jobs_scratch_dir" - interruptible: true - timeout: 60 minutes - retry: - max: 2 - when: - - always +# Generate without tags for cases using external runners +.generate-base: + extends: [ ".base-job", ".generate-common" ] -.darwin-generate: - extends: [ ".base-job", ".darwin-generate-base" ] +.generate-x86_64: + extends: [ ".generate-base" ] + tags: ["spack", "public", "medium", "x86_64"] +.generate-aarch64: + extends: [ ".generate-base" ] + tags: ["spack", "public", "medium", "aarch64"] .generate-deprecated: extends: [ ".base-job" ] @@ -204,10 +187,6 @@ default: - always tags: ["spack", "public", "medium", "x86_64"] -.generate-aarch64: - extends: [ ".base-job", ".generate" ] - tags: ["spack", "public", "medium", "aarch64"] - .build: extends: [ ".base-job" ] stage: build @@ -294,7 +273,7 @@ protected-publish: SPACK_CI_STACK_NAME: e4s e4s-generate: - extends: [ ".e4s", ".generate"] + extends: [ ".e4s", ".generate-x86_64"] image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 e4s-build: @@ -317,7 +296,7 @@ e4s-build: SPACK_CI_STACK_NAME: gpu-tests gpu-tests-generate: - extends: [ ".gpu-tests", ".generate"] + extends: [ ".gpu-tests", ".generate-x86_64"] image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 gpu-tests-build: @@ -340,7 +319,7 @@ gpu-tests-build: SPACK_CI_STACK_NAME: e4s-oneapi e4s-oneapi-generate: - extends: [ ".e4s-oneapi", ".generate"] + extends: [ ".e4s-oneapi", ".generate-x86_64"] image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023.07.21 e4s-oneapi-build: @@ -367,7 +346,7 @@ e4s-oneapi-build: SPACK_CI_STACK_NAME: e4s-power e4s-power-generate: - extends: [ ".e4s-power", ".generate", ".e4s-power-generate-tags-and-image"] + extends: [ ".e4s-power", ".generate-x86_64", ".e4s-power-generate-tags-and-image"] e4s-power-build: extends: [ ".e4s-power", ".build" ] @@ -389,7 +368,7 @@ e4s-power-build: SPACK_CI_STACK_NAME: build_systems build_systems-generate: - extends: [ ".build_systems", ".generate"] + extends: [ ".build_systems", ".generate-x86_64"] build_systems-build: extends: [ ".build_systems", ".build" ] @@ -411,7 +390,7 @@ build_systems-build: SPACK_CI_STACK_NAME: radiuss radiuss-generate: - extends: [ ".radiuss", ".generate" ] + extends: [ ".radiuss", ".generate-x86_64" ] radiuss-build: extends: [ ".radiuss", ".build" ] @@ -443,7 +422,7 @@ radiuss-build: SPACK_CI_STACK_NAME: radiuss-aws radiuss-aws-generate: - extends: [ ".radiuss-aws", ".generate", ".radiuss-aws-overrides", ".tags-x86_64_v4" ] + extends: [ ".radiuss-aws", ".generate-x86_64", ".radiuss-aws-overrides", ".tags-x86_64_v4" ] radiuss-aws-build: extends: [ ".radiuss-aws", ".build" ] @@ -487,7 +466,7 @@ radiuss-aws-aarch64-build: SPACK_CI_STACK_NAME: data-vis-sdk data-vis-sdk-generate: - extends: [ ".data-vis-sdk", ".generate"] + extends: [ ".data-vis-sdk", ".generate-x86_64"] image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 data-vis-sdk-build: @@ -517,7 +496,7 @@ data-vis-sdk-build: SPACK_CI_STACK_NAME: aws-isc aws-isc-generate: - extends: [ ".aws-isc", ".generate", ".aws-isc-overrides", ".tags-x86_64_v4" ] + extends: [ ".aws-isc", ".generate-x86_64", ".aws-isc-overrides", ".tags-x86_64_v4" ] aws-isc-build: extends: [ ".aws-isc", ".build" ] @@ -561,7 +540,7 @@ aws-isc-aarch64-build: SPACK_CI_STACK_NAME: tutorial tutorial-generate: - extends: [ ".tutorial", ".generate"] + extends: [ ".tutorial", ".generate-x86_64"] image: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-05-07 tutorial-build: @@ -584,7 +563,7 @@ tutorial-build: SPACK_CI_STACK_NAME: ml-linux-x86_64-cpu ml-linux-x86_64-cpu-generate: - extends: [ ".generate", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ] + extends: [ ".generate-x86_64", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ] image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly ml-linux-x86_64-cpu-build: @@ -607,7 +586,7 @@ ml-linux-x86_64-cpu-build: SPACK_CI_STACK_NAME: ml-linux-x86_64-cuda ml-linux-x86_64-cuda-generate: - extends: [ ".generate", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ] + extends: [ ".generate-x86_64", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ] image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly ml-linux-x86_64-cuda-build: @@ -630,7 +609,7 @@ ml-linux-x86_64-cuda-build: SPACK_CI_STACK_NAME: ml-linux-x86_64-rocm ml-linux-x86_64-rocm-generate: - extends: [ ".generate", .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ] + extends: [ ".generate-x86_64", .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ] image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly ml-linux-x86_64-rocm-build: @@ -648,12 +627,13 @@ ml-linux-x86_64-rocm-build: # Machine Learning - Darwin aarch64 (MPS) ######################################## .ml-darwin-aarch64-mps: + extends: [".darwin_aarch64"] variables: SPACK_CI_STACK_NAME: ml-darwin-aarch64-mps ml-darwin-aarch64-mps-generate: tags: [ "macos-ventura", "apple-clang-14", "aarch64-macos" ] - extends: [ ".ml-darwin-aarch64-mps", ".darwin-generate"] + extends: [ ".ml-darwin-aarch64-mps", ".generate-base"] ml-darwin-aarch64-mps-build: extends: [ ".ml-darwin-aarch64-mps", ".build" ] @@ -717,7 +697,7 @@ deprecated-ci-build: SPACK_CI_STACK_NAME: aws-pcluster-icelake # aws-pcluster-generate-icelake: -# extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".generate", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ] +# extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".generate-x86_64", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ] # aws-pcluster-build-icelake: # extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".build" ] @@ -736,7 +716,7 @@ deprecated-ci-build: SPACK_CI_STACK_NAME: aws-pcluster-skylake # aws-pcluster-generate-skylake: -# extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".generate", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ] +# extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".generate-x86_64", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ] # aws-pcluster-build-skylake: # extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".build" ] @@ -810,42 +790,12 @@ deprecated-ci-build: SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1" .generate-cray: - extends: [ ".base-cray-job" ] + extends: [ ".generate-common", ".base-cray-job" ] stage: generate - script: + before_script: - echo $PATH - module avail - module list - - export SPACK_DISABLE_LOCAL_CONFIG=1 - - export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc || true - - . "./share/spack/setup-env.sh" - - spack --version - - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME} - - spack env activate --without-view . - - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs" - - spack - --config-scope "${SPACK_CI_CONFIG_ROOT}" - --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}" - --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}" - ${CI_STACK_CONFIG_SCOPES} - ci generate --check-index-only - --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}" - --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir" - --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml" - after_script: - - cat /proc/loadavg || true - artifacts: - paths: - - "${CI_PROJECT_DIR}/jobs_scratch_dir" - interruptible: true - timeout: 60 minutes - retry: - max: 2 - when: - - always .generate-cray-rhel: tags: [ "cray-rhel-zen4", "public" ] @@ -903,4 +853,4 @@ e4s-cray-sles-build: strategy: depend needs: - artifacts: True - job: e4s-cray-sles-generate \ No newline at end of file + job: e4s-cray-sles-generate diff --git a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml index 53bc5f63d7..1b3e723d5a 100644 --- a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml +++ b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml @@ -10,19 +10,17 @@ ci: - build-job: before_script-: - - spack list --count # ensure that spack's cache is populated - script:: - - - spack compiler find - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . + - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR} + - spack compiler find - if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; fi - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data + - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust $k + - - k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust $k # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust $k - k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k - - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + script:: + - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) after_script: - - cat /proc/loadavg || true variables: @@ -47,8 +45,7 @@ ci: - if [[ $CI_COMMIT_TAG == "develop-"* ]]; then export SPACK_REPLACE_VERSION=develop; fi - export SPACK_BUILDCACHE_SOURCE=${SPACK_SOURCE_MIRROR//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}} script: - - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . + - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR} - echo Copying environment specs from ${SRC_MIRROR} to ${SPACK_BUILDCACHE_DESTINATION} - spack buildcache sync "${SPACK_BUILDCACHE_SOURCE}" "${SPACK_BUILDCACHE_DESTINATION}" - curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub @@ -89,12 +86,16 @@ ci: KUBERNETES_MEMORY_REQUEST: "500M" - any-job: - image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18" tags: ["spack"] + image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18" + variables: + # Disable local configs to avoid issues on shell runners + SPACK_DISABLE_LOCAL_CONFIG: "1" before_script: + - - export SPACK_USER_CACHE_PATH="${CI_PROJECT_DIR}/_user_cache/" - - uname -a || true - grep -E "vendor|model name" /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc + - nproc || true - - . "./share/spack/setup-env.sh" - spack --version - spack arch diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml index 3df3e1cc7d..a74ef3f98f 100644 --- a/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml +++ b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml @@ -5,287 +5,3 @@ ci: - build-job: variables: CI_GPG_KEY_ROOT: /etc/protected-runner - - match_behavior: first - submapping: - - match: - - hipblas - - llvm - - llvm-amdgpu - - pango - - paraview - - py-tensorflow - - py-torch - - qt - - rocblas - - visit - build-job: - tags: [ "spack", "huge" ] - variables: - CI_JOB_SIZE: huge - SPACK_BUILD_JOBS: "12" - - - match: - - ascent - - atk - - axom - - cistem - - cmake - - ctffind - - cuda - - dealii - - dray - - dyninst - - ecp-data-vis-sdk - - gcc - - ginkgo - - hdf5 - - hpx - - kokkos-kernels - - kokkos-nvcc-wrapper - - lbann - - magma - - mesa - - mfem - - mpich - - netlib-lapack - - nvhpc - - oce - - openblas - - openfoam - - openturns - - parallelio - - plumed - - precice - #- py-tensorflow - #- qt - - raja - - relion - #- rocblas - - rocfft - - rocsolver - - rocsparse - - rust - - slate - - strumpack - - sundials - - trilinos - - umpire - #- visit - - vtk - - vtk-h - - vtk-m - - warpx - - wrf - - wxwidgets - build-job: - tags: [ "spack", "large" ] - variables: - CI_JOB_SIZE: large - SPACK_BUILD_JOBS: "8" - - - match: - - adios2 - - amrex - - archer - - ascent - - autoconf-archive - - axom - - binutils - - blaspp - - blt - - boost - - butterflypack - - cabana - - caliper - - camp - - chai - - conduit - - curl - - datatransferkit - - double-conversion - - dray - - eigen - - faodel - - ffmpeg - - fftw - - fortrilinos - - gettext - - gperftools - - gptune - - hdf5 - - heffte - - hpctoolkit - - hwloc - - hydrogen - - hypre - - kokkos - - lammps - - lapackpp - - legion - - libtool - - libxml2 - - libzmq - - llvm-openmp-ompt - - mbedtls - - mfem - - mpich - - mvapich2 - - nasm - - netlib-scalapack - - omega-h - - openblas - - openjpeg - - openmpi - - openpmd-api - - pagmo2 - - papyrus - - parsec - - pdt - - pegtl - - petsc - - pumi - - py-beniget - - py-cinemasci - - pygmo - - py-ipython-genutils - - py-packaging - - py-petsc4py - - py-scipy - - py-statsmodels - - py-warlock - - py-warpx - - raja - - samrai - - slepc - - slurm - - sqlite - - strumpack - - sundials - - superlu-dist - - tasmanian - - tau - - upcxx - - vtk - - vtk-h - - vtk-m - - zfp - build-job: - tags: [ "spack", "medium" ] - variables: - CI_JOB_SIZE: "medium" - SPACK_BUILD_JOBS: "2" - - - match: - - alsa-lib - - ant - - antlr - - argobots - - autoconf-archive - - automake - - berkeley-db - - bison - - blt - - bzip2 - - camp - - cmake - - curl - - czmq - - darshan-util - - diffutils - - docbook-xml - - exmcutils - - expat - - findutils - - flit - - freetype - - gawk - - gdbm - - gettext - - glib - - gmake - - gotcha - - hpcviewer - - hwloc - - jansson - - json-c - - libbsd - - libedit - - libevent - - libfabric - - libffi - - libgcrypt - - libiconv - - libidn2 - - libjpeg-turbo - - libmd - - libnrm - - libpciaccess - - libpng - - libsigsegv - - libsodium - - libunistring - - libunwind - - libxml2 - - libyaml - - libzmq - - lua - - lua-luaposix - - lz4 - - m4 - - meson - - metis - - mpfr - - ncurses - - ninja - - numactl - - openblas - - openjdk - - openssh - - openssl - - papi - - parallel-netcdf - - pcre - - pcre2 - - pdsh - - perl - - perl-data-dumper - - pkgconf - - py-alembic - - py-cffi - - py-cycler - - py-decorator - - py-idna - - py-jsonschema - - py-kiwisolver - - py-mistune - - py-pycparser - - py-setuptools - - py-setuptools-scm - - py-six - - py-testpath - - py-wheel - - qhull - - readline - - sed - - slurm - - snappy - - sqlite - - superlu - - swig - - tar - - tcl - - texinfo - - tut - - unzip - - util-linux-uuid - - util-macros - - xz - - yaml-cpp - - zfp - - zlib - - zstd - build-job: - tags: [ "spack", "small" ] - variables: - CI_JOB_SIZE: "small" - SPACK_BUILD_JOBS: "1" diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml index 3df3e1cc7d..a74ef3f98f 100644 --- a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml +++ b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml @@ -5,287 +5,3 @@ ci: - build-job: variables: CI_GPG_KEY_ROOT: /etc/protected-runner - - match_behavior: first - submapping: - - match: - - hipblas - - llvm - - llvm-amdgpu - - pango - - paraview - - py-tensorflow - - py-torch - - qt - - rocblas - - visit - build-job: - tags: [ "spack", "huge" ] - variables: - CI_JOB_SIZE: huge - SPACK_BUILD_JOBS: "12" - - - match: - - ascent - - atk - - axom - - cistem - - cmake - - ctffind - - cuda - - dealii - - dray - - dyninst - - ecp-data-vis-sdk - - gcc - - ginkgo - - hdf5 - - hpx - - kokkos-kernels - - kokkos-nvcc-wrapper - - lbann - - magma - - mesa - - mfem - - mpich - - netlib-lapack - - nvhpc - - oce - - openblas - - openfoam - - openturns - - parallelio - - plumed - - precice - #- py-tensorflow - #- qt - - raja - - relion - #- rocblas - - rocfft - - rocsolver - - rocsparse - - rust - - slate - - strumpack - - sundials - - trilinos - - umpire - #- visit - - vtk - - vtk-h - - vtk-m - - warpx - - wrf - - wxwidgets - build-job: - tags: [ "spack", "large" ] - variables: - CI_JOB_SIZE: large - SPACK_BUILD_JOBS: "8" - - - match: - - adios2 - - amrex - - archer - - ascent - - autoconf-archive - - axom - - binutils - - blaspp - - blt - - boost - - butterflypack - - cabana - - caliper - - camp - - chai - - conduit - - curl - - datatransferkit - - double-conversion - - dray - - eigen - - faodel - - ffmpeg - - fftw - - fortrilinos - - gettext - - gperftools - - gptune - - hdf5 - - heffte - - hpctoolkit - - hwloc - - hydrogen - - hypre - - kokkos - - lammps - - lapackpp - - legion - - libtool - - libxml2 - - libzmq - - llvm-openmp-ompt - - mbedtls - - mfem - - mpich - - mvapich2 - - nasm - - netlib-scalapack - - omega-h - - openblas - - openjpeg - - openmpi - - openpmd-api - - pagmo2 - - papyrus - - parsec - - pdt - - pegtl - - petsc - - pumi - - py-beniget - - py-cinemasci - - pygmo - - py-ipython-genutils - - py-packaging - - py-petsc4py - - py-scipy - - py-statsmodels - - py-warlock - - py-warpx - - raja - - samrai - - slepc - - slurm - - sqlite - - strumpack - - sundials - - superlu-dist - - tasmanian - - tau - - upcxx - - vtk - - vtk-h - - vtk-m - - zfp - build-job: - tags: [ "spack", "medium" ] - variables: - CI_JOB_SIZE: "medium" - SPACK_BUILD_JOBS: "2" - - - match: - - alsa-lib - - ant - - antlr - - argobots - - autoconf-archive - - automake - - berkeley-db - - bison - - blt - - bzip2 - - camp - - cmake - - curl - - czmq - - darshan-util - - diffutils - - docbook-xml - - exmcutils - - expat - - findutils - - flit - - freetype - - gawk - - gdbm - - gettext - - glib - - gmake - - gotcha - - hpcviewer - - hwloc - - jansson - - json-c - - libbsd - - libedit - - libevent - - libfabric - - libffi - - libgcrypt - - libiconv - - libidn2 - - libjpeg-turbo - - libmd - - libnrm - - libpciaccess - - libpng - - libsigsegv - - libsodium - - libunistring - - libunwind - - libxml2 - - libyaml - - libzmq - - lua - - lua-luaposix - - lz4 - - m4 - - meson - - metis - - mpfr - - ncurses - - ninja - - numactl - - openblas - - openjdk - - openssh - - openssl - - papi - - parallel-netcdf - - pcre - - pcre2 - - pdsh - - perl - - perl-data-dumper - - pkgconf - - py-alembic - - py-cffi - - py-cycler - - py-decorator - - py-idna - - py-jsonschema - - py-kiwisolver - - py-mistune - - py-pycparser - - py-setuptools - - py-setuptools-scm - - py-six - - py-testpath - - py-wheel - - qhull - - readline - - sed - - slurm - - snappy - - sqlite - - superlu - - swig - - tar - - tcl - - texinfo - - tut - - unzip - - util-linux-uuid - - util-macros - - xz - - yaml-cpp - - zfp - - zlib - - zstd - build-job: - tags: [ "spack", "small" ] - variables: - CI_JOB_SIZE: "small" - SPACK_BUILD_JOBS: "1" diff --git a/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml new file mode 100644 index 0000000000..8dfb169a3e --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml @@ -0,0 +1,4 @@ +ci: + pipeline-gen: + - build-job-remove: + image: macos-run-on-metal diff --git a/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml index b774879329..8634042101 100644 --- a/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml +++ b/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml @@ -1,5 +1,10 @@ ci: pipeline-gen: + - build-job: + before_script-: + # Test package relocation on linux using a modified prefix + # This is not well supported on MacOS (https://github.com/spack/spack/issues/37162) + - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - match_behavior: first submapping: - match: diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml index 2407c17bde..aa756a6e6a 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml @@ -1,4 +1,7 @@ spack: + include: + - $spack/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml + view: false concretizer: @@ -62,4 +65,4 @@ spack: mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray-rhel" } cdash: - build-group: E4S Cray \ No newline at end of file + build-group: E4S Cray diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml index 9fa4625f08..0c0b3a0179 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml @@ -1,4 +1,7 @@ spack: + include: + - $spack/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml + view: false concretizer: @@ -45,4 +48,4 @@ spack: mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray-sles" } cdash: - build-group: E4S Cray SLES \ No newline at end of file + build-group: E4S Cray SLES diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml index 90fc297a0d..7f22e232d1 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml @@ -1,19 +1,6 @@ spack: view: false - concretizer: - unify: false - reuse: false - - config: - concretizer: clingo - db_lock_timeout: 120 - install_tree: - root: $spack/opt/spack - padded_length: 256 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - packages: all: require: target=aarch64 @@ -97,42 +84,11 @@ spack: ci: pipeline-gen: - build-job-remove: - image: no-image - tags: [spack, public] + tags: [ spack, public ] - build-job: + variables: + CI_GPG_KEY_ROOT: /etc/protected-runner tags: [ "macos-ventura", "apple-clang-14", "aarch64-macos" ] - script:: - - - spack compiler find - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; fi - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /etc/protected-runner/e4s.gpg ]]; then spack gpg trust /etc/protected-runner/e4s.gpg; fi - # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /etc/protected-runner/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /etc/protected-runner/intermediate_ci_signing_key.gpg; fi - - if [[ -r /etc/protected-runner/spack_public_key.gpg ]]; then spack gpg trust /etc/protected-runner/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - after_script: - - - cat /proc/loadavg || true - - signing-job: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["aws"] - script: - - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache - - aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp --recursive --exclude "*" --include "*.pub" - - any-job: - image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18" - tags: ["spack"] - before_script: - - - uname -a || true - - grep -E "vendor|model name" /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc || true - - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch cdash: build-group: Machine Learning MPS -- cgit v1.2.3-60-g2f50