From f3595da600cb79c16bf706320cbc6047239d4605 Mon Sep 17 00:00:00 2001 From: kwryankrattiger <80296582+kwryankrattiger@users.noreply.github.com> Date: Fri, 10 Mar 2023 13:25:35 -0600 Subject: CI boilerplate reduction (#34272) * CI configuration boilerplate reduction and refactor Configuration: - New notation for list concatenation (prepend/append) - New notation for string concatenation (prepend/append) - Break out configuration files for: ci.yaml, cdash.yaml, view.yaml - Spack CI section refactored to improve self-consistency and composability - Scripts are now lists of lists and/or lists of strings - Job attributes are now listed under precedence ordered list that are composed/merged using Spack config merge rules. - "service-jobs" are identified explicitly rather than as a batch CI: - Consolidate common, platform, and architecture configurations for all CI stacks into composable configuration files - Make padding consistent across all stacks (256) - Merge all package -> runner mappings to be consistent across all stacks Unit Test: - Refactor CI module unit-tests for refactor configuration Docs: - Add docs for new notations in configuration.rst - Rewrite docs on CI pipelines to be consistent with refactored CI workflow * Script verbose environ, dev bootstrap * Port #35409 --- share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml | 51 +++- .../gitlab/cloud_pipelines/configs/cdash.yaml | 6 + share/spack/gitlab/cloud_pipelines/configs/ci.yaml | 47 ++++ .../cloud_pipelines/configs/concretizer.yaml | 4 + .../gitlab/cloud_pipelines/configs/config.yaml | 10 + .../gitlab/cloud_pipelines/configs/darwin/ci.yaml | 14 + .../cloud_pipelines/configs/linux/aarch64/ci.yaml | 8 + .../gitlab/cloud_pipelines/configs/linux/ci.yaml | 290 +++++++++++++++++++++ .../cloud_pipelines/configs/linux/ppc64le/ci.yaml | 21 ++ .../cloud_pipelines/configs/linux/x86_64/ci.yaml | 8 + .../configs/linux/x86_64_v4/ci.yaml | 8 + .../gitlab/cloud_pipelines/configs/packages.yaml | 23 ++ .../stacks/aws-ahug-aarch64/spack.yaml | 116 +-------- .../cloud_pipelines/stacks/aws-ahug/spack.yaml | 120 +-------- .../stacks/aws-isc-aarch64/spack.yaml | 125 +-------- .../cloud_pipelines/stacks/aws-isc/spack.yaml | 130 +-------- .../stacks/build_systems/spack.yaml | 115 -------- .../cloud_pipelines/stacks/data-vis-sdk/spack.yaml | 155 +---------- .../cloud_pipelines/stacks/e4s-mac/spack.yaml | 55 +--- .../cloud_pipelines/stacks/e4s-oneapi/spack.yaml | 245 +---------------- .../cloud_pipelines/stacks/e4s-power/spack.yaml | 225 +--------------- .../gitlab/cloud_pipelines/stacks/e4s/spack.yaml | 256 +----------------- .../cloud_pipelines/stacks/gpu-tests/spack.yaml | 267 +------------------ .../stacks/ml-linux-x86_64-cpu/spack.yaml | 84 +----- .../stacks/ml-linux-x86_64-cuda/spack.yaml | 84 +----- .../stacks/ml-linux-x86_64-rocm/spack.yaml | 86 +----- .../stacks/radiuss-aws-aarch64/spack.yaml | 124 +-------- .../cloud_pipelines/stacks/radiuss-aws/spack.yaml | 127 +-------- .../cloud_pipelines/stacks/radiuss/spack.yaml | 154 ----------- .../cloud_pipelines/stacks/tutorial/spack.yaml | 142 +--------- 30 files changed, 573 insertions(+), 2527 deletions(-) create mode 100644 share/spack/gitlab/cloud_pipelines/configs/cdash.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/configs/ci.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/configs/concretizer.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/configs/config.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/configs/linux/aarch64/ci.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/configs/linux/ppc64le/ci.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/configs/linux/x86_64/ci.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/configs/linux/x86_64_v4/ci.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/configs/packages.yaml (limited to 'share') diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml index 8903af5006..002280911c 100644 --- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml +++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml @@ -3,6 +3,32 @@ stages: [ "generate", "build", "publish" ] default: image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } +# CI Platform-Arch +.darwin_x86_64: + variables: + SPACK_TARGET_PLATFORM: "darwin" + SPACK_TARGET_ARCH: "x86_64" + +.linux_x86_64: + variables: + SPACK_TARGET_PLATFORM: "linux" + SPACK_TARGET_ARCH: "x86_64" + +.linux_x86_64_v4: + variables: + SPACK_TARGET_PLATFORM: "linux" + SPACK_TARGET_ARCH: "x86_64_v4" + +.linux_aarch64: + variables: + SPACK_TARGET_PLATFORM: "linux" + SPACK_TARGET_ARCH: "aarch64" + +.linux_power: + variables: + SPACK_TARGET_PLATFORM: "linux" + SPACK_TARGET_ARCH: "ppc64le" + ######################################## # Job templates ######################################## @@ -52,7 +78,13 @@ default: - spack --version - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME} - spack env activate --without-view . - - spack ci generate --check-index-only + - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs" + - spack + --config-scope "${SPACK_CI_CONFIG_ROOT}" + --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}" + --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}" + ${CI_STACK_CONFIG_SCOPES} + ci generate --check-index-only --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}" --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir" --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml" @@ -137,6 +169,7 @@ protected-publish: # My Super Cool Pipeline ######################################## # .my-super-cool-stack: +# extends: [ ".linux_x86_64" ] # variables: # SPACK_CI_STACK_NAME: my-super-cool-stack # tags: [ "all", "tags", "your", "job", "needs"] @@ -291,6 +324,7 @@ protected-publish: # E4S pipeline ######################################## .e4s: + extends: [ ".linux_x86_64" ] variables: SPACK_CI_STACK_NAME: e4s @@ -365,6 +399,7 @@ gpu-tests-protected-build: # E4S OneAPI Pipeline ######################################## .e4s-oneapi: + extends: [ ".linux_x86_64" ] variables: SPACK_CI_STACK_NAME: e4s-oneapi @@ -406,6 +441,7 @@ e4s-oneapi-protected-build: tags: ["spack", "public", "large", "ppc64le"] .e4s-power: + extends: [".linux_power"] variables: SPACK_CI_STACK_NAME: e4s-power @@ -441,6 +477,7 @@ e4s-power-protected-build: # Build tests for different build-systems ######################################### .build_systems: + extends: [ ".linux_x86_64" ] variables: SPACK_CI_STACK_NAME: build_systems @@ -476,6 +513,7 @@ build_systems-protected-build: # RADIUSS ######################################### .radiuss: + extends: [ ".linux_x86_64" ] variables: SPACK_CI_STACK_NAME: radiuss @@ -523,6 +561,7 @@ radiuss-protected-build: image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } .radiuss-aws: + extends: [ ".linux_x86_64_v4" ] variables: SPACK_CI_STACK_NAME: radiuss-aws @@ -558,6 +597,7 @@ radiuss-aws-protected-build: # Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64) .radiuss-aws-aarch64: + extends: [ ".linux_aarch64" ] variables: SPACK_CI_STACK_NAME: radiuss-aws-aarch64 @@ -593,6 +633,7 @@ radiuss-aws-aarch64-protected-build: # ECP Data & Vis SDK ######################################## .data-vis-sdk: + extends: [ ".linux_x86_64" ] variables: SPACK_CI_STACK_NAME: data-vis-sdk @@ -637,6 +678,7 @@ data-vis-sdk-protected-build: image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } .aws-ahug: + extends: [ ".linux_x86_64_v4" ] variables: SPACK_CI_STACK_NAME: aws-ahug @@ -671,6 +713,7 @@ aws-ahug-protected-build: # Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64) .aws-ahug-aarch64: + extends: [ ".linux_aarch64" ] variables: SPACK_CI_STACK_NAME: aws-ahug-aarch64 @@ -713,6 +756,7 @@ aws-ahug-aarch64-protected-build: image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } .aws-isc: + extends: [ ".linux_x86_64_v4" ] variables: SPACK_CI_STACK_NAME: aws-isc @@ -748,6 +792,7 @@ aws-isc-protected-build: # Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64) .aws-isc-aarch64: + extends: [ ".linux_aarch64" ] variables: SPACK_CI_STACK_NAME: aws-isc-aarch64 @@ -784,6 +829,7 @@ aws-isc-aarch64-protected-build: # Spack Tutorial ######################################## .tutorial: + extends: [ ".linux_x86_64" ] variables: SPACK_CI_STACK_NAME: tutorial @@ -819,6 +865,7 @@ tutorial-protected-build: # Machine Learning - Linux x86_64 (CPU) ####################################### .ml-linux-x86_64-cpu: + extends: [ ".linux_x86_64_v4" ] variables: SPACK_CI_STACK_NAME: ml-linux-x86_64-cpu @@ -858,6 +905,7 @@ ml-linux-x86_64-cpu-protected-build: # Machine Learning - Linux x86_64 (CUDA) ######################################## .ml-linux-x86_64-cuda: + extends: [ ".linux_x86_64_v4" ] variables: SPACK_CI_STACK_NAME: ml-linux-x86_64-cuda @@ -897,6 +945,7 @@ ml-linux-x86_64-cuda-protected-build: # Machine Learning - Linux x86_64 (ROCm) ######################################## .ml-linux-x86_64-rocm: + extends: [ ".linux_x86_64_v4" ] variables: SPACK_CI_STACK_NAME: ml-linux-x86_64-rocm diff --git a/share/spack/gitlab/cloud_pipelines/configs/cdash.yaml b/share/spack/gitlab/cloud_pipelines/configs/cdash.yaml new file mode 100644 index 0000000000..5deff13b33 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/cdash.yaml @@ -0,0 +1,6 @@ +cdash: + build-group: Spack CI + url: https://cdash.spack.io + project: Spack Testing + site: Cloud Gitlab Infrastructure + diff --git a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml new file mode 100644 index 0000000000..61de203360 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml @@ -0,0 +1,47 @@ +ci: + target: gitlab + + broken-tests-packages: + - gptune + + broken-specs-url: "https://dummy.io" # s3://spack-binaries/broken-specs" + + pipeline-gen: + - build-job: + script:: + - - spack compiler find + - cd ${SPACK_CONCRETE_ENV_DIR} + - spack env activate --without-view . + - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data + # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) + - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi + # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) + - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi + - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + after_script: + - - cat /proc/loadavg || true + variables: + CI_JOB_SIZE: "default" + SPACK_VERBOSE_SCRIPT: "1" + + - signing-job: + image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } + tags: ["aws"] + script: + - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp + - /sign.sh + - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + - aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp --recursive --exclude "*"" --include "*.pub" + + - any-job: + image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18" + tags: ["spack"] + before_script: + - - uname -a || true + - grep -E "vendor|model name" /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true + - nproc + - - . "./share/spack/setup-env.sh" + - spack --version + - spack arch diff --git a/share/spack/gitlab/cloud_pipelines/configs/concretizer.yaml b/share/spack/gitlab/cloud_pipelines/configs/concretizer.yaml new file mode 100644 index 0000000000..3e98ced6fd --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/concretizer.yaml @@ -0,0 +1,4 @@ +concretizer: + reuse: false + unify: false + diff --git a/share/spack/gitlab/cloud_pipelines/configs/config.yaml b/share/spack/gitlab/cloud_pipelines/configs/config.yaml new file mode 100644 index 0000000000..bb0d502310 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/config.yaml @@ -0,0 +1,10 @@ +config: + build_jobs: 32 + concretizer: clingo + db_lock_timeout: 120 + install_tree: + root: /home/software/spack + padded_length: 256 + projections: + all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + diff --git a/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml new file mode 100644 index 0000000000..f689e0c37a --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml @@ -0,0 +1,14 @@ +ci: + pipeline-gen: + - build-job: + script: | + - tmp="$(mktemp -d)"; export SPACK_USER_CONFIG_PATH="$tmp"; export SPACK_USER_CACHE_PATH="$tmp" + - . "./share/spack/setup-env.sh" + - spack --version + - spack arch + - cd ${SPACK_CONCRETE_ENV_DIR} + - spack env activate --without-view . + - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data + - spack --color=always --backtrace ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + tags: ["lambda"] diff --git a/share/spack/gitlab/cloud_pipelines/configs/linux/aarch64/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/linux/aarch64/ci.yaml new file mode 100644 index 0000000000..6b5de2fe55 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/linux/aarch64/ci.yaml @@ -0,0 +1,8 @@ +ci: + pipeline-gen: + - build-job: + before_script: + - - curl -Lf --no-progress-meter "https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.aarch64-linux-gnu.tar.gz" -o gmake.tar.gz + - printf "2322c175fb092b426f9eb6c24ee22d94ffa6759c3d0c260b74d81abd8120122b gmake.tar.gz" | sha256sum --check --strict --quiet + - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null + tags: ["aarch64"] diff --git a/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml new file mode 100644 index 0000000000..8503c89bd7 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml @@ -0,0 +1,290 @@ +ci: + pipeline-gen: + - match_behavior: first + submapping: + - match: + - hipblas + - llvm + - llvm-amdgpu + - pango + - paraview + - py-tensorflow + - py-torch + - qt + - rocblas + - visit + build-job: + tags: [ "spack", "huge" ] + variables: + CI_JOB_SIZE: huge + KUBERNETES_CPU_REQUEST: 11000m + KUBERNETES_MEMORY_REQUEST: 42G + + - match: + - ascent + - atk + - axom + - cistem + - cmake + - ctffind + - cuda + - dealii + - dray + - dyninst + - ecp-data-vis-sdk + - gcc + - ginkgo + - hdf5 + - hpx + - kokkos-kernels + - kokkos-nvcc-wrapper + - lbann + - magma + - mesa + - mfem + - mpich + - netlib-lapack + - nvhpc + - oce + - openblas + - openfoam + - openturns + - parallelio + - plumed + - precice + #- py-tensorflow + #- qt + - raja + - relion + #- rocblas + - rocfft + - rocsolver + - rocsparse + - rust + - slate + - strumpack + - sundials + - trilinos + - umpire + #- visit + - vtk + - vtk-h + - vtk-m + - warpx + - wrf + - wxwidgets + build-job: + tags: [ "spack", "large" ] + variables: + CI_JOB_SIZE: large + KUBERNETES_CPU_REQUEST: 8000m + KUBERNETES_MEMORY_REQUEST: 12G + + - match: + - adios2 + - amrex + - archer + - ascent + - autoconf-archive + - axom + - binutils + - blaspp + - blt + - boost + - butterflypack + - cabana + - caliper + - camp + - chai + - conduit + - curl + - datatransferkit + - double-conversion + - dray + - eigen + - faodel + - ffmpeg + - fftw + - fortrilinos + - gettext + - gperftools + - gptune + - hdf5 + - heffte + - hpctoolkit + - hwloc + - hydrogen + - hypre + - kokkos + - lammps + - lapackpp + - legion + - libtool + - libxml2 + - libzmq + - llvm-openmp-ompt + - mbedtls + - mfem + - mpich + - mvapich2 + - nasm + - netlib-scalapack + - omega-h + - openblas + - openjpeg + - openmpi + - openpmd-api + - pagmo2 + - papyrus + - parsec + - pdt + - pegtl + - petsc + - pumi + - py-beniget + - py-cinemasci + - pygmo + - py-ipython-genutils + - py-packaging + - py-petsc4py + - py-scipy + - py-statsmodels + - py-warlock + - py-warpx + - raja + - samrai + - slepc + - slurm + - sqlite + - strumpack + - sundials + - superlu-dist + - tasmanian + - tau + - upcxx + - vtk + - vtk-h + - vtk-m + - zfp + build-job: + tags: [ "spack", "medium" ] + variables: + CI_JOB_SIZE: "medium" + KUBERNETES_CPU_REQUEST: "2000m" + KUBERNETES_MEMORY_REQUEST: "4G" + + - match: + - alsa-lib + - ant + - antlr + - argobots + - autoconf-archive + - automake + - berkeley-db + - bison + - blt + - bzip2 + - camp + - cmake + - curl + - czmq + - darshan-util + - diffutils + - docbook-xml + - exmcutils + - expat + - findutils + - flit + - freetype + - gawk + - gdbm + - gettext + - glib + - gmake + - gotcha + - hpcviewer + - hwloc + - jansson + - json-c + - libbsd + - libedit + - libevent + - libfabric + - libffi + - libgcrypt + - libiconv + - libidn2 + - libjpeg-turbo + - libmd + - libnrm + - libpciaccess + - libpng + - libsigsegv + - libsodium + - libunistring + - libunwind + - libxml2 + - libyaml + - libzmq + - lua + - lua-luaposix + - lz4 + - m4 + - meson + - metis + - mpfr + - ncurses + - ninja + - numactl + - openblas + - openjdk + - openssh + - openssl + - papi + - parallel-netcdf + - pcre + - pcre2 + - pdsh + - perl + - perl-data-dumper + - pkgconf + - py-alembic + - py-cffi + - py-cycler + - py-decorator + - py-idna + - py-jsonschema + - py-kiwisolver + - py-mistune + - py-pycparser + - py-setuptools + - py-setuptools-scm + - py-six + - py-testpath + - py-wheel + - qhull + - readline + - sed + - slurm + - snappy + - sqlite + - superlu + - swig + - tar + - tcl + - texinfo + - tut + - unzip + - util-linux-uuid + - util-macros + - xz + - yaml-cpp + - zfp + - zlib + - zstd + build-job: + tags: [ "spack", "small" ] + variables: + CI_JOB_SIZE: "small" + KUBERNETES_CPU_REQUEST: "500m" + KUBERNETES_MEMORY_REQUEST: "500M" diff --git a/share/spack/gitlab/cloud_pipelines/configs/linux/ppc64le/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/linux/ppc64le/ci.yaml new file mode 100644 index 0000000000..df0956d23b --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/linux/ppc64le/ci.yaml @@ -0,0 +1,21 @@ +ci: + pipeline-gen: + - build-job: + tags: ["ppc64le"] + # Power runners overrides the default script + # - don't download make + # - no intermediate keys + script:: + - uname -a || true + - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true + - nproc + - . "./share/spack/setup-env.sh" + - spack --version + - spack arch + - cd ${SPACK_CONCRETE_ENV_DIR} + - spack env activate --without-view . + - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data + - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi + - spack --color=always --backtrace ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) diff --git a/share/spack/gitlab/cloud_pipelines/configs/linux/x86_64/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/linux/x86_64/ci.yaml new file mode 100644 index 0000000000..7844fbae3e --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/linux/x86_64/ci.yaml @@ -0,0 +1,8 @@ +ci: + pipeline-gen: + - build-job: + before_script: + - - curl -Lf --no-progress-meter "https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz" -o gmake.tar.gz + - printf "fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz" | sha256sum --check --strict --quiet + - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null + tags: ["x86_64"] diff --git a/share/spack/gitlab/cloud_pipelines/configs/linux/x86_64_v4/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/linux/x86_64_v4/ci.yaml new file mode 100644 index 0000000000..6076119382 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/linux/x86_64_v4/ci.yaml @@ -0,0 +1,8 @@ +ci: + pipeline-gen: + - build-job: + before_script: + - - curl -Lf --no-progress-meter "https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz" -o gmake.tar.gz + - printf "fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz" | sha256sum --check --strict --quiet + - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null + tags: ["x86_64_v4"] diff --git a/share/spack/gitlab/cloud_pipelines/configs/packages.yaml b/share/spack/gitlab/cloud_pipelines/configs/packages.yaml new file mode 100644 index 0000000000..067e60f4e4 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/packages.yaml @@ -0,0 +1,23 @@ +packages: + #all: + # CI should never build develop/main/master versions of + # packages. + # Current issues: + # - e4s/dav + # - hdf5-vol-async => argobot@main + # - aws-ahug-* + # - snap + # - tycho2 + # - amg2013 + # - cosp2 + # - snbone + # - vpfft + # - vpic + # - aws-isc-aarch64 + # - sse2neon + #require: "@:999999999" + python: + # This is redundant after https://github.com/spack/spack/pull/33898 + # but specified to allow the existance of a package.yaml for CI configs + require: "@3.7:" + diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml index bd0aa1170b..3bd3cce2fd 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml @@ -1,19 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: false - - config: - build_jobs: 32 - concretizer: clingo - install_tree: - root: /home/software/spack - padded_length: 384 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - packages: all: providers: @@ -241,102 +227,12 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/aws-ahug-aarch64" } - gitlab-ci: - - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.aarch64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf '2322c175fb092b426f9eb6c24ee22d94ffa6759c3d0c260b74d81abd8120122b gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - after_script: - - cat /proc/loadavg || true - - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - match_behavior: first - mappings: - - match: - - llvm - - llvm-amdgpu - - paraview - runner-attributes: - tags: [ "spack", "huge", "aarch64" ] - variables: - CI_JOB_SIZE: huge - KUBERNETES_CPU_REQUEST: 11000m - KUBERNETES_MEMORY_REQUEST: 42G - - - - match: - - ascent - - axom - - cuda - - dyninst - - gcc - - ginkgo - - hpx - - kokkos-kernels - - kokkos-nvcc-wrapper - - magma - - mfem - - mpich - - openturns - - precice - - raja - - rocblas - - rocsolver - - rust - - slate - - strumpack - - sundials - - trilinos - - umpire - - vtk-h - - vtk-m - - warpx - runner-attributes: - tags: [ "spack", "large", "aarch64" ] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - - match: ['os=amzn2'] - runner-attributes: - tags: ["spack", "aarch64"] - variables: - CI_JOB_SIZE: "default" - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - tags: ["spack", "public", "aarch64"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + ci: + pipeline-gen: + - build-job: + image: + name: "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21" + entrypoint: [""] cdash: build-group: AHUG ARM HPC User Group - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml index 997cfe10e6..82d2d0f344 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml @@ -1,19 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: false - - config: - build_jobs: 32 - concretizer: clingo - install_tree: - root: /home/software/spack - padded_length: 384 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - packages: all: providers: @@ -238,106 +224,12 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/aws-ahug" } - gitlab-ci: - - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf 'fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi - # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - after_script: - - cat /proc/loadavg || true - - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - match_behavior: first - mappings: - - match: - - llvm - - llvm-amdgpu - - paraview - runner-attributes: - tags: [ "spack", "huge", "x86_64_v4" ] - variables: - CI_JOB_SIZE: huge - KUBERNETES_CPU_REQUEST: 11000m - KUBERNETES_MEMORY_REQUEST: 42G - - - - match: - - ascent - - axom - - cuda - - dyninst - - gcc - - ginkgo - - hpx - - kokkos-kernels - - kokkos-nvcc-wrapper - - magma - - mfem - - mpich - - nvhpc - - openturns - - precice - - raja - - rocblas - - rocsolver - - rust - - slate - - strumpack - - sundials - - trilinos - - umpire - - vtk-h - - vtk-m - - warpx - runner-attributes: - tags: [ "spack", "large", "x86_64_v4" ] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - - match: ['os=amzn2'] - runner-attributes: - tags: ["spack", "x86_64_v4"] - variables: - CI_JOB_SIZE: "default" - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - tags: ["spack", "public", "x86_64_v4"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + ci: + pipeline-gen: + - build-job: + image: + name: "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21" + entrypoint: [""] cdash: build-group: AHUG ARM HPC User Group - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml index 6c36b4e1f2..a563421263 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml @@ -1,19 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: false - - config: - build_jobs: 32 - concretizer: clingo - install_tree: - root: /home/software/spack - padded_length: 384 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - packages: all: providers: @@ -148,113 +134,10 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/aws-isc-aarch64" } - gitlab-ci: - - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.aarch64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf '2322c175fb092b426f9eb6c24ee22d94ffa6759c3d0c260b74d81abd8120122b gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - after_script: - - cat /proc/loadavg || true - - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - match_behavior: first - mappings: - - match: - - llvm - - llvm-amdgpu - - paraview - runner-attributes: - tags: [ "spack", "huge", "aarch64" ] - variables: - CI_JOB_SIZE: huge - KUBERNETES_CPU_REQUEST: 15000m - KUBERNETES_MEMORY_REQUEST: 62G - - - match: - - ascent - - atk - - axom - - cistem - - ctffind - - cuda - - dyninst - - gcc - - ginkgo - - hdf5 - - hpx - - kokkos-kernels - - kokkos-nvcc-wrapper - - magma - - mfem - - mpich - - openfoam - - openturns - - parallelio - - precice - - qt - - raja - - relion - - rocblas - - rocsolver - - rust - - slate - - strumpack - - sundials - - trilinos - - umpire - - visit - - vtk - - vtk-h - - vtk-m - - warpx - - wrf - - wxwidgets - runner-attributes: - tags: [ "spack", "large", "aarch64" ] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - - match: ['os=amzn2'] - runner-attributes: - tags: ["spack", "aarch64"] - variables: - CI_JOB_SIZE: "default" - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - tags: ["spack", "public", "aarch64"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + ci: + pipeline-gen: + - build-job: + image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } cdash: build-group: AWS Packages - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml index 079fc09ffd..8e56d94709 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml @@ -1,19 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: false - - config: - build_jobs: 32 - concretizer: clingo - install_tree: - root: /home/software/spack - padded_length: 384 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - packages: all: providers: @@ -159,118 +145,10 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/aws-isc" } - gitlab-ci: - - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf 'fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi - # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - after_script: - - cat /proc/loadavg || true - - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - match_behavior: first - mappings: - - match: - - llvm - - llvm-amdgpu - - pango - - paraview - runner-attributes: - tags: [ "spack", "huge", "x86_64_v4" ] - variables: - CI_JOB_SIZE: huge - KUBERNETES_CPU_REQUEST: 11000m - KUBERNETES_MEMORY_REQUEST: 42G - - - - match: - - ascent - - atk - - axom - - cistem - - ctffind - - cuda - - dyninst - - gcc - - ginkgo - - hdf5 - - hpx - - kokkos-kernels - - kokkos-nvcc-wrapper - - magma - - mfem - - mpich - - openfoam - - openturns - - parallelio - - precice - - qt - - raja - - relion - - rocblas - - rocsolver - - rust - - slate - - strumpack - - sundials - - trilinos - - umpire - - visit - - vtk - - vtk-h - - vtk-m - - warpx - - wrf - - wxwidgets - runner-attributes: - tags: [ "spack", "large", "x86_64_v4" ] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - - match: ['os=amzn2'] - runner-attributes: - tags: ["spack", "x86_64_v4"] - variables: - CI_JOB_SIZE: "default" - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - tags: ["spack", "public", "x86_64_v4"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + ci: + pipeline-gen: + - build-job: + image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } cdash: build-group: AWS Packages - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml index 607af45b99..5e227dc23d 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml @@ -1,18 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: when_possible - - config: - build_jobs: 32 - install_tree: - root: /home/software/spack - padded_length: 512 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - definitions: - default_specs: - 'uncrustify build_system=autotools' @@ -33,107 +20,5 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/build_systems" } - gitlab-ci: - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf 'fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi - # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild - after_script: - - cat /proc/loadavg || true - - image: - name: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18" - entrypoint: [ "" ] - - match_behavior: first - mappings: - - match: - - cmake - runner-attributes: - tags: [ "spack", "large", "x86_64"] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - - match: - - curl - - gettext - - mpich - - openjpeg - - sqlite - runner-attributes: - tags: [ "spack", "medium", "x86_64" ] - variables: - CI_JOB_SIZE: "medium" - KUBERNETES_CPU_REQUEST: "2000m" - KUBERNETES_MEMORY_REQUEST: "4G" - - - match: - - bzip2 - - diffutils - - findutils - - libffi - - libidn2 - - libmd - - libsigsegv - - libxml2 - - lz4 - - openssl - - pkgconf - - tut - - util-linux-uuid - - util-macros - - xz - - zlib - runner-attributes: - tags: [ "spack", "medium", "x86_64" ] - variables: - CI_JOB_SIZE: "small" - KUBERNETES_CPU_REQUEST: "500m" - KUBERNETES_MEMORY_REQUEST: "500M" - - - match: - - 'os=ubuntu18.04' - runner-attributes: - tags: ["spack", "x86_64"] - variables: - CI_JOB_SIZE: "default" - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } - tags: ["spack", "public", "x86_64"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache - cdash: build-group: Build tests for different build systems - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml index 3cd17d34ec..0bfa7f9f49 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml @@ -1,18 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: false - - config: - build_jobs: 32 - install_tree: - root: /home/software/spack - padded_length: 512 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - packages: cmake: variants: ~ownlibs @@ -72,142 +59,10 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/data-vis-sdk" } - gitlab-ci: - image: ecpe4s/ubuntu20.04-runner-x86_64:2022-10-01 - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf 'fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi - # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild - after_script: - - cat /proc/loadavg || true - - match_behavior: first - mappings: - - match: - - llvm - - qt - - paraview - - visit - runner-attributes: - tags: [ "spack", "huge", "x86_64" ] - variables: - CI_JOB_SIZE: huge - KUBERNETES_CPU_REQUEST: 11000m - KUBERNETES_MEMORY_REQUEST: 42G - - - match: - - ecp-data-vis-sdk - - mesa - - openblas - - vtk - - vtk-m - runner-attributes: - tags: [ "spack", "large", "x86_64" ] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - - match: - - adios2 - - ascent - - binutils - - blt - - boost - - conduit - - double-conversion - - dray - - eigen - - faodel - - hdf5 - - mfem - - nasm - - openmpi - - pegtl - - py-cinemasci - - raja - - vtk-h - runner-attributes: - tags: [ "spack", "medium", "x86_64" ] - variables: - CI_JOB_SIZE: "medium" - KUBERNETES_CPU_REQUEST: "2000m" - KUBERNETES_MEMORY_REQUEST: "4G" - - - match: - - darshan-util - - docbook-xml - - gdbm - - gettext - - hwloc - - libevent - - libmd - - libpciaccess - - libsigsegv - - libunwind - - libxml2 - - libzmq - - numactl - - openssh - - pcre - - perl-data-dumper - - py-cycler - - py-decorator - - py-mistune - - py-pycparser - - py-setuptools - - py-wheel - - readline - - sqlite - - tar - - util-linux-uuid - runner-attributes: - tags: [ "spack", "small", "x86_64" ] - variables: - CI_JOB_SIZE: "small" - KUBERNETES_CPU_REQUEST: "500m" - KUBERNETES_MEMORY_REQUEST: "500M" - - - match: ['@:'] - runner-attributes: - tags: ["spack", "x86_64"] - variables: - CI_JOB_SIZE: "default" - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - image: ecpe4s/ubuntu20.04-runner-x86_64:2022-10-01 - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - tags: ["spack", "public", "medium", "x86_64"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + ci: + pipeline-gen: + - build-job: + image: { "name": "ecpe4s/ubuntu20.04-runner-x86_64:2022-10-01", "entrypoint": [""] } cdash: - build-group: Data and Vis SDK - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure + build-group:: Data and Vis SDK diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml index 539dcd5d59..88b081d2eb 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml @@ -1,18 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: false - - config: - concretizer: clingo - install_tree: - root: $SPACK_ROOT/opt/spack - padded_length: 512 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - packages: all: compiler: [apple-clang@13.1.6] @@ -39,39 +26,15 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/e4s-mac" } - gitlab-ci: - - script: - - tmp="$(mktemp -d)"; export SPACK_USER_CONFIG_PATH="$tmp"; export SPACK_USER_CACHE_PATH="$tmp" - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - - spack --color=always --backtrace ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - - match_behavior: first - mappings: - - match: ['os=monterey'] - runner-attributes: - tags: - - lambda - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - export SPACK_USER_CACHE_PATH=$(pwd)/.spack-user-cache - - export SPACK_USER_CONFIG_PATH=$(pwd)/.spack-user-config - - . "./share/spack/setup-env.sh" - - spack --version - tags: - - lambda + ci: + pipeline-gen: + - cleanup-job: + before_script: | + - export SPACK_USER_CACHE_PATH=$(pwd)/.spack-user-cache + - export SPACK_USER_CONFIG_PATH=$(pwd)/.spack-user-config + - . "./share/spack/setup-env.sh" + - spack --version + tags: [lambda] cdash: build-group: E4S Mac - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml index a5357a4206..2c4d996f10 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml @@ -1,19 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: false - - config: - build_jobs: 32 - concretizer: clingo - install_tree: - root: /home/software/spack - padded_length: 256 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - compilers: - compiler: spec: dpcpp@2023.0.0 @@ -279,233 +265,10 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/e4s-oneapi" } - gitlab-ci: - - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf 'fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi - # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - export PATH=/bootstrap/runner/view/bin:${PATH} - - . /bootstrap/runner/install/linux-ubuntu20.04-x86_64/gcc-11.1.0/lmod-8.7.2-b7eq7rjeckn3m4o4lglsakilibkznjom/lmod/8.7.2/init/bash - - module use /opt/intel/oneapi/modulefiles - - module load compiler - - spack --color=always --backtrace ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - after_script: - - cat /proc/loadavg || true - - image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023-01-01 - - match_behavior: first - mappings: - - match: - - hipblas - - llvm - - llvm-amdgpu - - paraview - - rocblas - runner-attributes: - tags: [ "spack", "huge", "x86_64" ] - variables: - CI_JOB_SIZE: huge - KUBERNETES_CPU_REQUEST: 11000m - KUBERNETES_MEMORY_REQUEST: 42G - - - match: - - cuda - - dyninst - - ginkgo - - hpx - - kokkos-kernels - - kokkos-nvcc-wrapper - - magma - - mfem - - mpich - - openturns - - plumed - - precice - - qt - - raja - - rust - - slate - - trilinos - - vtk-m - - warpx - runner-attributes: - tags: [ "spack", "large", "x86_64" ] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - - match: - - adios2 - - amrex - - archer - - ascent - - axom - - binutils - - blaspp - - boost - - butterflypack - - cabana - - caliper - - camp - - chai - - conduit - - datatransferkit - - faodel - - ffmpeg - - fftw - - fortrilinos - - gperftools - - gptune - - hdf5 - - heffte - - hpctoolkit - - hwloc - - hypre - - kokkos - - lammps - - lapackpp - - legion - - libzmq - - llvm-openmp-ompt - - mbedtls - - netlib-scalapack - - omega-h - - openmpi - - openpmd-api - - pagmo2 - - papyrus - - parsec - - pdt - - petsc - - pumi - - py-ipython-genutils - - py-petsc4py - - py-scipy - - py-statsmodels - - py-warlock - - py-warpx - - pygmo - - slepc - - slurm - - strumpack - - sundials - - superlu-dist - - tasmanian - - tau - - upcxx - - vtk-h - - zfp - runner-attributes: - tags: [ "spack", "medium", "x86_64" ] - variables: - CI_JOB_SIZE: "medium" - KUBERNETES_CPU_REQUEST: "2000m" - KUBERNETES_MEMORY_REQUEST: "4G" - - - match: - - alsa-lib - - ant - - antlr - - argobots - - automake - - berkeley-db - - bison - - blt - - cmake - - curl - - darshan-util - - diffutils - - exmcutils - - expat - - flit - - freetype - - gdbm - - gotcha - - hpcviewer - - jansson - - json-c - - libbsd - - libevent - - libjpeg-turbo - - libnrm - - libpng - - libunistring - - lua-luaposix - - m4 - - mpfr - - ncurses - - openblas - - openjdk - - papi - - parallel-netcdf - - pcre2 - - perl-data-dumper - - pkgconf - - py-alembic - - py-idna - - py-testpath - - qhull - - snappy - - swig - - tar - - tcl - - texinfo - - unzip - - util-linux-uuid - - util-macros - - yaml-cpp - - zlib - - zstd - runner-attributes: - tags: [ "spack", "small", "x86_64" ] - variables: - CI_JOB_SIZE: "small" - KUBERNETES_CPU_REQUEST: "500m" - KUBERNETES_MEMORY_REQUEST: "500M" - - - match: ['os=ubuntu20.04'] - runner-attributes: - tags: ["spack", "x86_64"] - variables: - CI_JOB_SIZE: "default" - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023-01-01 - tags: ["spack", "public", "x86_64"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + ci: + pipeline-gen: + - build-job: + image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023-01-01 cdash: build-group: E4S OneAPI - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml index be7d5e267d..ab8efe112c 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml @@ -1,18 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: false - - config: - concretizer: clingo - install_tree: - root: /home/software/spack - padded_length: 512 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - packages: all: compiler: [gcc@11.1.0] @@ -212,214 +199,10 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/e4s-power" } - gitlab-ci: - - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - after_script: - - cat /proc/loadavg || true - - image: ecpe4s/ubuntu20.04-runner-ppc64le:2023-01-01 - - match_behavior: first - mappings: - - match: - - hipblas - - llvm - - llvm-amdgpu - - rocblas - - paraview - - visit - runner-attributes: - tags: [ "spack", "huge", "ppc64le" ] - variables: - CI_JOB_SIZE: huge - - - match: - - cuda - - dyninst - - ginkgo - - hpx - - kokkos-kernels - - kokkos-nvcc-wrapper - - magma - - mfem - - mpich - - openturns - - precice - - raja - - rust - - slate - - trilinos - - vtk-m - - warpx - runner-attributes: - tags: [ "spack", "large", "ppc64le" ] - variables: - CI_JOB_SIZE: large - - - match: - - adios2 - - amrex - - archer - - ascent - - axom - - binutils - - blaspp - - boost - - butterflypack - - cabana - - caliper - - camp - - chai - - conduit - - datatransferkit - - faodel - - ffmpeg - - fftw - - fortrilinos - - gperftools - - gptune - - hdf5 - - heffte - - hpctoolkit - - hwloc - - hypre - - kokkos - - lammps - - lapackpp - - legion - - libzmq - - llvm-openmp-ompt - - mbedtls - - netlib-scalapack - - omega-h - - openmpi - - openpmd-api - - pagmo2 - - papyrus - - parsec - - pdt - - petsc - - pumi - - py-ipython-genutils - - py-petsc4py - - py-scipy - - py-statsmodels - - py-warlock - - py-warpx - - pygmo - - slepc - - slurm - - strumpack - - sundials - - superlu-dist - - tasmanian - - tau - - upcxx - - vtk-h - - zfp - runner-attributes: - tags: [ "spack", "medium", "ppc64le" ] - variables: - CI_JOB_SIZE: "medium" - - - match: - - alsa-lib - - ant - - antlr - - argobots - - automake - - berkeley-db - - bison - - blt - - cmake - - curl - - darshan-util - - diffutils - - exmcutils - - expat - - flit - - freetype - - gdbm - - gotcha - - hpcviewer - - jansson - - json-c - - libbsd - - libevent - - libjpeg-turbo - - libnrm - - libpng - - libunistring - - lua-luaposix - - m4 - - mpfr - - ncurses - - openblas - - openjdk - - papi - - parallel-netcdf - - pcre2 - - perl-data-dumper - - pkgconf - - py-alembic - - py-idna - - py-testpath - - qhull - - snappy - - swig - - tar - - tcl - - texinfo - - unzip - - util-linux-uuid - - util-macros - - yaml-cpp - - zlib - - zstd - runner-attributes: - tags: [ "spack", "small", "ppc64le" ] - variables: - CI_JOB_SIZE: "small" - - - match: ['os=ubuntu20.04'] - runner-attributes: - tags: ["spack", "ppc64le"] - variables: - CI_JOB_SIZE: "default" - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - image: ecpe4s/ubuntu20.04-runner-ppc64le:2023-01-01 - tags: ["spack", "public", "ppc64le"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + ci: + pipeline-gen: + - build-job: + image: ecpe4s/ubuntu20.04-runner-ppc64le:2023-01-01 cdash: build-group: E4S Power - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml index 8b961a32e0..e4cd4ed3f2 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml @@ -1,19 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: false - - config: - build_jobs: 32 - concretizer: clingo - install_tree: - root: /home/software/spack - padded_length: 512 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - packages: all: compiler: [gcc@11.1.0] @@ -238,244 +224,10 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/e4s" } - gitlab-ci: - - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf 'fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi - # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - after_script: - - cat /proc/loadavg || true - - image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 - - broken-tests-packages: - - gptune - - match_behavior: first - mappings: - - - match: - - hipblas - - llvm - - llvm-amdgpu - - rocblas - - paraview - - py-torch - runner-attributes: - tags: [ "spack", "huge", "x86_64" ] - variables: - CI_JOB_SIZE: huge - KUBERNETES_CPU_REQUEST: 11000m - KUBERNETES_MEMORY_REQUEST: 42G - - - match: - - cuda - - dealii - - dray - - dyninst - - ginkgo - - hpx - - kokkos-kernels - - kokkos-nvcc-wrapper - - magma - - mfem - - mpich - - nvhpc - - oce - - openturns - - plumed - - precice - - py-tensorflow - - qt - - raja - - rocfft - - rocsolver - - rocsparse - - rust - - slate - - trilinos - - visit - - vtk - - vtk-m - - warpx - runner-attributes: - tags: [ "spack", "large", "x86_64" ] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - - match: - - adios2 - - amrex - - archer - - ascent - - axom - - binutils - - blaspp - - boost - - butterflypack - - cabana - - caliper - - camp - - chai - - conduit - - datatransferkit - - faodel - - ffmpeg - - fftw - - fortrilinos - - gperftools - - gptune - - hdf5 - - heffte - - hpctoolkit - - hwloc - - hypre - - kokkos - - lammps - - lapackpp - - legion - - libzmq - - llvm-openmp-ompt - - mbedtls - - netlib-scalapack - - omega-h - - openmpi - - openpmd-api - - pagmo2 - - papyrus - - parsec - - pdt - - petsc - - pumi - - py-ipython-genutils - - py-petsc4py - - py-scipy - - py-statsmodels - - py-warlock - - py-warpx - - pygmo - - slepc - - slurm - - strumpack - - sundials - - superlu-dist - - tasmanian - - tau - - upcxx - - vtk-h - - zfp - runner-attributes: - tags: [ "spack", "medium", "x86_64" ] - variables: - CI_JOB_SIZE: "medium" - KUBERNETES_CPU_REQUEST: "2000m" - KUBERNETES_MEMORY_REQUEST: "4G" - - - match: - - alsa-lib - - ant - - antlr - - argobots - - automake - - berkeley-db - - bison - - blt - - cmake - - curl - - darshan-util - - diffutils - - exmcutils - - expat - - flit - - freetype - - gdbm - - gotcha - - hpcviewer - - jansson - - json-c - - libbsd - - libevent - - libjpeg-turbo - - libnrm - - libpng - - libunistring - - lua-luaposix - - m4 - - mpfr - - ncurses - - openblas - - openjdk - - papi - - parallel-netcdf - - pcre2 - - perl-data-dumper - - pkgconf - - py-alembic - - py-idna - - py-testpath - - qhull - - snappy - - swig - - tar - - tcl - - texinfo - - unzip - - util-linux-uuid - - util-macros - - yaml-cpp - - zlib - - zstd - runner-attributes: - tags: [ "spack", "small", "x86_64" ] - variables: - CI_JOB_SIZE: "small" - KUBERNETES_CPU_REQUEST: "500m" - KUBERNETES_MEMORY_REQUEST: "500M" - - - match: ['os=ubuntu20.04'] - runner-attributes: - tags: ["spack", "x86_64"] - variables: - CI_JOB_SIZE: "default" - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 - tags: ["spack", "public", "x86_64"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + ci: + pipeline-gen: + - build-job: + image: "ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01" cdash: build-group: E4S - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml index dde2f11102..96b5809c71 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml @@ -1,19 +1,6 @@ spack: view: false - concretizer: - reuse: false - unify: false - - config: - build_jobs: 32 - concretizer: clingo - install_tree: - root: /home/software/spack - padded_length: 512 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - packages: all: compiler: [gcc@11.1.0] @@ -66,260 +53,22 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/gpu-tests" } - gitlab-ci: - - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - nvidia-smi || true - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf 'fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi - # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - - after_script: - - cat /proc/loadavg || true - - image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 - - broken-tests-packages: - - gptune - - match_behavior: first - mappings: + ci: + pipeline-gen: + - build-job: + image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 + - match_behavior: first + submapping: - match: - kokkos +rocm amdgpu_target=gfx90a - runner-attributes: + build-job: tags: [ "rocm-5.4.0", "mi210" ] - variables: - CI_JOB_SIZE: large - match: - kokkos +cuda cuda_arch=80 ^cuda@12.0.0 - raja +cuda cuda_arch=80 ^cuda@12.0.0 - runner-attributes: + build-job: tags: [ "nvidia-525.85.12", "cuda-12.0", "a100" ] - variables: - CI_JOB_SIZE: large - - - match: - - hipblas - - llvm - - llvm-amdgpu - - rocblas - - paraview - - py-torch - runner-attributes: - tags: [ "spack", "huge", "x86_64" ] - variables: - CI_JOB_SIZE: huge - KUBERNETES_CPU_REQUEST: 11000m - KUBERNETES_MEMORY_REQUEST: 42G - - - match: - - cuda - - dealii - - dray - - dyninst - - ginkgo - - hpx - - kokkos-kernels - - kokkos-nvcc-wrapper - - magma - - mfem - - mpich - - nvhpc - - oce - - openturns - - plumed - - precice - - py-tensorflow - - qt - - raja - - rocfft - - rocsolver - - rocsparse - - rust - - slate - - trilinos - - visit - - vtk - - vtk-m - - warpx - runner-attributes: - tags: [ "spack", "large", "x86_64" ] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - - match: - - adios2 - - amrex - - archer - - ascent - - axom - - binutils - - blaspp - - boost - - butterflypack - - cabana - - caliper - - camp - - chai - - conduit - - datatransferkit - - faodel - - ffmpeg - - fftw - - fortrilinos - - gperftools - - gptune - - hdf5 - - heffte - - hpctoolkit - - hwloc - - hypre - - kokkos - - lammps - - lapackpp - - legion - - libzmq - - llvm-openmp-ompt - - mbedtls - - netlib-scalapack - - omega-h - - openmpi - - openpmd-api - - pagmo2 - - papyrus - - parsec - - pdt - - petsc - - pumi - - py-ipython-genutils - - py-petsc4py - - py-scipy - - py-statsmodels - - py-warlock - - py-warpx - - pygmo - - slepc - - slurm - - strumpack - - sundials - - superlu-dist - - tasmanian - - tau - - upcxx - - vtk-h - - zfp - runner-attributes: - tags: [ "spack", "medium", "x86_64" ] - variables: - CI_JOB_SIZE: "medium" - KUBERNETES_CPU_REQUEST: "2000m" - KUBERNETES_MEMORY_REQUEST: "4G" - - - match: - - alsa-lib - - ant - - antlr - - argobots - - automake - - berkeley-db - - bison - - blt - - cmake - - curl - - darshan-util - - diffutils - - exmcutils - - expat - - flit - - freetype - - gdbm - - gotcha - - hpcviewer - - jansson - - json-c - - libbsd - - libevent - - libjpeg-turbo - - libnrm - - libpng - - libunistring - - lua-luaposix - - m4 - - mpfr - - ncurses - - openblas - - openjdk - - papi - - parallel-netcdf - - pcre2 - - perl-data-dumper - - pkgconf - - py-alembic - - py-idna - - py-testpath - - qhull - - snappy - - swig - - tar - - tcl - - texinfo - - unzip - - util-linux-uuid - - util-macros - - yaml-cpp - - zlib - - zstd - runner-attributes: - tags: [ "spack", "small", "x86_64" ] - variables: - CI_JOB_SIZE: "small" - KUBERNETES_CPU_REQUEST: "500m" - KUBERNETES_MEMORY_REQUEST: "500M" - - - match: ['os=ubuntu20.04'] - runner-attributes: - tags: ["spack", "x86_64"] - variables: - CI_JOB_SIZE: "default" - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 - tags: ["spack", "public", "x86_64"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache cdash: build-group: GPU Testing - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml index db171050de..f4345a2bfa 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml @@ -1,19 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: false - - config: - build_jobs: 32 - concretizer: clingo - install_tree: - root: /home/software/spack - padded_length: 384 - projections: - all: "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}" - packages: all: target: [x86_64_v3] @@ -96,72 +82,10 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/ml-linux-x86_64-cpu" } - gitlab-ci: - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf 'fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi - # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - after_script: - - cat /proc/loadavg || true - - match_behavior: first - mappings: - - match: - - llvm - - py-tensorflow - - py-torch - runner-attributes: - tags: [ "spack", "huge", "x86_64_v4" ] - variables: - CI_JOB_SIZE: huge - KUBERNETES_CPU_REQUEST: 11000m - KUBERNETES_MEMORY_REQUEST: 42G - - match: - - "@:" - runner-attributes: - tags: [ "spack", "large", "x86_64_v4" ] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - tags: ["spack", "public", "x86_64_v4"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + ci: + pipeline-gen: + - build-job: + image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } cdash: build-group: Machine Learning - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml index f35457239b..6e9d2cff4e 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml @@ -1,19 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: false - - config: - build_jobs: 32 - concretizer: clingo - install_tree: - root: /home/software/spack - padded_length: 384 - projections: - all: "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}" - packages: all: target: [x86_64_v3] @@ -99,72 +85,10 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/ml-linux-x86_64-cuda" } - gitlab-ci: - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf 'fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi - # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - after_script: - - cat /proc/loadavg || true - - match_behavior: first - mappings: - - match: - - llvm - - py-tensorflow - - py-torch - runner-attributes: - tags: [ "spack", "huge", "x86_64_v4" ] - variables: - CI_JOB_SIZE: huge - KUBERNETES_CPU_REQUEST: 11000m - KUBERNETES_MEMORY_REQUEST: 42G - - match: - - "@:" - runner-attributes: - tags: [ "spack", "large", "x86_64_v4" ] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - tags: ["spack", "public", "x86_64_v4"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + ci: + pipeline-gen: + - build-job: + image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } cdash: build-group: Machine Learning - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml index eb861de4cb..df23bcdcfa 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml @@ -1,19 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: false - - config: - build_jobs: 32 - concretizer: clingo - install_tree: - root: /home/software/spack - padded_length: 384 - projections: - all: "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}" - packages: all: target: [x86_64_v3] @@ -102,74 +88,10 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/ml-linux-x86_64-rocm" } - gitlab-ci: - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf 'fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi - # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - after_script: - - cat /proc/loadavg || true - - match_behavior: first - mappings: - - match: - - llvm - - llvm-amdgpu - - py-tensorflow - - py-torch - - rocblas - runner-attributes: - tags: [ "spack", "huge", "x86_64_v4" ] - variables: - CI_JOB_SIZE: huge - KUBERNETES_CPU_REQUEST: 11000m - KUBERNETES_MEMORY_REQUEST: 42G - - match: - - "@:" - runner-attributes: - tags: [ "spack", "large", "x86_64_v4" ] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - tags: ["spack", "public", "x86_64_v4"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + ci: + pipeline-gen: + - build-job: + image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } cdash: build-group: Machine Learning - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml index f5a12df7b2..6fe36a2462 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml @@ -1,19 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: false - - config: - build_jobs: 32 - concretizer: clingo - install_tree: - root: /home/software/spack - padded_length: 384 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - packages: all: providers: @@ -54,112 +40,10 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/radiuss-aws-aarch64" } - gitlab-ci: - - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.aarch64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf '2322c175fb092b426f9eb6c24ee22d94ffa6759c3d0c260b74d81abd8120122b gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - after_script: - - cat /proc/loadavg || true - - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - match_behavior: first - mappings: - - match: - - llvm - - llvm-amdgpu - - pango - - paraview - runner-attributes: - tags: [ "spack", "huge", "aarch64" ] - variables: - CI_JOB_SIZE: huge - KUBERNETES_CPU_REQUEST: 11000m - KUBERNETES_MEMORY_REQUEST: 42G - - - - match: - - ascent - - atk - - axom - - cistem - - ctffind - - cuda - - dyninst - - gcc - - ginkgo - - hdf5 - - hpx - - kokkos-kernels - - kokkos-nvcc-wrapper - - magma - - mfem - - mpich - - openturns - - parallelio - - precice - - raja - - relion - - rocblas - - rocsolver - - rust - - slate - - strumpack - - sundials - - trilinos - - umpire - - vtk - - vtk-h - - vtk-m - - warpx - - wrf - - wxwidgets - runner-attributes: - tags: [ "spack", "large", "aarch64" ] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - - match: ['os=amzn2'] - runner-attributes: - tags: ["spack", "aarch64"] - variables: - CI_JOB_SIZE: "default" - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - tags: ["spack", "public", "aarch64"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + ci: + pipeline-gen: + - build-job: + image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } cdash: build-group: RADIUSS AWS Packages - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml index 5a71d8399d..c5d478fa94 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml @@ -1,19 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: false - - config: - build_jobs: 32 - concretizer: clingo - install_tree: - root: /home/software/spack - padded_length: 384 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - packages: all: providers: @@ -59,115 +45,10 @@ spack: mirrors: { "mirror": "s3://spack-binaries/develop/radiuss-aws" } - gitlab-ci: - - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf 'fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi - # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) - after_script: - - cat /proc/loadavg || true - - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - match_behavior: first - mappings: - - match: - - llvm - - llvm-amdgpu - - pango - - paraview - runner-attributes: - tags: [ "spack", "huge", "x86_64_v4" ] - variables: - CI_JOB_SIZE: huge - KUBERNETES_CPU_REQUEST: 11000m - KUBERNETES_MEMORY_REQUEST: 42G - - - - match: - - ascent - - atk - - axom - - cistem - - ctffind - - cuda - - dyninst - - gcc - - ginkgo - - hdf5 - - hpx - - kokkos-kernels - - kokkos-nvcc-wrapper - - magma - - mfem - - mpich - - openturns - - parallelio - - precice - - raja - - relion - - rocblas - - rocsolver - - rust - - slate - - strumpack - - sundials - - trilinos - - umpire - - vtk - - vtk-h - - vtk-m - - warpx - - wrf - - wxwidgets - runner-attributes: - tags: [ "spack", "large", "x86_64_v4" ] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - - match: ['os=amzn2'] - runner-attributes: - tags: ["spack", "x86_64_v4"] - variables: - CI_JOB_SIZE: "default" - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } - tags: ["spack", "public", "x86_64_v4"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + ci: + pipeline-gen: + - build-job: + image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } cdash: build-group: RADIUSS AWS Packages - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml index 4ba8ede3b4..020975618f 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml @@ -1,19 +1,5 @@ spack: view: false - - concretizer: - reuse: false - unify: false - - config: - build_jobs: 32 - concretizer: clingo - install_tree: - root: /home/software/radiuss - padded_length: 512 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - packages: all: target: [x86_64] @@ -62,145 +48,5 @@ spack: - [$radiuss] - [$compilers] - gitlab-ci: - image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf 'fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi - # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild - after_script: - - cat /proc/loadavg || true - - match_behavior: first - mappings: - - match: - - lbann - - openblas - - qt - - rust - - visit - runner-attributes: - tags: ["spack", "large", "x86_64"] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - - match: - - ascent - - axom - - conduit - - hdf5 - - hydrogen - - mfem - - mvapich2 - - py-packaging - - py-scipy - - samrai - - vtk-h - - vtk-m - - vtk - runner-attributes: - tags: ["spack", "medium", "x86_64"] - variables: - CI_JOB_SIZE: "medium" - KUBERNETES_CPU_REQUEST: "2000m" - KUBERNETES_MEMORY_REQUEST: "4G" - - - match: - - autoconf-archive - - camp - - cmake - - curl - - czmq - - diffutils - - gawk - - gdbm - - gettext - - glib - - gmake - - libgcrypt - - libpciaccess - - libpng - - libsigsegv - - libsodium - - libxml2 - - libyaml - - libzmq - - lua - - lz4 - - m4 - - meson - - metis - - ninja - - pcre - - pdsh - - perl - - pkgconf - - py-cffi - - py-jsonschema - - py-kiwisolver - - py-pycparser - - py-setuptools-scm - - py-six - - py-wheel - - qhull - - readline - - sed - - slurm - - sqlite - - unzip - - util-linux-uuid - - util-macros - - zfp - - zlib - runner-attributes: - tags: ["spack", "small", "x86_64"] - variables: - CI_JOB_SIZE: "small" - KUBERNETES_CPU_REQUEST: "500m" - KUBERNETES_MEMORY_REQUEST: "500M" - - - match: ['os=ubuntu18.04'] - runner-attributes: - tags: ["spack", "x86_64"] - variables: - CI_JOB_SIZE: "default" - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } - tags: ["spack", "public", "x86_64"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache - cdash: build-group: RADIUSS - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml index 83f652057e..836a455b67 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml @@ -1,27 +1,15 @@ spack: - view: false - - concretizer: - reuse: false - unify: false - config: - build_jobs: 32 - install_tree: - root: /home/software/spack - padded_length: 512 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - # allow deprecated versions in concretizations + # required for zlib deprecated: true + view: false packages: all: target: [x86_64] tbb: - require: 'intel-tbb' - + require: intel-tbb definitions: - gcc_system_packages: - matrix: @@ -62,122 +50,12 @@ spack: - $gcc_spack_built_packages mirrors: - mirror: 's3://spack-binaries/develop/tutorial' - - gitlab-ci: - script: - - uname -a || true - - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - - nproc - - curl -Lfs 'https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz' -o gmake.tar.gz - - printf 'fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz' | sha256sum --check --strict --quiet - - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null - - . "./share/spack/setup-env.sh" - - spack --version - - spack arch - - spack compiler find - - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view . - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi - # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - - spack --color=always --backtrace ci rebuild - after_script: - - cat /proc/loadavg || true - - image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] } - match_behavior: first - mappings: - - match: - - cmake - - dyninst - - gcc - - mpich - - netlib-lapack - - trilinos - runner-attributes: - tags: ["spack", "large", "x86_64"] - variables: - CI_JOB_SIZE: large - KUBERNETES_CPU_REQUEST: 8000m - KUBERNETES_MEMORY_REQUEST: 12G - - - match: - - autoconf-archive - - boost - - hdf5 - - libtool - - libxml2 - - openblas - - openmpi - - py-beniget - - py-scipy - - slurm - runner-attributes: - tags: ["spack", "medium", "x86_64"] - variables: - CI_JOB_SIZE: "medium" - KUBERNETES_CPU_REQUEST: "2000m" - KUBERNETES_MEMORY_REQUEST: "4G" - - - match: - - automake - - bzip2 - - expat - - findutils - - gdbm - - json-c - - libedit - - libevent - - libfabric - - libffi - - libiconv - - libidn2 - - libmd - - libpciaccess - - lua - - meson - - pdsh - - pkgconf - - readline - - superlu - - tar - - util-linux-uuid - runner-attributes: - tags: ["spack", "small", "x86_64"] - variables: - CI_JOB_SIZE: "small" - KUBERNETES_CPU_REQUEST: "500m" - KUBERNETES_MEMORY_REQUEST: "500M" - - - match: ['@:'] - runner-attributes: - tags: ["spack", "x86_64"] - variables: - CI_JOB_SIZE: default - - broken-specs-url: "s3://spack-binaries/broken-specs" - - service-job-attributes: - image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] } - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - tags: ["spack", "public", "x86_64"] - - signing-job-attributes: - image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } - tags: ["spack", "aws"] - script: - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp - - /sign.sh - - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache - + mirror: s3://spack-binaries/develop/tutorial + ci: + pipeline-gen: + - build-job: + image: + name: ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02 + entrypoint: [''] cdash: build-group: Spack Tutorial - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure -- cgit v1.2.3-70-g09d2