summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristopher Kotfila <chris.kotfila@kitware.com>2022-04-19 12:09:32 -0400
committerGitHub <noreply@github.com>2022-04-19 10:09:32 -0600
commit4067a281828516d9a8a1c6308ccf6617e520c57e (patch)
tree5029a6eeddef8d6fd04cb8d95d8455f13d42778b
parentf0ceeac0a8a1a8630c65ec6b1f090b30da719a1f (diff)
downloadspack-4067a281828516d9a8a1c6308ccf6617e520c57e.tar.gz
spack-4067a281828516d9a8a1c6308ccf6617e520c57e.tar.bz2
spack-4067a281828516d9a8a1c6308ccf6617e520c57e.tar.xz
spack-4067a281828516d9a8a1c6308ccf6617e520c57e.zip
Set resource requests on package builds (#29922)
gitlab ci: Set resource requests explicitly This PR sets resource requests for the Kubernetes executor, which should aid in better workload scheduling in the cluster. The specific values were derived from profile data taken from several full "from scratch" rebuilds in a separate worker pool. Co-authored-by: Zack Galbreath <zack.galbreath@kitware.com>
-rw-r--r--share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml62
-rw-r--r--share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml92
-rw-r--r--share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml187
-rw-r--r--share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml94
-rw-r--r--share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml76
5 files changed, 465 insertions, 46 deletions
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
index 3aea593747..767007132f 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
@@ -37,11 +37,64 @@ spack:
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- spack -d ci rebuild
+ image:
+ name: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
+ entrypoint: [ "" ]
+
mappings:
- - match: [ 'os=ubuntu18.04' ]
+ - match:
+ - cmake
+ runner-attributes:
+ tags: [ "spack", "public", "large", "x86_64"]
+ variables:
+ CI_JOB_SIZE: large
+ KUBERNETES_CPU_REQUEST: 8000m
+ KUBERNETES_MEMORY_REQUEST: 12G
+
+ - match:
+ - curl
+ - gettext
+ - mpich
+ - openjpeg
+ - sqlite
+ runner-attributes:
+ tags: [ "spack", "public", "medium", "x86_64" ]
+ variables:
+ CI_JOB_SIZE: "medium"
+ KUBERNETES_CPU_REQUEST: "2000m"
+ KUBERNETES_MEMORY_REQUEST: "4G"
+
+ - match:
+ - bzip2
+ - diffutils
+ - findutils
+ - libffi
+ - libidn2
+ - libmd
+ - libsigsegv
+ - libxml2
+ - lz4
+ - openssl
+ - pkgconf
+ - tut
+ - util-linux-uuid
+ - util-macros
+ - xz
+ - zlib
runner-attributes:
- image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [ "" ] }
- tags: [ "spack", "public", "large", "x86_64" ]
+ tags: [ "spack", "public", "medium", "x86_64" ]
+ variables:
+ CI_JOB_SIZE: "small"
+ KUBERNETES_CPU_REQUEST: "500m"
+ KUBERNETES_MEMORY_REQUEST: "500M"
+
+ - match:
+ - 'os=ubuntu18.04'
+ runner-attributes:
+ tags: ["spack", "public", "x86_64"]
+ variables:
+ CI_JOB_SIZE: "default"
+
temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage"
broken-specs-url: "s3://spack-binaries-develop/broken-specs"
@@ -50,8 +103,7 @@ spack:
- . "./share/spack/setup-env.sh"
- spack --version
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
- tags: ["spack", "public", "medium", "x86_64"]
-
+ tags: ["spack", "public", "x86_64"]
cdash:
build-group: Build tests for different build systems
url: https://cdash.spack.io
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
index d6b550ec99..a8dd1b67a3 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
@@ -51,15 +51,97 @@ spack:
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- spack -d ci rebuild
mappings:
- - match: [llvm]
+ - match:
+ - llvm
runner-attributes:
- tags: ["spack", "public", "huge", "x86_64"]
- - match: [vtk-h, vtk-m, paraview, vtk]
+ tags: [ "spack", "public", "huge", "x86_64" ]
+ variables:
+ CI_JOB_SIZE: huge
+ KUBERNETES_CPU_REQUEST: 11000m
+ KUBERNETES_MEMORY_REQUEST: 42G
+
+ - match:
+ - ecp-data-vis-sdk
+ - mesa
+ - openblas
+ - paraview
+ - vtk-m
+ runner-attributes:
+ tags: [ "spack", "public", "large", "x86_64" ]
+ variables:
+ CI_JOB_SIZE: large
+ KUBERNETES_CPU_REQUEST: 8000m
+ KUBERNETES_MEMORY_REQUEST: 12G
+
+ - match:
+ - adios2
+ - ascent
+ - binutils
+ - blt
+ - boost
+ - conduit
+ - double-conversion
+ - dray
+ - eigen
+ - faodel
+ - hdf5
+ - mfem
+ - nasm
+ - openmpi
+ - pegtl
+ - py-cinemasci
+ - raja
+ - vtk-h
+ runner-attributes:
+ tags: [ "spack", "public", "medium", "x86_64" ]
+ variables:
+ CI_JOB_SIZE: "medium"
+ KUBERNETES_CPU_REQUEST: "2000m"
+ KUBERNETES_MEMORY_REQUEST: "4G"
+
+ - match:
+ - darshan-util
+ - docbook-xml
+ - gdbm
+ - gettext
+ - hwloc
+ - libevent
+ - libmd
+ - libpciaccess
+ - libsigsegv
+ - libunwind
+ - libxml2
+ - libzmq
+ - numactl
+ - openssh
+ - pcre
+ - perl-data-dumper
+ - py-cycler
+ - py-decorator
+ - py-mistune
+ - py-pycparser
+ - py-setuptools
+ - py-wheel
+ - readline
+ - sqlite
+ - tar
+ - util-linux-uuid
+
runner-attributes:
- tags: ["spack", "public", "xlarge", "x86_64"]
+ tags: [ "spack", "public", "small", "x86_64" ]
+ variables:
+ CI_JOB_SIZE: "small"
+ KUBERNETES_CPU_REQUEST: "500m"
+ KUBERNETES_MEMORY_REQUEST: "500M"
+
+
- match: ['@:']
runner-attributes:
- tags: ["spack", "public", "large", "x86_64"]
+ tags: ["spack", "public", "x86_64"]
+ variables:
+ CI_JOB_SIZE: "default"
+
+
temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage"
broken-specs-url: "s3://spack-binaries-develop/broken-specs"
service-job-attributes:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
index 54f02bf940..9248754826 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
@@ -232,34 +232,177 @@ spack:
- mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
- spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+ image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
mappings:
- match:
- - llvm
- - llvm-amdgpu
+ - llvm
runner-attributes:
- image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
- tags: ["spack", "public", "huge", "x86_64"]
+ tags: [ "spack", "public", "huge", "x86_64" ]
+ variables:
+ CI_JOB_SIZE: huge
+ KUBERNETES_CPU_REQUEST: 11000m
+ KUBERNETES_MEMORY_REQUEST: 42G
+
+ - match:
+ - cuda
+ - dyninst
+ - ginkgo
+ - hpx
+ - kokkos-kernels
+ - kokkos-nvcc-wrapper
+ - magma
+ - mfem
+ - mpich
+ - openturns
+ - precice
+ - raja
+ - rust
+ - slate
+ - trilinos
+ - vtk-m
+ - warpx
+ runner-attributes:
+ tags: [ "spack", "public", "large", "x86_64" ]
+ variables:
+ CI_JOB_SIZE: large
+ KUBERNETES_CPU_REQUEST: 8000m
+ KUBERNETES_MEMORY_REQUEST: 12G
+
- match:
- - cuda
- - dyninst
- - hpx
- - kokkos-kernels
- - precice
- - rocblas
- - rocsolver
- - strumpack
- - sundials
- - trilinos
- - vtk-h
- - vtk-m
- - warpx
+ - adios2
+ - amrex
+ - archer
+ - ascent
+ - axom
+ - binutils
+ - blaspp
+ - boost
+ - butterflypack
+ - cabana
+ - caliper
+ - camp
+ - chai
+ - conduit
+ - datatransferkit
+ - faodel
+ - ffmpeg
+ - fftw
+ - fortrilinos
+ - gperftools
+ - gptune
+ - hdf5
+ - heffte
+ - hpctoolkit
+ - hwloc
+ - hypre
+ - kokkos
+ - lammps
+ - lapackpp
+ - legion
+ - libzmq
+ - llvm-openmp-ompt
+ - mbedtls
+ - netlib-scalapack
+ - omega-h
+ - openmpi
+ - openpmd-api
+ - pagmo2
+ - papyrus
+ - parsec
+ - pdt
+ - petsc
+ - pumi
+ - py-ipython-genutils
+ - py-petsc4py
+ - py-scipy
+ - py-statsmodels
+ - py-warlock
+ - py-warpx
+ - pygmo
+ - slepc
+ - slurm
+ - strumpack
+ - sundials
+ - superlu-dist
+ - tasmanian
+ - tau
+ - upcxx
+ - vtk-h
+ - zfp
runner-attributes:
- image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
- tags: ["spack", "public", "xlarge", "x86_64"]
+ tags: [ "spack", "public", "medium", "x86_64" ]
+ variables:
+ CI_JOB_SIZE: "medium"
+ KUBERNETES_CPU_REQUEST: "2000m"
+ KUBERNETES_MEMORY_REQUEST: "4G"
+
+ - match:
+ - alsa-lib
+ - ant
+ - antlr
+ - argobots
+ - automake
+ - berkeley-db
+ - bison
+ - blt
+ - cmake
+ - curl
+ - darshan-util
+ - diffutils
+ - exmcutils
+ - expat
+ - flit
+ - freetype
+ - gdbm
+ - gotcha
+ - hpcviewer
+ - jansson
+ - json-c
+ - libbsd
+ - libevent
+ - libjpeg-turbo
+ - libnrm
+ - libpng
+ - libunistring
+ - lua-luaposix
+ - m4
+ - mpfr
+ - ncurses
+ - openblas
+ - openjdk
+ - papi
+ - parallel-netcdf
+ - pcre2
+ - perl-data-dumper
+ - pkgconf
+ - py-alembic
+ - py-idna
+ - py-testpath
+ - qhull
+ - snappy
+ - swig
+ - tar
+ - tcl
+ - texinfo
+ - unzip
+ - util-linux-uuid
+ - util-macros
+ - yaml-cpp
+ - zlib
+ - zstd
+ runner-attributes:
+ tags: [ "spack", "public", "small", "x86_64" ]
+ variables:
+ CI_JOB_SIZE: "small"
+ KUBERNETES_CPU_REQUEST: "500m"
+ KUBERNETES_MEMORY_REQUEST: "500M"
+
- match: ['os=ubuntu18.04']
runner-attributes:
- image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
- tags: ["spack", "public", "large", "x86_64"]
+ tags: ["spack", "public", "x86_64"]
+ variables:
+ CI_JOB_SIZE: "default"
+
temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage"
broken-specs-url: "s3://spack-binaries-develop/broken-specs"
service-job-attributes:
@@ -267,7 +410,7 @@ spack:
- . "./share/spack/setup-env.sh"
- spack --version
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
- tags: ["spack", "public", "medium", "x86_64"]
+ tags: ["spack", "public", "x86_64"]
cdash:
build-group: New PR testing workflow
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
index e8ba238e06..d98999fa63 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
@@ -20,7 +20,7 @@ spack:
definitions:
#- compilers: ['%gcc@8.3.1', '%clang@10.0.0']
- compilers: ['%gcc@7.5.0']
-
+
# Note skipping spot since no spack package for it
- radiuss:
- ascent # ^conduit@0.6.0
@@ -68,19 +68,103 @@ spack:
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- spack -d ci rebuild
mappings:
- - match: [ascent, axom, sundials, umpire, vtk-h, vtk-m]
+ - match:
+ - lbann
+ - openblas
+ - rust
+ runner-attributes:
+ tags: ["spack", "public", "large", "x86_64"]
+ variables:
+ CI_JOB_SIZE: large
+ KUBERNETES_CPU_REQUEST: 8000m
+ KUBERNETES_MEMORY_REQUEST: 12G
+
+ - match:
+ - ascent
+ - axom
+ - conduit
+ - hdf5
+ - hydrogen
+ - mfem
+ - mvapich2
+ - py-packaging
+ - py-scipy
+ - samrai
+ - vtk-h
+ - vtk-m
runner-attributes:
- tags: ["spack", "public", "xlarge", "x86_64"]
+ tags: ["spack", "public", "medium", "x86_64"]
+ variables:
+ CI_JOB_SIZE: "medium"
+ KUBERNETES_CPU_REQUEST: "2000m"
+ KUBERNETES_MEMORY_REQUEST: "4G"
+
+ - match:
+ - autoconf-archive
+ - camp
+ - cmake
+ - curl
+ - czmq
+ - diffutils
+ - gawk
+ - gdbm
+ - gettext
+ - glib
+ - gmake
+ - libgcrypt
+ - libpciaccess
+ - libpng
+ - libsigsegv
+ - libsodium
+ - libxml2
+ - libyaml
+ - libzmq
+ - lua
+ - lz4
+ - m4
+ - meson
+ - metis
+ - ninja
+ - pcre
+ - pdsh
+ - perl
+ - pkgconf
+ - py-cffi
+ - py-jsonschema
+ - py-kiwisolver
+ - py-pycparser
+ - py-setuptools-scm
+ - py-six
+ - py-wheel
+ - qhull
+ - readline
+ - sed
+ - slurm
+ - sqlite
+ - unzip
+ - util-linux-uuid
+ - util-macros
+ - zfp
+ - zlib
+ runner-attributes:
+ tags: ["spack", "public", "small", "x86_64"]
+ variables:
+ CI_JOB_SIZE: "small"
+ KUBERNETES_CPU_REQUEST: "500m"
+ KUBERNETES_MEMORY_REQUEST: "500M"
+
- match: ['os=ubuntu18.04']
runner-attributes:
- tags: ["spack", "public", "large", "x86_64"]
+ tags: ["spack", "public", "x86_64"]
+ variables:
+ CI_JOB_SIZE: "default"
temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage"
service-job-attributes:
before_script:
- . "./share/spack/setup-env.sh"
- spack --version
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
- tags: ["spack", "public", "medium", "x86_64"]
+ tags: ["spack", "public", "x86_64"]
cdash:
build-group: RADIUSS
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
index c07dca5010..40875cf31f 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
@@ -64,19 +64,76 @@ spack:
- spack env activate --without-view .
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- spack -d ci rebuild
+
+ image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] }
mappings:
- - match: [llvm]
+ - match:
+ - cmake
+ - dyninst
+ - gcc
+ - mpich
+ - netlib-lapack
+ - trilinos
+ runner-attributes:
+ tags: ["spack", "public", "large", "x86_64"]
+ variables:
+ CI_JOB_SIZE: large
+ KUBERNETES_CPU_REQUEST: 8000m
+ KUBERNETES_MEMORY_REQUEST: 12G
+
+ - match:
+ - autoconf-archive
+ - boost
+ - hdf5
+ - libtool
+ - libxml2
+ - openblas
+ - openmpi
+ - py-beniget
+ - py-scipy
+ - slurm
runner-attributes:
- image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] }
- tags: ["spack", "public", "huge", "x86_64"]
- - match: [trilinos, gcc]
+ tags: ["spack", "public", "medium", "x86_64"]
+ variables:
+ CI_JOB_SIZE: "medium"
+ KUBERNETES_CPU_REQUEST: "2000m"
+ KUBERNETES_MEMORY_REQUEST: "4G"
+
+ - match:
+ - automake
+ - bzip2
+ - expat
+ - findutils
+ - gdbm
+ - json-c
+ - libedit
+ - libevent
+ - libfabric
+ - libffi
+ - libiconv
+ - libidn2
+ - libmd
+ - libpciaccess
+ - lua
+ - meson
+ - pdsh
+ - pkgconf
+ - readline
+ - superlu
+ - tar
+ - util-linux-uuid
runner-attributes:
- image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] }
- tags: ["spack", "public", "xlarge", "x86_64"]
+ tags: ["spack", "public", "small", "x86_64"]
+ variables:
+ CI_JOB_SIZE: "small"
+ KUBERNETES_CPU_REQUEST: "500m"
+ KUBERNETES_MEMORY_REQUEST: "500M"
+
- match: ['@:']
runner-attributes:
- image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] }
- tags: ["spack", "public", "large", "x86_64"]
+ tags: ["spack", "public", "x86_64"]
+ variables:
+ CI_JOB_SIZE: default
temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage"
broken-specs-url: "s3://spack-binaries-develop/broken-specs"
service-job-attributes:
@@ -84,7 +141,8 @@ spack:
before_script:
- . "./share/spack/setup-env.sh"
- spack --version
- tags: ["spack", "public", "medium", "x86_64"]
+ tags: ["spack", "public", "x86_64"]
+
cdash:
build-group: Spack Tutorial