From 6b509a95da0497926a27e963adf1a862c71244e2 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Thu, 18 Feb 2021 18:50:57 -0700 Subject: Pipelines: Move PR testing stacks (currently only E4S) into spack (#21714) --- share/spack/gitlab/cloud_e4s_pipelines.yml | 24 ---- .../gitlab/cloud_e4s_pipelines/.gitlab-ci.yml | 61 +++++++++ .../cloud_e4s_pipelines/stacks/e4s/spack.yaml | 151 +++++++++++++++++++++ 3 files changed, 212 insertions(+), 24 deletions(-) delete mode 100644 share/spack/gitlab/cloud_e4s_pipelines.yml create mode 100644 share/spack/gitlab/cloud_e4s_pipelines/.gitlab-ci.yml create mode 100644 share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s/spack.yaml (limited to 'share') diff --git a/share/spack/gitlab/cloud_e4s_pipelines.yml b/share/spack/gitlab/cloud_e4s_pipelines.yml deleted file mode 100644 index 67a74cf971..0000000000 --- a/share/spack/gitlab/cloud_e4s_pipelines.yml +++ /dev/null @@ -1,24 +0,0 @@ -pr_pipeline: - only: - - /^github\/pr[\d]+_.*$/ - variables: - SPACK_REF: ${CI_COMMIT_SHA} - SPACK_PR_BRANCH: ${CI_COMMIT_REF_NAME} - SPACK_IS_PR_PIPELINE: "True" - AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY} - trigger: - project: spack/e4s - strategy: depend - -develop_pipeline: - only: - - /^github\/develop$/ - variables: - SPACK_REF: ${CI_COMMIT_SHA} - AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} - SPACK_SIGNING_KEY: ${SPACK_SIGNING_KEY} - trigger: - project: spack/e4s - strategy: depend diff --git a/share/spack/gitlab/cloud_e4s_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_e4s_pipelines/.gitlab-ci.yml new file mode 100644 index 0000000000..6f26eaf574 --- /dev/null +++ b/share/spack/gitlab/cloud_e4s_pipelines/.gitlab-ci.yml @@ -0,0 +1,61 @@ +stages: [generate, build] + +.generate: + stage: generate + script: + - . "./share/spack/setup-env.sh" + - spack --version + - cd share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s + - spack env activate --without-view . + - spack -d ci generate --check-index-only + --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/e4s_pipeline.yml" + artifacts: + paths: + - "${CI_PROJECT_DIR}/jobs_scratch_dir/e4s_pipeline.yml" + tags: ["spack", "public", "medium"] + +e4s-pr-generate: + only: + - /^github\/pr[\d]+_.*$/ + - move-pr-stack-into-spack + extends: .generate + before_script: + - export SPACK_PR_BRANCH=${CI_COMMIT_REF_NAME} + - export SPACK_IS_PR_PIPELINE="True" + image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } + +e4s-develop-generate: + only: + - /^github\/develop$/ + extends: .generate + image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } + +e4s-pr-build: + only: + - /^github\/pr[\d]+_.*$/ + - move-pr-stack-into-spack + stage: build + trigger: + include: + - artifact: "jobs_scratch_dir/e4s_pipeline.yml" + job: e4s-pr-generate + strategy: depend + variables: + AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY} + SPACK_PR_BRANCH: ${CI_COMMIT_REF_NAME} + SPACK_IS_PR_PIPELINE: "True" + +e4s-develop-build: + only: + - /^github\/develop$/ + stage: build + trigger: + include: + - artifact: "jobs_scratch_dir/e4s_pipeline.yml" + job: e4s-develop-generate + strategy: depend + variables: + AWS_ACCESS_KEY_ID: ${MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${MIRRORS_AWS_SECRET_ACCESS_KEY} + SPACK_SIGNING_KEY: ${PACKAGE_SIGNING_KEY} diff --git a/share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s/spack.yaml new file mode 100644 index 0000000000..fc5dad423b --- /dev/null +++ b/share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s/spack.yaml @@ -0,0 +1,151 @@ +spack: + view: false + concretization: separately + + config: + install_tree: + root: /home/software/spack + padded_length: 512 + projections: + all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + + packages: + all: + target: [x86_64] + providers: + blas: + - openblas + mpi: + - mpich + variants: +mpi + binutils: + variants: +gold+headers+libiberty~nls + version: + - 2.33.1 + cmake: + version: [3.18.4] + hdf5: + variants: +fortran + mpich: + variants: ~wrapperrpath + openblas: + version: [0.3.10] + slepc: + version: [3.14.0] + + definitions: + - e4s: + - adios + - adios2 + - aml + - amrex + - arborx + - argobots + - ascent + - axom + - bolt + - caliper + - darshan-runtime + - darshan-util + - dyninst + - faodel + - flecsi+cinch + - flit + - gasnet + - ginkgo + - globalarrays + - gotcha + - hdf5 + - hpctoolkit + - hpx + - hypre + - kokkos-kernels+openmp + - kokkos+openmp + - legion + - libnrm + - libquo + - magma cuda_arch=70 ^cuda@10.2.89 + - mercury + - mfem + - mpifileutils@develop~xattr + - ninja + - omega-h + - openmpi + - openpmd-api + - papi + - papyrus@1.0.1 + - parallel-netcdf + - pdt + - petsc + - phist + - plasma + - precice + - pumi + - py-jupyterhub + - py-libensemble + - py-petsc4py + - qthreads scheduler=distrib + - raja + - rempi + - scr + - slate ^openblas@0.3.5 threads=openmp ^cuda@10.2.89 + - slepc + - stc + - strumpack ~slate + - sundials + - superlu + - superlu-dist + - swig + - sz + - tasmanian + - tau + - trilinos + - turbine + - umap + - umpire + - unifyfs + - upcxx + - veloc + - zfp + - arch: + - '%gcc@7.5.0 arch=linux-ubuntu18.04-x86_64' + + specs: + - matrix: + - - $e4s + - - $arch + + mirrors: { "mirror": "s3://spack-binaries-develop/e4s-new-cluster" } + + gitlab-ci: + script: + - . "./share/spack/setup-env.sh" + - spack --version + - cd share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s + - spack env activate --without-view . + - spack -d ci rebuild + mappings: + - match: [cuda, dyninst, hpx, precice, strumpack, sundials, trilinos, vtk-h, vtk-m] + runner-attributes: + image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } + tags: ["spack", "public", "xlarge"] + - match: ['os=ubuntu18.04'] + runner-attributes: + image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } + tags: ["spack", "public", "large"] + temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage" + service-job-attributes: + before_script: + - . "./share/spack/setup-env.sh" + - spack --version + - cd share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s + - spack env activate --without-view . + image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } + tags: ["spack", "public", "medium"] + + cdash: + build-group: New PR testing workflow + url: https://cdash.spack.io + project: Spack Testing + site: Cloud Gitlab Infrastructure + -- cgit v1.2.3-60-g2f50