From e3054c3318acd729e4900fd94c69b5bd5dfc5d92 Mon Sep 17 00:00:00 2001 From: Chuck Atkins Date: Mon, 26 Apr 2021 09:13:16 -0500 Subject: ci: Generalize the GitLab CI pipeline yaml (#23225) * ci: Generalize the GitLab CI pipeline yaml * ci: Rename cloud_e4s_pipelines to the more general cloud_pipelines --- .../gitlab/cloud_e4s_pipelines/.gitlab-ci.yml | 129 +++++++++++++----- .../cloud_e4s_pipelines/stacks/e4s/spack.yaml | 151 --------------------- share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml | 125 +++++++++++++++++ .../gitlab/cloud_pipelines/stacks/e4s/spack.yaml | 151 +++++++++++++++++++++ 4 files changed, 373 insertions(+), 183 deletions(-) delete mode 100644 share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s/spack.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml create mode 100644 share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml diff --git a/share/spack/gitlab/cloud_e4s_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_e4s_pipelines/.gitlab-ci.yml index c1ea9383d8..579d7b56f3 100644 --- a/share/spack/gitlab/cloud_e4s_pipelines/.gitlab-ci.yml +++ b/share/spack/gitlab/cloud_e4s_pipelines/.gitlab-ci.yml @@ -1,60 +1,125 @@ -stages: [generate, build] +stages: [ "generate", "build" ] + +default: + image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } + +######################################## +# Job templates +######################################## + +.pr: + only: + - /^github\/pr[\d]+_.*$/ + variables: + SPACK_PR_BRANCH: ${CI_COMMIT_REF_NAME} + SPACK_IS_PR_PIPELINE: "True" + +.develop: + only: + - /^github\/develop$/ + variables: + SPACK_IS_PR_PIPELINE: "False" .generate: stage: generate script: - . "./share/spack/setup-env.sh" - spack --version - - cd share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s + - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME} - spack env activate --without-view . - spack ci generate --check-index-only - --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/e4s_pipeline.yml" + --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml" artifacts: paths: - - "${CI_PROJECT_DIR}/jobs_scratch_dir/e4s_pipeline.yml" + - "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml" tags: ["spack", "public", "medium", "x86_64"] interruptible: true +.pr-generate: + extends: [ ".pr", ".generate" ] + +.develop-generate: + extends: [ ".develop", ".generate" ] + +.build: + stage: build + +.pr-build: + extends: [ ".pr", ".build" ] + variables: + AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY} + +.develop-build: + extends: [ ".develop", ".build" ] + variables: + AWS_ACCESS_KEY_ID: ${MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${MIRRORS_AWS_SECRET_ACCESS_KEY} + SPACK_SIGNING_KEY: ${PACKAGE_SIGNING_KEY} + +######################################## +# TEMPLATE FOR ADDING ANOTHER PIPELINE +######################################## +# +# First add a new spack.yml defining the pipline to run in +# share/spack/gitlab/cloud_pipelines/stacks/my-super-cool-stack/spack.yaml +# +# Then add the following entries at the bottom of this file: +# +######################################## +# My Super Cool Pipeline +######################################## +# .my-super-cool-stack: +# variables: +# SPACK_CI_STACK_NAME: my-super-cool-stack +# +# my-super-cool-stack-pr-generate: +# extends: [ ".my-super-cool-stack", ".pr-generate"] +# +# my-super-cool-stack-develop-generate: +# extends: [ ".my-super-cool-stack", ".develop-generate"] +# +# my-super-cool-stack-pr-build: +# extends: [ ".my-super-cool-stack", ".pr-build" ] +# trigger: +# include: +# - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml +# job: my-super-cool-stack-pr-generate +# strategy: depend +# +# my-super-cool-stack-develop-build: +# extends: [ ".my-super-cool-stack", ".develop-build" ] +# trigger: +# include: +# - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml +# job: my-super-cool-stack-develop-generate +# strategy: depend + +######################################## +# E4S pipeline +######################################## +.e4s: + variables: + SPACK_CI_STACK_NAME: e4s + e4s-pr-generate: - only: - - /^github\/pr[\d]+_.*$/ - extends: .generate - before_script: - - export SPACK_PR_BRANCH=${CI_COMMIT_REF_NAME} - - export SPACK_IS_PR_PIPELINE="True" - image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } + extends: [ ".e4s", ".pr-generate"] e4s-develop-generate: - only: - - /^github\/develop$/ - extends: .generate - image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } + extends: [ ".e4s", ".develop-generate"] e4s-pr-build: - only: - - /^github\/pr[\d]+_.*$/ - stage: build + extends: [ ".e4s", ".pr-build" ] trigger: include: - - artifact: "jobs_scratch_dir/e4s_pipeline.yml" + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml job: e4s-pr-generate strategy: depend - variables: - AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY} - SPACK_PR_BRANCH: ${CI_COMMIT_REF_NAME} - SPACK_IS_PR_PIPELINE: "True" e4s-develop-build: - only: - - /^github\/develop$/ - stage: build + extends: [ ".e4s", ".develop-build" ] trigger: include: - - artifact: "jobs_scratch_dir/e4s_pipeline.yml" + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml job: e4s-develop-generate strategy: depend - variables: - AWS_ACCESS_KEY_ID: ${MIRRORS_AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY: ${MIRRORS_AWS_SECRET_ACCESS_KEY} - SPACK_SIGNING_KEY: ${PACKAGE_SIGNING_KEY} diff --git a/share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s/spack.yaml deleted file mode 100644 index bd004722c1..0000000000 --- a/share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s/spack.yaml +++ /dev/null @@ -1,151 +0,0 @@ -spack: - view: false - concretization: separately - - config: - install_tree: - root: /home/software/spack - padded_length: 512 - projections: - all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - - packages: - all: - target: [x86_64] - providers: - blas: - - openblas - mpi: - - mpich - variants: +mpi - binutils: - variants: +gold+headers+libiberty~nls - version: - - 2.33.1 - cmake: - version: [3.18.4] - hdf5: - variants: +fortran - mpich: - variants: ~wrapperrpath - openblas: - version: [0.3.10] - slepc: - version: [3.14.0] - - definitions: - - e4s: - # - adios - # - adios2 - # - aml - # - amrex - # - arborx - - argobots - # - ascent - # - axom - # - bolt - # - caliper - # - darshan-runtime - - darshan-util - # - dyninst - # - faodel - # - flecsi+cinch - # - flit - # - gasnet - # - ginkgo - # - globalarrays - # - gotcha - # - hdf5 - # - hpctoolkit - # - hpx - # - hypre - # - kokkos-kernels+openmp - # - kokkos+openmp - # - legion - # - libnrm - # - libquo - # - magma cuda_arch=70 ^cuda@10.2.89 - # - mercury - # - mfem - # - mpifileutils@develop~xattr - # - ninja - # - omega-h - # - openmpi - # - openpmd-api - # - papi - # - papyrus@1.0.1 - # - parallel-netcdf - # - pdt - # - petsc - # - phist - # - plasma - # - precice - # - pumi - # - py-jupyterhub - # - py-libensemble - # - py-petsc4py - # - qthreads scheduler=distrib - # - raja - # - rempi - # - scr - # - slate ^openblas@0.3.6 threads=openmp ^cuda@10.2.89 - # - slepc - # - stc - # - strumpack ~slate ^openblas@0.3.6 threads=openmp - # - sundials - # - superlu - # - superlu-dist - - swig - # - sz - # - tasmanian - # - tau - # - trilinos - # - turbine - # - umap - # - umpire - # - unifyfs - # - upcxx - # - veloc - # - zfp - - arch: - - '%gcc@7.5.0 arch=linux-ubuntu18.04-x86_64' - - specs: - - matrix: - - - $e4s - - - $arch - - mirrors: { "mirror": "s3://spack-binaries-develop/e4s-new-cluster" } - - gitlab-ci: - script: - - . "./share/spack/setup-env.sh" - - spack --version - - cd share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s - - spack env activate --without-view . - - spack -d ci rebuild - mappings: - - match: [cuda, dyninst, hpx, precice, strumpack, sundials, trilinos, vtk-h, vtk-m] - runner-attributes: - image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } - tags: ["spack", "public", "xlarge", "x86_64"] - - match: ['os=ubuntu18.04'] - runner-attributes: - image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } - tags: ["spack", "public", "large", "x86_64"] - temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage" - service-job-attributes: - before_script: - - . "./share/spack/setup-env.sh" - - spack --version - - cd share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s - - spack env activate --without-view . - image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } - tags: ["spack", "public", "medium", "x86_64"] - - cdash: - build-group: New PR testing workflow - url: https://cdash.spack.io - project: Spack Testing - site: Cloud Gitlab Infrastructure - diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml new file mode 100644 index 0000000000..579d7b56f3 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml @@ -0,0 +1,125 @@ +stages: [ "generate", "build" ] + +default: + image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } + +######################################## +# Job templates +######################################## + +.pr: + only: + - /^github\/pr[\d]+_.*$/ + variables: + SPACK_PR_BRANCH: ${CI_COMMIT_REF_NAME} + SPACK_IS_PR_PIPELINE: "True" + +.develop: + only: + - /^github\/develop$/ + variables: + SPACK_IS_PR_PIPELINE: "False" + +.generate: + stage: generate + script: + - . "./share/spack/setup-env.sh" + - spack --version + - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME} + - spack env activate --without-view . + - spack ci generate --check-index-only + --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml" + artifacts: + paths: + - "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml" + tags: ["spack", "public", "medium", "x86_64"] + interruptible: true + +.pr-generate: + extends: [ ".pr", ".generate" ] + +.develop-generate: + extends: [ ".develop", ".generate" ] + +.build: + stage: build + +.pr-build: + extends: [ ".pr", ".build" ] + variables: + AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY} + +.develop-build: + extends: [ ".develop", ".build" ] + variables: + AWS_ACCESS_KEY_ID: ${MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${MIRRORS_AWS_SECRET_ACCESS_KEY} + SPACK_SIGNING_KEY: ${PACKAGE_SIGNING_KEY} + +######################################## +# TEMPLATE FOR ADDING ANOTHER PIPELINE +######################################## +# +# First add a new spack.yml defining the pipline to run in +# share/spack/gitlab/cloud_pipelines/stacks/my-super-cool-stack/spack.yaml +# +# Then add the following entries at the bottom of this file: +# +######################################## +# My Super Cool Pipeline +######################################## +# .my-super-cool-stack: +# variables: +# SPACK_CI_STACK_NAME: my-super-cool-stack +# +# my-super-cool-stack-pr-generate: +# extends: [ ".my-super-cool-stack", ".pr-generate"] +# +# my-super-cool-stack-develop-generate: +# extends: [ ".my-super-cool-stack", ".develop-generate"] +# +# my-super-cool-stack-pr-build: +# extends: [ ".my-super-cool-stack", ".pr-build" ] +# trigger: +# include: +# - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml +# job: my-super-cool-stack-pr-generate +# strategy: depend +# +# my-super-cool-stack-develop-build: +# extends: [ ".my-super-cool-stack", ".develop-build" ] +# trigger: +# include: +# - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml +# job: my-super-cool-stack-develop-generate +# strategy: depend + +######################################## +# E4S pipeline +######################################## +.e4s: + variables: + SPACK_CI_STACK_NAME: e4s + +e4s-pr-generate: + extends: [ ".e4s", ".pr-generate"] + +e4s-develop-generate: + extends: [ ".e4s", ".develop-generate"] + +e4s-pr-build: + extends: [ ".e4s", ".pr-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: e4s-pr-generate + strategy: depend + +e4s-develop-build: + extends: [ ".e4s", ".develop-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: e4s-develop-generate + strategy: depend diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml new file mode 100644 index 0000000000..f187b2c6b3 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml @@ -0,0 +1,151 @@ +spack: + view: false + concretization: separately + + config: + install_tree: + root: /home/software/spack + padded_length: 512 + projections: + all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + + packages: + all: + target: [x86_64] + providers: + blas: + - openblas + mpi: + - mpich + variants: +mpi + binutils: + variants: +gold+headers+libiberty~nls + version: + - 2.33.1 + cmake: + version: [3.18.4] + hdf5: + variants: +fortran + mpich: + variants: ~wrapperrpath + openblas: + version: [0.3.10] + slepc: + version: [3.14.0] + + definitions: + - e4s: + # - adios + # - adios2 + # - aml + # - amrex + # - arborx + - argobots + # - ascent + # - axom + # - bolt + # - caliper + # - darshan-runtime + - darshan-util + # - dyninst + # - faodel + # - flecsi+cinch + # - flit + # - gasnet + # - ginkgo + # - globalarrays + # - gotcha + # - hdf5 + # - hpctoolkit + # - hpx + # - hypre + # - kokkos-kernels+openmp + # - kokkos+openmp + # - legion + # - libnrm + # - libquo + # - magma cuda_arch=70 ^cuda@10.2.89 + # - mercury + # - mfem + # - mpifileutils@develop~xattr + # - ninja + # - omega-h + # - openmpi + # - openpmd-api + # - papi + # - papyrus@1.0.1 + # - parallel-netcdf + # - pdt + # - petsc + # - phist + # - plasma + # - precice + # - pumi + # - py-jupyterhub + # - py-libensemble + # - py-petsc4py + # - qthreads scheduler=distrib + # - raja + # - rempi + # - scr + # - slate ^openblas@0.3.6 threads=openmp ^cuda@10.2.89 + # - slepc + # - stc + # - strumpack ~slate ^openblas@0.3.6 threads=openmp + # - sundials + # - superlu + # - superlu-dist + - swig + # - sz + # - tasmanian + # - tau + # - trilinos + # - turbine + # - umap + # - umpire + # - unifyfs + # - upcxx + # - veloc + # - zfp + - arch: + - '%gcc@7.5.0 arch=linux-ubuntu18.04-x86_64' + + specs: + - matrix: + - - $e4s + - - $arch + + mirrors: { "mirror": "s3://spack-binaries-develop/e4s-new-cluster" } + + gitlab-ci: + script: + - . "./share/spack/setup-env.sh" + - spack --version + - cd share/spack/gitlab/cloud_pipelines/stacks/e4s + - spack env activate --without-view . + - spack -d ci rebuild + mappings: + - match: [cuda, dyninst, hpx, precice, strumpack, sundials, trilinos, vtk-h, vtk-m] + runner-attributes: + image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } + tags: ["spack", "public", "xlarge", "x86_64"] + - match: ['os=ubuntu18.04'] + runner-attributes: + image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } + tags: ["spack", "public", "large", "x86_64"] + temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage" + service-job-attributes: + before_script: + - . "./share/spack/setup-env.sh" + - spack --version + - cd share/spack/gitlab/cloud_pipelines/stacks/e4s + - spack env activate --without-view . + image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } + tags: ["spack", "public", "medium", "x86_64"] + + cdash: + build-group: New PR testing workflow + url: https://cdash.spack.io + project: Spack Testing + site: Cloud Gitlab Infrastructure + -- cgit v1.2.3-70-g09d2