From 98860c6a5fdee5c2ac2b71b20af0bc8eecf3a627 Mon Sep 17 00:00:00 2001 From: Evan Bollig Date: Sat, 28 May 2022 19:32:53 +0200 Subject: Alinux isc buildcache (#30462) Add two new stacks targeted at x86_64 and arm, representing an initial list of packages used by current and planned AWS Workshops, and built in conjunction with the ISC22 announcement of the spack public binary cache. Co-authored-by: Scott Wittenburg --- share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml | 90 +++++++ .../stacks/aws-isc-aarch64/spack.yaml | 256 ++++++++++++++++++++ .../cloud_pipelines/stacks/aws-isc/spack.yaml | 258 +++++++++++++++++++++ 3 files changed, 604 insertions(+) create mode 100644 share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml (limited to 'share') diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml index b0936dfc4e..f956ebed9e 100644 --- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml +++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml @@ -52,12 +52,22 @@ default: - runner_system_failure - stuck_or_timeout_failure +.generate-aarch64: + extends: [ ".generate" ] + tags: ["spack", "aws", "public", "medium", "aarch64"] + .pr-generate: extends: [ ".pr", ".generate" ] +.pr-generate-aarch64: + extends: [ ".pr", ".generate-aarch64" ] + .protected-generate: extends: [ ".protected", ".generate" ] +.protected-generate-aarch64: + extends: [ ".protected", ".generate-aarch64" ] + .build: stage: build @@ -423,6 +433,86 @@ data-vis-sdk-protected-build: - artifacts: True job: data-vis-sdk-protected-generate +######################################## +# AWS ISC Applications (x86_64) +######################################## + +# Call this AFTER .*-generate +.aws-isc-overrides: + # This controls image for generate step; build step is controlled by spack.yaml + # Note that generator emits OS info for build so these should be the same. + image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + +.aws-isc: + variables: + SPACK_CI_STACK_NAME: aws-isc + +aws-isc-pr-generate: + extends: [ ".aws-isc", ".pr-generate", ".aws-isc-overrides" ] + tags: ["spack", "public", "medium", "x86_64_v4"] + +aws-isc-protected-generate: + extends: [ ".aws-isc", ".protected-generate", ".aws-isc-overrides" ] + tags: ["spack", "public", "medium", "x86_64_v4"] + +aws-isc-pr-build: + extends: [ ".aws-isc", ".pr-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: aws-isc-pr-generate + strategy: depend + needs: + - artifacts: True + job: aws-isc-pr-generate + +aws-isc-protected-build: + extends: [ ".aws-isc", ".protected-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: aws-isc-protected-generate + strategy: depend + needs: + - artifacts: True + job: aws-isc-protected-generate + + +# Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64) + +.aws-isc-aarch64: + variables: + SPACK_CI_STACK_NAME: aws-isc-aarch64 + +aws-isc-aarch64-pr-generate: + extends: [ ".aws-isc-aarch64", ".pr-generate-aarch64", ".aws-isc-overrides" ] + +aws-isc-aarch64-protected-generate: + extends: [ ".aws-isc-aarch64", ".protected-generate-aarch64", ".aws-isc-overrides" ] + +aws-isc-aarch64-pr-build: + extends: [ ".aws-isc-aarch64", ".pr-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: aws-isc-aarch64-pr-generate + strategy: depend + needs: + - artifacts: True + job: aws-isc-aarch64-pr-generate + +aws-isc-aarch64-protected-build: + extends: [ ".aws-isc-aarch64", ".protected-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: aws-isc-aarch64-protected-generate + strategy: depend + needs: + - artifacts: True + job: aws-isc-aarch64-protected-generate + + ######################################## # Spack Tutorial ######################################## diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml new file mode 100644 index 0000000000..4d76ffee72 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml @@ -0,0 +1,256 @@ +spack: + view: false + + concretizer: + reuse: false + unify: false + + config: + concretizer: clingo + install_tree: + root: /home/software/spack + padded_length: 384 + projections: + all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + + packages: + all: + providers: + blas: + - openblas + mkl: + - intel-oneapi-mkl + mpi: + - openmpi + - mpich + variants: +mpi + binutils: + variants: +ld +gold +headers +libiberty ~nls + version: + - 2.36.1 + doxygen: + version: + - 1.8.20 + elfutils: + variants: +bzip2 ~nls +xz + hdf5: + variants: +fortran +hl +shared + libfabric: + variants: fabrics=efa,tcp,udp,sockets,verbs,shm,mrail,rxd,rxm + libunwind: + variants: +pic +xz + mesa: + variants: ~llvm + mesa18: + variants: ~llvm + mpich: + variants: ~wrapperrpath netmod=ofi device=ch4 + ncurses: + variants: +termlib + openblas: + variants: threads=openmp + openmpi: + variants: fabrics=ofi +legacylaunchers + openturns: + version: [1.18] + relion: + variants: ~mklfft +# texlive: +# version: [20210325] + trilinos: + variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long + xz: + variants: +pic + + definitions: + + - compiler_specs: + - gcc@11.2 + # Licensing OK? + # - intel-oneapi-compilers@2022.1 + # - nvhpc + + - cuda_specs: + # Depends on ctffind which embeds fsincos (x86-specific asm) within code. Will not build on ARM + #- relion +cuda cuda_arch=70 + - raja +cuda cuda_arch=70 + - mfem +cuda cuda_arch=70 + + - app_specs: + - bwa +# Depends on simde which requires newer compiler? + #- bowtie2 + # Requires x86_64 specific ASM + #- cistem + - cromwell + - fastqc + - flux-sched + - flux-core + - flux-pmix + - gatk + - gromacs + - lammps + - wrf build_type=dm+sm + - mfem + - mpas-model ^parallelio+pnetcdf + - nextflow + - octave + - openfoam + - osu-micro-benchmarks + - parallel + - paraview + - picard + - quantum-espresso + - raja +# Depends on bowtie2 -> simde which requires newer compiler? + #- rsem + # Errors on texlive + #- rstudio + - salmon + - samtools + - seqtk + - snakemake + - star + # Requires gcc@9: + #- ufs-weather-model +# requires LLVM which fails without constraint + #- visit + + - lib_specs: + - openmpi fabrics=ofi + - openmpi fabrics=ofi +legacylaunchers + - openmpi fabrics=auto + - mpich + - libfabric + + - compiler: + - '%gcc@7.3.1' + + - target: + - 'target=aarch64' + - 'target=graviton2' + + + specs: + + - matrix: + - - $cuda_specs + - - $compiler + - - $target + + - matrix: + - - $app_specs + - - $compiler + - - $target + + - matrix: + - - $lib_specs + - - $compiler + - - $target + + - matrix: + - - $compiler_specs + - - $compiler + - - $target + + + mirrors: { "mirror": "s3://spack-binaries/develop/aws-isc-aarch64" } + + gitlab-ci: + + script: + - . "./share/spack/setup-env.sh" + - spack --version + - cd ${SPACK_CONCRETE_ENV_DIR} + - spack env activate --without-view . + - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data + - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi + - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + + image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + mappings: + - match: + - llvm + - llvm-amdgpu + - paraview + runner-attributes: + tags: [ "spack", "huge", "aarch64" ] + variables: + CI_JOB_SIZE: huge + KUBERNETES_CPU_REQUEST: 15000m + KUBERNETES_MEMORY_REQUEST: 62G + + + - match: + - ascent + - atk + - axom + - cistem + - ctffind + - cuda + - dyninst + - gcc + - ginkgo + - hdf5 + - hpx + - kokkos-kernels + - kokkos-nvcc-wrapper + - magma + - mfem + - mpich + - openturns + - parallelio + - precice + - raja + - relion + - rocblas + - rocsolver + - rust + - slate + - strumpack + - sundials + - trilinos + - umpire + - vtk + - vtk-h + - vtk-m + - warpx + - wrf + - wxwidgets + runner-attributes: + tags: [ "spack", "large", "aarch64" ] + variables: + CI_JOB_SIZE: large + KUBERNETES_CPU_REQUEST: 8000m + KUBERNETES_MEMORY_REQUEST: 12G + + - match: ['os=amzn2'] + runner-attributes: + tags: ["spack", "aarch64"] + variables: + CI_JOB_SIZE: "default" + + broken-specs-url: "s3://spack-binaries/broken-specs" + + service-job-attributes: + before_script: + - . "./share/spack/setup-env.sh" + - spack --version + image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + tags: ["spack", "public", "aarch64"] + + signing-job-attributes: + image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } + tags: ["spack", "aws"] + script: + - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp + - /sign.sh + - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + + cdash: + build-group: AWS Packages + url: https://cdash.spack.io + project: Spack Testing + site: Cloud Gitlab Infrastructure diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml new file mode 100644 index 0000000000..27de37b1f5 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml @@ -0,0 +1,258 @@ +spack: + view: false + + concretizer: + reuse: false + unify: false + + config: + concretizer: clingo + install_tree: + root: /home/software/spack + padded_length: 384 + projections: + all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + + packages: + all: + providers: + blas: + - openblas + mkl: + - intel-oneapi-mkl + mpi: + - openmpi + - mpich + variants: +mpi + binutils: + variants: +ld +gold +headers +libiberty ~nls + version: + - 2.36.1 + doxygen: + version: + - 1.8.20 + elfutils: + variants: +bzip2 ~nls +xz + hdf5: + variants: +fortran +hl +shared + libfabric: + variants: fabrics=efa,tcp,udp,sockets,verbs,shm,mrail,rxd,rxm + libunwind: + variants: +pic +xz + mesa: + variants: ~llvm + mesa18: + variants: ~llvm + mpich: + variants: ~wrapperrpath netmod=ofi device=ch4 + ncurses: + variants: +termlib + openblas: + variants: threads=openmp + openmpi: + variants: fabrics=ofi +legacylaunchers + openturns: + version: [1.18] + relion: + variants: ~mklfft +# texlive: +# version: [20210325] + trilinos: + variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long + xz: + variants: +pic + + definitions: + + - compiler_specs: + - gcc@11.2 + # Licensing OK? + # - intel-oneapi-compilers@2022.1 + # - nvhpc + + - cuda_specs: +# Disabled for consistency with aarch64 + #- relion +cuda cuda_arch=70 + - raja +cuda cuda_arch=70 + - mfem +cuda cuda_arch=70 + + - app_specs: + - bwa +# Disabled for consistency with aarch64 + #- bowtie2 +# Disabled for consistency with aarch64 + #- cistem + - cromwell + - fastqc + - flux-sched + - flux-core + - flux-pmix + - gatk + - gromacs + - lammps + - wrf build_type=dm+sm + - mfem + - mpas-model ^parallelio+pnetcdf + - nextflow + - octave + - openfoam + - osu-micro-benchmarks + - parallel + - paraview + - picard + - quantum-espresso +# Build broken for gcc@7.3.1 x86_64_v4 (error: '_mm512_loadu_epi32' was not declared in this scope) + #- raja +# Disabled for consistency with aarch64 + #- rsem + # Errors on texlive + #- rstudio + - salmon + - samtools + - seqtk + - snakemake + - star + # Requires gcc@9: + #- ufs-weather-model +# Disabled for consistency with aarch64 + #- visit + + - lib_specs: + - openmpi fabrics=ofi + - openmpi fabrics=ofi +legacylaunchers + - openmpi fabrics=auto + - mpich + - libfabric + + - compiler: + - '%gcc@7.3.1' + + - target: + - 'target=x86_64_v3' + - 'target=x86_64_v4' + + + specs: + + - matrix: + - - $cuda_specs + - - $compiler + - - $target + + - matrix: + - - $app_specs + - - $compiler + - - $target + + - matrix: + - - $lib_specs + - - $compiler + - - $target + + - matrix: + - - $compiler_specs + - - $compiler + - - $target + + + mirrors: { "mirror": "s3://spack-binaries/develop/aws-isc" } + + gitlab-ci: + + script: + - . "./share/spack/setup-env.sh" + - spack --version + - cd ${SPACK_CONCRETE_ENV_DIR} + - spack env activate --without-view . + - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data + - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi + - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + + image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + mappings: + - match: + - llvm + - llvm-amdgpu + - pango + - paraview + runner-attributes: + tags: [ "spack", "huge", "x86_64_v4" ] + variables: + CI_JOB_SIZE: huge + KUBERNETES_CPU_REQUEST: 11000m + KUBERNETES_MEMORY_REQUEST: 42G + + + - match: + - ascent + - atk + - axom + - cistem + - ctffind + - cuda + - dyninst + - gcc + - ginkgo + - hdf5 + - hpx + - kokkos-kernels + - kokkos-nvcc-wrapper + - magma + - mfem + - mpich + - openturns + - parallelio + - precice + - raja + - relion + - rocblas + - rocsolver + - rust + - slate + - strumpack + - sundials + - trilinos + - umpire + - vtk + - vtk-h + - vtk-m + - warpx + - wrf + - wxwidgets + runner-attributes: + tags: [ "spack", "large", "x86_64_v4" ] + variables: + CI_JOB_SIZE: large + KUBERNETES_CPU_REQUEST: 8000m + KUBERNETES_MEMORY_REQUEST: 12G + + - match: ['os=amzn2'] + runner-attributes: + tags: ["spack", "x86_64_v4"] + variables: + CI_JOB_SIZE: "default" + + broken-specs-url: "s3://spack-binaries/broken-specs" + + service-job-attributes: + before_script: + - . "./share/spack/setup-env.sh" + - spack --version + image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] } + tags: ["spack", "public", "x86_64_v4"] + + signing-job-attributes: + image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } + tags: ["spack", "aws"] + script: + - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp + - /sign.sh + - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + + cdash: + build-group: AWS Packages + url: https://cdash.spack.io + project: Spack Testing + site: Cloud Gitlab Infrastructure -- cgit v1.2.3-70-g09d2