Added AWS-AHUG alinux2 pipeline (#24601)

Add spack stacks targeted at Spack + AWS + ARM HPC User Group hackathon.  Includes
a list of miniapps and full-apps that are ready to run on both x86_64 and aarch64.

Co-authored-by: Scott Wittenburg <scott.wittenburg@kitware.com>

3 files changed, 737 insertions, 0 deletions

diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
index f956ebed9e..45efff7c99 100644
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -434,6 +434,84 @@ data-vis-sdk-protected-build:
       job: data-vis-sdk-protected-generate
 
 ########################################
+# AWS AHUG Applications (x86_64)
+########################################
+
+# Call this AFTER .*-generate
+.aws-ahug-overrides:
+  # This controls image for generate step; build step is controlled by spack.yaml
+  # Note that generator emits OS info for build so these should be the same.
+  image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] }
+
+.aws-ahug:
+  variables:
+    SPACK_CI_STACK_NAME: aws-ahug
+
+aws-ahug-pr-generate:
+  extends: [ ".aws-ahug", ".pr-generate", ".aws-ahug-overrides" ]
+  tags: ["spack", "public", "medium", "x86_64_v4"]
+
+aws-ahug-protected-generate:
+  extends: [ ".aws-ahug", ".protected-generate", ".aws-ahug-overrides" ]
+  tags: ["spack", "public", "medium", "x86_64_v4"]
+
+aws-ahug-pr-build:
+  extends: [ ".aws-ahug", ".pr-build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: aws-ahug-pr-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: aws-ahug-pr-generate
+
+aws-ahug-protected-build:
+  extends: [ ".aws-ahug", ".protected-build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: aws-ahug-protected-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: aws-ahug-protected-generate
+
+
+# Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64)
+.aws-ahug-aarch64:
+  variables:
+    SPACK_CI_STACK_NAME: aws-ahug-aarch64
+
+aws-ahug-aarch64-pr-generate:
+  extends: [ ".aws-ahug-aarch64", ".pr-generate-aarch64", ".aws-ahug-overrides" ]
+
+aws-ahug-aarch64-protected-generate:
+  extends: [ ".aws-ahug-aarch64", ".protected-generate-aarch64", ".aws-ahug-overrides" ]
+
+aws-ahug-aarch64-pr-build:
+  extends: [ ".aws-ahug-aarch64", ".pr-build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: aws-ahug-aarch64-pr-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: aws-ahug-aarch64-pr-generate
+
+aws-ahug-aarch64-protected-build:
+  extends: [ ".aws-ahug-aarch64", ".protected-build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: aws-ahug-aarch64-protected-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: aws-ahug-aarch64-protected-generate
+
+########################################
 # AWS ISC Applications (x86_64)
 ########################################
 
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml
new file mode 100644
index 0000000000..67ac499003
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml
@@ -0,0 +1,329 @@
+spack:
+  view: false
+
+  concretizer:
+    reuse: false
+    unify: false
+
+  config:
+    concretizer: clingo
+    install_tree:
+      root: /home/software/spack
+      padded_length: 384
+      projections:
+        all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
+
+  packages:
+    all:
+      providers:
+        blas:
+          - openblas
+        mkl:
+          - intel-oneapi-mkl
+        mpi:
+          - openmpi
+          - mpich
+      variants: +mpi
+    binutils:
+      variants: +ld +gold +headers +libiberty ~nls
+      version:
+        - 2.36.1
+    doxygen:
+      version:
+        - 1.8.20
+    elfutils:
+      variants: +bzip2 ~nls +xz
+    hdf5:
+      variants: +fortran +hl +shared
+    libfabric:
+      variants: fabrics=efa,tcp,udp,sockets,verbs,shm,mrail,rxd,rxm
+    libunwind:
+      variants: +pic +xz
+    #m4:
+    #  version:
+    #    - 1.4.18
+    mesa:
+      variants: ~llvm
+    mesa18:
+      variants: ~llvm
+    mpich:
+      #variants: ~wrapperrpath pmi=pmi netmod=ofi device=ch4
+      variants: ~wrapperrpath netmod=ofi device=ch4
+    #munge:
+    #  variants: localstatedir=/var
+    ncurses:
+      variants: +termlib
+    openblas:
+      variants: threads=openmp
+    openmpi:
+      #variants: +pmi +internal-hwloc fabrics=ofi schedulers=slurm
+      variants: fabrics=ofi
+    openturns:
+      version: [1.18]
+    #slurm:
+    #  variants: +pmix sysconfdir=/opt/slurm/etc
+    trilinos:
+      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+    xz:
+      variants: +pic
+
+  definitions:
+
+  - compiler_specs:
+    - gcc
+    #- nvhpc@22.1
+
+
+  # - compilers:
+  #   - '%gcc@7.5.0'
+  #   - '%arm@21.0.0.879'
+  #   - '%nvhpc@21.2'
+    # - 'arm@21.0.0.879'
+    #- when: arch.satisfies('os=ubuntu18.04')
+    #  compilers: ['gcc@7.5.0', $bootstrap-compilers]
+    #- when: arch.satisfies('os=amzn2')
+    #  compilers: ['gcc@7.3.1', $bootstrap-compilers]
+
+    # Note skipping spot since no spack package for it
+  - ahug_miniapps:
+    - cloverleaf
+  # - coevp
+  # Bad code pushed to github; needs specific versioning
+  # - cohmm
+  # - examinimd
+  #  - exampm
+  # depends on openblas version conflicting on gcc@7.3.1
+  # - exasp2
+  # depends on openblas version conflicting on gcc@7.3.1
+  # - gamess-ri-mp2-miniapp
+    - hpcg
+  # depends on openblas version conflicting on gcc@7.3.1
+  # - laghos
+    - lulesh
+  # - miniaero
+    - miniamr
+    - minife
+    - minighost
+  # fails due to x86 specific timer (asm instructions)
+  # - minigmg
+    - minimd
+  # depends on openblas version conflicting on gcc@7.3.1
+  # - miniqmc
+    - minismac2d
+    - minitri
+    - minivite
+    - minixyce
+    - pennant
+    - picsarlite
+    - quicksilver
+  # - remhos
+    - rsbench
+    - simplemoc
+    - snap
+    - snappy
+    - tealeaf
+  # depends on openblas version conflicting on gcc@7.3.1
+  # - thornado-mini
+    - tycho2
+    - xsbench
+
+  - ahug_fullapps:
+  # depends on openblas version conflicting on gcc@7.3.1
+  # - abinit
+    - abyss
+  # conflicts on trilinos
+  # - albany
+  # - amber
+    - amg2013
+  # Bad variant fftw
+  # - aoflagger
+  # - athena
+  # - bowtie2
+    - branson
+  # - camx
+  # Bad variant gpu
+  # - candle-benchmarks
+    - cbench
+    - cgm
+    - chatterbug
+  # - cistem
+    - comd
+  # old version of openmpi
+  # - converge
+  # bad variant tensor ops mpi
+  # - cosmoflow-benchmark
+  # - cosmomc
+    - cosp2
+  # libxsmm not avail on arm
+  # - cp2k
+  # - dock
+  # - elk
+    - elmerfem
+  # - exabayes
+  # - examl
+  # - flecsph
+  # trilinos variant mumps
+  # - frontistr
+    - gatk
+    - graph500
+    - hpgmg
+    - lammps
+    - latte
+    - macsio
+  # - meep
+    - meme
+  # - modylas
+  # - mrbayes
+  # - mrchem
+ # cudnn depednency
+ #  - mxnet
+ # trilinos variant mumps
+ #  - nalu
+  # - nalu-wind
+  # - namd
+    - nek5000
+    - nekbone
+  # - nektar
+  # - nest
+    - nut
+  # - nwchem
+    - octopus
+    - openmm
+    - pathfinder
+  # - picsar
+    - pism
+  # meson version
+  # - qbox
+    - qmcpack
+    - quantum-espresso
+  # - relion
+  # - siesta
+    - snbone
+    - star
+    - su2
+    - swfft
+    - tinker
+  # gfortran lt 9 unsupported
+  # - vasp
+    - vpfft
+    - vpic
+    - warpx
+  # - yambo
+
+  - compiler:
+    - '%gcc@7.3.1'
+
+  - target:
+    - 'target=aarch64'
+    - 'target=graviton2'
+
+
+  specs:
+
+  - matrix:
+    - - $ahug_miniapps
+    - - $compiler
+    - - $target
+
+  - matrix:
+    - - $ahug_fullapps
+    - - $compiler
+    - - $target
+
+  # Build compilers to stage in binary cache
+  - matrix:
+    - - $compiler_specs
+    - - $compiler
+    - - $target
+
+  mirrors: { "mirror": "s3://spack-binaries/develop/aws-ahug-aarch64" }
+
+  gitlab-ci:
+
+    script:
+      - . "./share/spack/setup-env.sh"
+      - spack --version
+      - cd ${SPACK_CONCRETE_ENV_DIR}
+      - spack env activate --without-view .
+      - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
+      - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
+      - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
+      - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
+      - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+
+    image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] }
+    mappings:
+      - match:
+        - llvm
+        - llvm-amdgpu
+        - paraview
+        runner-attributes:
+          tags: [ "spack", "huge", "aarch64" ]
+          variables:
+            CI_JOB_SIZE: huge
+            KUBERNETES_CPU_REQUEST: 11000m
+            KUBERNETES_MEMORY_REQUEST: 42G
+
+
+      - match:
+        - ascent
+        - axom
+        - cuda
+        - dyninst
+        - gcc
+        - ginkgo
+        - hpx
+        - kokkos-kernels
+        - kokkos-nvcc-wrapper
+        - magma
+        - mfem
+        - mpich
+        - openturns
+        - precice
+        - raja
+        - rocblas
+        - rocsolver
+        - rust
+        - slate
+        - strumpack
+        - sundials
+        - trilinos
+        - umpire
+        - vtk-h
+        - vtk-m
+        - warpx
+        runner-attributes:
+          tags: [ "spack", "large", "aarch64" ]
+          variables:
+            CI_JOB_SIZE: large
+            KUBERNETES_CPU_REQUEST: 8000m
+            KUBERNETES_MEMORY_REQUEST: 12G
+
+      - match: ['os=amzn2']
+        runner-attributes:
+          tags: ["spack", "aarch64"]
+          variables:
+            CI_JOB_SIZE: "default"
+
+    broken-specs-url: "s3://spack-binaries/broken-specs"
+
+    service-job-attributes:
+      before_script:
+        - . "./share/spack/setup-env.sh"
+        - spack --version
+      image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] }
+      tags: ["spack", "public", "aarch64"]
+
+    signing-job-attributes:
+      image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
+      tags: ["spack", "aws"]
+      script:
+        - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
+        - /sign.sh
+        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
+
+  cdash:
+    build-group: AHUG ARM HPC User Group
+    url: https://cdash.spack.io
+    project: Spack Testing
+    site: Cloud Gitlab Infrastructure
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml
new file mode 100644
index 0000000000..f81eb0383e
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml
@@ -0,0 +1,330 @@
+spack:
+  view: false
+
+  concretizer:
+    reuse: false
+    unify: false
+
+  config:
+    concretizer: clingo
+    install_tree:
+      root: /home/software/spack
+      padded_length: 384
+      projections:
+        all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
+
+  packages:
+    all:
+      providers:
+        blas:
+          - openblas
+        mkl:
+          - intel-oneapi-mkl
+        mpi:
+          - openmpi
+          - mpich
+      variants: +mpi
+    binutils:
+      variants: +ld +gold +headers +libiberty ~nls
+      version:
+        - 2.36.1
+    doxygen:
+      version:
+        - 1.8.20
+    elfutils:
+      variants: +bzip2 ~nls +xz
+    hdf5:
+      variants: +fortran +hl +shared
+    libfabric:
+      variants: fabrics=efa,tcp,udp,sockets,verbs,shm,mrail,rxd,rxm
+    libunwind:
+      variants: +pic +xz
+    #m4:
+    #  version:
+    #    - 1.4.18
+    mesa:
+      variants: ~llvm
+    mesa18:
+      variants: ~llvm
+    mpich:
+      #variants: ~wrapperrpath pmi=pmi netmod=ofi device=ch4
+      variants: ~wrapperrpath netmod=ofi device=ch4
+    #munge:
+    #  variants: localstatedir=/var
+    ncurses:
+      variants: +termlib
+    openblas:
+      variants: threads=openmp
+    openmpi:
+      #variants: +pmi +internal-hwloc fabrics=ofi schedulers=slurm
+      variants: fabrics=ofi
+    openturns:
+      version: [1.18]
+    #slurm:
+    #  variants: +pmix sysconfdir=/opt/slurm/etc
+    trilinos:
+      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+    xz:
+      variants: +pic
+
+  definitions:
+
+  - compiler_specs:
+    - gcc
+    #- nvhpc@22.1
+
+
+  # - compilers:
+  #   - '%gcc@7.5.0'
+  #   - '%arm@21.0.0.879'
+  #   - '%nvhpc@21.2'
+    # - 'arm@21.0.0.879'
+    #- when: arch.satisfies('os=ubuntu18.04')
+    #  compilers: ['gcc@7.5.0', $bootstrap-compilers]
+    #- when: arch.satisfies('os=amzn2')
+    #  compilers: ['gcc@7.3.1', $bootstrap-compilers]
+
+    # Note skipping spot since no spack package for it
+  - ahug_miniapps:
+    - cloverleaf
+  # - coevp
+  # Bad code pushed to github; needs specific versioning
+  # - cohmm
+  # - examinimd
+  #  - exampm
+  # depends on openblas version conflicting on gcc@7.3.1
+  # - exasp2
+  # depends on openblas version conflicting on gcc@7.3.1
+  # - gamess-ri-mp2-miniapp
+    - hpcg
+  # depends on openblas version conflicting on gcc@7.3.1
+  # - laghos
+    - lulesh
+  # - miniaero
+    - miniamr
+    - minife
+    - minighost
+  # fails due to x86 specific timer (asm instructions)
+  # - minigmg
+    - minimd
+  # depends on openblas version conflicting on gcc@7.3.1
+  # - miniqmc
+    - minismac2d
+    - minitri
+    - minivite
+    - minixyce
+    - pennant
+    - picsarlite
+    - quicksilver
+  # - remhos
+    - rsbench
+    - simplemoc
+    - snap
+    - snappy
+    - tealeaf
+  # depends on openblas version conflicting on gcc@7.3.1
+  # - thornado-mini
+    - tycho2
+    - xsbench
+
+  - ahug_fullapps:
+  # depends on openblas version conflicting on gcc@7.3.1
+  # - abinit
+    - abyss
+  # conflicts on trilinos
+  # - albany
+  # - amber
+    - amg2013
+  # Bad variant fftw
+  # - aoflagger
+  # - athena
+  # - bowtie2
+    - branson
+  # - camx
+  # Bad variant gpu
+  # - candle-benchmarks
+    - cbench
+    - cgm
+    - chatterbug
+  # - cistem
+    - comd
+  # old version of openmpi
+  # - converge
+  # bad variant tensor ops mpi
+  # - cosmoflow-benchmark
+  # - cosmomc
+    - cosp2
+  # libxsmm not avail on arm
+  # - cp2k
+  # - dock
+  # - elk
+    - elmerfem
+  # - exabayes
+  # - examl
+  # - flecsph
+  # trilinos variant mumps
+  # - frontistr
+    - gatk
+    - graph500
+    - hpgmg
+    - lammps
+    - latte
+    - macsio
+  # - meep
+    - meme
+  # - modylas
+  # - mrbayes
+  # - mrchem
+ # cudnn depednency
+ #  - mxnet
+ # trilinos variant mumps
+ #  - nalu
+  # - nalu-wind
+  # - namd
+    - nek5000
+    - nekbone
+  # - nektar
+  # - nest
+    - nut
+  # - nwchem
+    - octopus
+    - openmm
+    - pathfinder
+  # - picsar
+    - pism
+  # meson version
+  # - qbox
+    - qmcpack
+    - quantum-espresso
+  # - relion
+  # - siesta
+    - snbone
+    - star
+    - su2
+    - swfft
+    - tinker
+  # gfortran lt 9 unsupported
+  # - vasp
+    - vpfft
+    - vpic
+    - warpx
+  # - yambo
+
+  - compiler:
+    - '%gcc@7.3.1'
+
+  - target:
+    #- 'target=x86_64'
+    - 'target=x86_64_v3'
+    - 'target=x86_64_v4'
+
+
+  specs:
+
+  - matrix:
+    - - $ahug_miniapps
+    - - $compiler
+    - - $target
+
+  - matrix:
+    - - $ahug_fullapps
+    - - $compiler
+    - - $target
+
+  # Build compilers to stage in binary cache
+  - matrix:
+    - - $compiler_specs
+    - - $compiler
+    - - $target
+
+  mirrors: { "mirror": "s3://spack-binaries/develop/aws-ahug" }
+
+  gitlab-ci:
+
+    script:
+      - . "./share/spack/setup-env.sh"
+      - spack --version
+      - cd ${SPACK_CONCRETE_ENV_DIR}
+      - spack env activate --without-view .
+      - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
+      - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
+      - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
+      - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
+      - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+
+    image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] }
+    mappings:
+      - match:
+        - llvm
+        - llvm-amdgpu
+        - paraview
+        runner-attributes:
+          tags: [ "spack", "huge", "x86_64_v4" ]
+          variables:
+            CI_JOB_SIZE: huge
+            KUBERNETES_CPU_REQUEST: 11000m
+            KUBERNETES_MEMORY_REQUEST: 42G
+
+
+      - match:
+        - ascent
+        - axom
+        - cuda
+        - dyninst
+        - gcc
+        - ginkgo
+        - hpx
+        - kokkos-kernels
+        - kokkos-nvcc-wrapper
+        - magma
+        - mfem
+        - mpich
+        - openturns
+        - precice
+        - raja
+        - rocblas
+        - rocsolver
+        - rust
+        - slate
+        - strumpack
+        - sundials
+        - trilinos
+        - umpire
+        - vtk-h
+        - vtk-m
+        - warpx
+        runner-attributes:
+          tags: [ "spack", "large", "x86_64_v4" ]
+          variables:
+            CI_JOB_SIZE: large
+            KUBERNETES_CPU_REQUEST: 8000m
+            KUBERNETES_MEMORY_REQUEST: 12G
+
+      - match: ['os=amzn2']
+        runner-attributes:
+          tags: ["spack", "x86_64_v4"]
+          variables:
+            CI_JOB_SIZE: "default"
+
+    broken-specs-url: "s3://spack-binaries/broken-specs"
+
+    service-job-attributes:
+      before_script:
+        - . "./share/spack/setup-env.sh"
+        - spack --version
+      image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] }
+      tags: ["spack", "public", "x86_64_v4"]
+
+    signing-job-attributes:
+      image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
+      tags: ["spack", "aws"]
+      script:
+        - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
+        - /sign.sh
+        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
+
+  cdash:
+    build-group: AHUG ARM HPC User Group
+    url: https://cdash.spack.io
+    project: Spack Testing
+    site: Cloud Gitlab Infrastructure