Add applications to aws-pcluster-* stacks (#43901)

* Add openfoam to aws-pcluster-neoverse_v1 stack

* Add more apps to aws-pcluster-x86_64_v4 stack

* Remove WRF while hdf5 cannot build in buildcache at the moment

* Update comment

* Add more apps for aws-pcluster-neoverse_v1 stack

* Remove apps that currently do not build

* Disable those packages that won't build

* Modify syntax such that correct cflags are used

* Changing syntax again to what works with other packages

* Fix overriding packages.yaml entry for gettext

* Use new `prefer` and `require:when` clauses to clarify intent

* Use newer spack version to install intel compiler

This removes the need for patches and makes sure the `prefer` directives in
`package.yaml` are understood.

* `prefer` not strong enough, need to set compilers

* Revert "Use newer spack version to install intel compiler"

This reverts commit ecb25a192cc430ca946415108847d637b4000795.

Cannot update the spack version to install intel compiler as this changes the
compiler hash but not the version. This leads to incompatible compiler paths. If
we update this spack version in the future make sure the compiler version also updates.

Tested-by: Stephen Sachs <stesachs@amazon.com>

* Remove `prefer` clause as it is not strong enough for our needs

This way we can safely go back to installing the intel compiler with an older
spack version.

* Prefer gcc or oneapi to build gettext

* Pin gettext version compatible with system glibc-headers

* relax gettext version requirement to enable later versions

* oneapi cannot build older gettext version

5 files changed, 73 insertions, 38 deletions

diff --git a/share/spack/gitlab/cloud_pipelines/scripts/pcluster/setup-pcluster.sh b/share/spack/gitlab/cloud_pipelines/scripts/pcluster/setup-pcluster.sh
index dfd5af1b43..484be10f06 100755
--- a/share/spack/gitlab/cloud_pipelines/scripts/pcluster/setup-pcluster.sh
+++ b/share/spack/gitlab/cloud_pipelines/scripts/pcluster/setup-pcluster.sh
@@ -10,6 +10,7 @@ set -e
 # The best solution would be to have the compilers hash (or packages contents) be part of the
 # individual packages hashes. I don't see this at the moment.
 # Set to the latest tag including a recent oneapi compiler.
+# NOTE: If we update this spack version in the future make sure the compiler version also updates.
 spack_intel_compiler_commit="develop-2023-08-06"
 
 set_pcluster_defaults() {
@@ -23,10 +24,9 @@ set_pcluster_defaults() {
 
 setup_spack() {
     spack compiler add --scope site
-    spack external find --scope site
-    # Remove all autotools/buildtools packages. These versions need to be managed by spack or it will
+    # Do not add  autotools/buildtools packages. These versions need to be managed by spack or it will
     # eventually end up in a version mismatch (e.g. when compiling gmp).
-    spack tags build-tools | xargs -I {} spack config --scope site rm packages:{}
+    spack external find --scope site --tag core-packages
 }
 
 patch_compilers_yaml() {
@@ -99,7 +99,7 @@ install_compilers() {
     # The compilers needs to be in the same install tree as the rest of the software such that the path
     # relocation works correctly. This holds the danger that this part will fail when the current spack gets
     # incompatible with the one in $spack_intel_compiler_commit. Therefore, we make intel installations optional
-    # in package.yaml files.
+    # in package.yaml files and add a fallback `%gcc` version for each application.
     if [ "x86_64" == "$(arch)" ]; then
         (
             CURRENT_SPACK_ROOT=${SPACK_ROOT}
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/packages.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/packages.yaml
index dcb7eb80df..d896216fdf 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/packages.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/packages.yaml
@@ -19,7 +19,10 @@ packages:
   llvm:
     variants: ~lldb
   mpas-model:
-    require: "precision=single make_target=llvm %arm ^parallelio+pnetcdf"
+    require:
+      - one_of:
+          - "precision=single make_target=llvm %arm ^parallelio+pnetcdf"
+          - "precision=single %gcc ^parallelio+pnetcdf"
   mpich:
     require: "mpich pmi=pmi2 device=ch4 netmod=ofi +slurm"
   nvhpc:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
index 5e50716b26..2100189549 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
@@ -2,13 +2,23 @@ spack:
   view: false
 
   definitions:
-  - optimized_configs:
-    - gromacs target=neoverse_v1
-    - gromacs target=neoverse_n1
+  - apps:
+    - gromacs
+    # - mpas-model: Spack currently forces REAL(8) when using GCC. This conflicts with `precision=single`
+    # Fix proposed in https://github.com/spack/spack/pull/43547
+    - openfoam
+    # - quantum-espresso : %gcc@12.3.0 on neoverse_v1 fails.
+    # Root cause: internal compiler error: in compute_live_loop_exits, at tree-ssa-loop-manip.cc:247
+    - wrf
 
-  specs:
-  - $optimized_configs
+  - targets:
+      - 'target=neoverse_v1'
+      - 'target=neoverse_n1'
 
+  specs:
+  - matrix:
+      - [$apps]
+      - [$targets]
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/packages.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/packages.yaml
index 2684b27f4a..ca8cf09ef6 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/packages.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/packages.yaml
@@ -5,13 +5,20 @@ packages:
       - one_of:
           - "cflags=-std=c18 target=x86_64_v4"
           - "cflags=-std=c18 target=x86_64_v3"
-          - "%gcc"
+        when: "%intel"
+  gettext:
+    # Newer gettext cannot build with gcc@12 and old AL2 glibc headers
+    # Older gettext versions do not build correctly with oneapi.
+    require:
+      - one_of:
+          - '@:0.20'
+          - '%oneapi'
   gromacs:
     require:
       - one_of:
-          - "+intel_provided_gcc %intel ^intel-oneapi-mkl target=x86_64_v4"
-          - "+intel_provided_gcc %intel ^intel-oneapi-mkl target=x86_64_v3"
-          - "%gcc"
+          - "+intel_provided_gcc ^intel-oneapi-mkl target=x86_64_v4"
+          - "+intel_provided_gcc ^intel-oneapi-mkl target=x86_64_v3"
+        when: "%intel"
   intel-mpi:
     variants: +external-libfabric
   intel-oneapi-compilers:
@@ -21,15 +28,15 @@ packages:
   lammps:
     require:
       - one_of:
-          - "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package +intel %intel ^intel-oneapi-mkl target=x86_64_v4"
-          - "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package %intel ^intel-oneapi-mkl target=x86_64_v3"
-          - "%gcc"
+          - "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package +intel ^intel-oneapi-mkl target=x86_64_v4"
+          - "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package ^intel-oneapi-mkl target=x86_64_v3"
+        when: "%intel"
   libidn2:
     require:
       - one_of:
           - "cflags=-std=c18 target=x86_64_v4"
           - "cflags=-std=c18 target=x86_64_v3"
-          - '%gcc'
+        when: "%intel"
   libfabric:
     buildable: true
     externals:
@@ -41,13 +48,13 @@ packages:
       - one_of:
           - "cflags=-std=c18 target=x86_64_v4"
           - "cflags=-std=c18 target=x86_64_v3"
-          - "%gcc"
+        when: "%intel"
   mpas-model:
     require:
       - one_of:
-          - "precision=single %intel ^parallelio+pnetcdf target=x86_64_v4"
-          - "precision=single %intel ^parallelio+pnetcdf target=x86_64_v3"
-          - "%gcc"
+          - "precision=single ^parallelio+pnetcdf target=x86_64_v4"
+          - "precision=single ^parallelio+pnetcdf target=x86_64_v3"
+        when: "%intel"
   mpich:
     require:
       - one_of:
@@ -67,9 +74,12 @@ packages:
   palace:
     require:
       - one_of:
-          - "palace %oneapi ^fmt@9.1.0 target=x86_64_v4"
-          - "palace %oneapi ^fmt@9.1.0 target=x86_64_v3"
-          - "%gcc ^fmt@9.1.0"
+          - "palace ^fmt@9.1.0 target=x86_64_v4"
+          - "palace ^fmt@9.1.0 target=x86_64_v3"
+        when: "%oneapi"
+      - one_of:
+          - "palace ^fmt@9.1.0"
+        when: "%gcc"
   pmix:
     require:
       - one_of:
@@ -78,9 +88,9 @@ packages:
   quantum-espresso:
     require:
       - one_of:
-          - "quantum-espresso@6.6 %intel ^intel-oneapi-mkl+cluster target=x86_64_v4"
-          - "quantum-espresso@6.6 %intel ^intel-oneapi-mkl+cluster target=x86_64_v3"
-          - "%gcc"
+          - "quantum-espresso@6.6 ^intel-oneapi-mkl+cluster target=x86_64_v4"
+          - "quantum-espresso@6.6 ^intel-oneapi-mkl+cluster target=x86_64_v3"
+        when: "%intel"
   slurm:
     buildable: false
     externals:
@@ -89,12 +99,13 @@ packages:
   wrf:
     require:
       - one_of:
-          - "wrf@4 build_type=dm+sm %intel target=x86_64_v4"
-          - "wrf@4 build_type=dm+sm %intel target=x86_64_v3"
-          - "wrf@4.2.2 +netcdf_classic fflags=\"-fp-model fast=2 -no-heap-arrays -no-prec-div -no-prec-sqrt -fno-common\" build_type=dm+sm %intel  target=x86_64_v3"
-          - "%gcc"
+          - "wrf@4 build_type=dm+sm target=x86_64_v4"
+          - "wrf@4 build_type=dm+sm target=x86_64_v3"
+          - "wrf@4.2.2 +netcdf_classic fflags=\"-fp-model fast=2 -no-heap-arrays -no-prec-div -no-prec-sqrt -fno-common\" build_type=dm+sm target=x86_64_v3"
+        when: "%intel"
+
   all:
-    compiler: [intel, gcc]
+    compiler: [intel, oneapi, gcc]
     permissions:
       read: world
       write: user
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/spack.yaml
index 35d099f014..52e88506c0 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/spack.yaml
@@ -2,14 +2,24 @@ spack:
   view: false
 
   definitions:
+  - apps:
+    - gromacs %intel
+    - lammps %intel
+    - mpas-model %intel
+    - openfoam %gcc
+    - palace %oneapi
+    - quantum-espresso %intel
+    # - wrf : While building hdf5 cmake errors out with Detecting Fortran/C Interface: Failed to compile
+    # Root cause: ifort cannot deal with arbitrarily long file names.
 
-  - optimized_configs:
-    - palace target=x86_64_v4
-    - palace target=x86_64_v3
+  - targets:
+      - 'target=x86_64_v4'
+      - 'target=x86_64_v3'
 
   specs:
-  - $optimized_configs
-
+  - matrix:
+      - [$apps]
+      - [$targets]
   ci:
     pipeline-gen:
     - build-job:
@@ -28,5 +38,6 @@ spack:
           # Do not distribute Intel & ARM binaries
           - - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
             - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
+
   cdash:
     build-group: AWS Packages