Update exawind packages (#40793)

7 files changed, 197 insertions, 23 deletions

diff --git a/var/spack/repos/builtin/packages/amr-wind/package.py b/var/spack/repos/builtin/packages/amr-wind/package.py
index cdd0167946..75a900cab9 100644
--- a/var/spack/repos/builtin/packages/amr-wind/package.py
+++ b/var/spack/repos/builtin/packages/amr-wind/package.py
@@ -20,6 +20,7 @@ class AmrWind(CMakePackage, CudaPackage, ROCmPackage):
     license("BSD-3-Clause")
 
     version("main", branch="main", submodules=True)
+    version("0.9.0", tag="v0.9.0", submodules=True)
 
     variant("hypre", default=True, description="Enable Hypre integration")
     variant("ascent", default=False, description="Enable Ascent integration")
@@ -31,9 +32,22 @@ class AmrWind(CMakePackage, CudaPackage, ROCmPackage):
     variant("shared", default=True, description="Build shared libraries")
     variant("tests", default=True, description="Activate regression tests")
     variant("tiny_profile", default=False, description="Activate tiny profile")
+    variant("hdf5", default=False, description="Enable HDF5 plots with ZFP compression")
+    variant("umpire", default=False, description="Enable Umpire")
+    variant("sycl", default=False, description="Enable SYCL backend")
+    variant("gpu-aware-mpi", default=False, description="gpu-aware-mpi")
 
     depends_on("hypre~int64@2.20.0:", when="+hypre")
     depends_on("hypre+mpi", when="+hypre+mpi")
+    depends_on("hdf5~mpi", when="+hdf5~mpi")
+    depends_on("hdf5+mpi", when="+hdf5+mpi")
+    depends_on("h5z-zfp", when="+hdf5")
+    depends_on("zfp", when="+hdf5")
+    depends_on("hypre+umpire", when="+umpire")
+    depends_on("hypre+sycl", when="+sycl")
+    depends_on("hypre+gpu-aware-mpi", when="+gpu-aware-mpi")
+    depends_on("hypre@2.29.0:", when="@0.9.0:+hypre")
+
     for arch in CudaPackage.cuda_arch_values:
         depends_on("hypre+cuda cuda_arch=%s" % arch, when="+cuda+hypre cuda_arch=%s" % arch)
     for arch in ROCmPackage.amdgpu_targets:
@@ -88,6 +102,13 @@ class AmrWind(CMakePackage, CudaPackage, ROCmPackage):
         if "+mpi" in self.spec:
             args.append(define("MPI_HOME", self.spec["mpi"].prefix))
 
+        if "+hdf5" in self.spec:
+            cmake_options.append(self.define("AMR_WIND_ENABLE_HDF5", True))
+            cmake_options.append(self.define("AMR_WIND_ENABLE_HDF5_ZFP", True))
+            # Help AMReX understand if HDF5 is parallel or not.
+            # Building HDF5 with CMake as Spack does, causes this inspection to break.
+            cmake_options.append(self.define("HDF5_IS_PARALLEL", spec.satisfies("+mpi")))
+
         if "+cuda" in self.spec:
             amrex_arch = [
                 "{0:.1f}".format(float(i) / 10.0) for i in self.spec.variants["cuda_arch"].value
@@ -100,4 +121,16 @@ class AmrWind(CMakePackage, CudaPackage, ROCmPackage):
             targets = self.spec.variants["amdgpu_target"].value
             args.append("-DAMReX_AMD_ARCH=" + ";".join(str(x) for x in targets))
 
+        if "+sycl" in self.spec:
+            cmake_options.append(self.define("AMR_WIND_ENABLE_SYCL", True))
+            requires(
+                "%dpcpp",
+                "%oneapi",
+                policy="one_of",
+                msg=(
+                    "AMReX's SYCL GPU Backend requires DPC++ (dpcpp) "
+                    "or the oneAPI CXX (icpx) compiler."
+                ),
+            )
+
         return args
diff --git a/var/spack/repos/builtin/packages/exawind/package.py b/var/spack/repos/builtin/packages/exawind/package.py
index b7ba5e378d..8ea3720414 100644
--- a/var/spack/repos/builtin/packages/exawind/package.py
+++ b/var/spack/repos/builtin/packages/exawind/package.py
@@ -6,37 +6,93 @@
 from spack.package import *
 
 
-class Exawind(CMakePackage):
+class Exawind(CMakePackage, CudaPackage, ROCmPackage):
     """Multi-application driver for Exawind project."""
 
     homepage = "https://github.com/Exawind/exawind-driver"
     git = "https://github.com/Exawind/exawind-driver.git"
 
-    maintainers("jrood-nrel", "psakievich")
+    maintainers("jrood-nrel")
 
     tags = ["ecp", "ecp-apps"]
 
+    # Testing is currently always enabled, but should be optional in the future
+    # to avoid cloning the mesh submodule
+    version("master", branch="main", submodules=True)
+    version("1.0.0", tag="v1.0.0", submodules=True)
     license("Apache-2.0")
 
-    version("master", branch="main")
+    variant("openfast", default=False, description="Enable OpenFAST integration")
+    variant("hypre", default=True, description="Enable hypre solver")
+    variant("stk_simd", default=False, description="Enable SIMD in STK")
+    variant("umpire", default=False, description="Enable Umpire")
+    variant("tiny_profile", default=False, description="Turn on AMR-wind with tiny profile")
+    variant("sycl", default=False, description="Enable SYCL backend for AMR-Wind")
+    variant("gpu-aware-mpi", default=False, description="gpu-aware-mpi")
 
-    depends_on("trilinos+stk")
-    depends_on("tioga+shared~nodegid")
-    depends_on("nalu-wind+hypre+openfast+tioga+wind-utils")
-    depends_on("amr-wind+hypre+mpi+netcdf+openfast")
-    depends_on("openfast+cxx+shared@2.6.0:")
+    conflicts("amr-wind+hypre", when="+sycl")
+
+    for arch in CudaPackage.cuda_arch_values:
+        depends_on("amr-wind+cuda cuda_arch=%s" % arch, when="+cuda cuda_arch=%s" % arch)
+        depends_on("nalu-wind+cuda cuda_arch=%s" % arch, when="+cuda cuda_arch=%s" % arch)
+
+    for arch in ROCmPackage.amdgpu_targets:
+        depends_on("amr-wind+rocm amdgpu_target=%s" % arch, when="+rocm amdgpu_target=%s" % arch)
+        depends_on("nalu-wind+rocm amdgpu_target=%s" % arch, when="+rocm amdgpu_target=%s" % arch)
+
+    depends_on("nalu-wind+tioga")
+    depends_on("amr-wind+netcdf+mpi")
+    depends_on("tioga~nodegid")
     depends_on("yaml-cpp@0.6:")
+    depends_on("nalu-wind+openfast", when="+openfast")
+    depends_on("amr-wind+hypre", when="+hypre~sycl")
+    depends_on("amr-wind~hypre", when="~hypre")
+    depends_on("nalu-wind+hypre", when="+hypre")
+    depends_on("nalu-wind~hypre", when="~hypre")
+    depends_on("amr-wind+sycl", when="+sycl")
+    depends_on("nalu-wind+umpire", when="+umpire")
+    depends_on("amr-wind+umpire", when="+umpire")
+    depends_on("amr-wind+tiny_profile", when="+tiny_profile")
+    depends_on("nalu-wind+gpu-aware-mpi", when="+gpu-aware-mpi")
+    depends_on("amr-wind+gpu-aware-mpi", when="+gpu-aware-mpi")
+    depends_on("nalu-wind@2.0.0:", when="@1.0.0:")
+    depends_on("amr-wind@0.9.0:", when="@1.0.0:")
+    depends_on("tioga@1.0.0:", when="@1.0.0:")
 
     def cmake_args(self):
         spec = self.spec
 
-        args = [
-            self.define("Trilinos_DIR", spec["trilinos"].prefix),
-            self.define("TIOGA_DIR", spec["tioga"].prefix),
-            self.define("Nalu-Wind_DIR", spec["nalu-wind"].prefix),
-            self.define("AMR-Wind_DIR", spec["amr-wind"].prefix),
-            self.define("OpenFAST_DIR", spec["openfast"].prefix),
-            self.define("YAML-CPP_DIR", spec["yaml-cpp"].prefix),
-        ]
+        args = [self.define("MPI_HOME", spec["mpi"].prefix)]
+
+        if "+umpire" in self.spec:
+            args.append(self.define_from_variant("EXAWIND_ENABLE_UMPIRE", "umpire"))
+            args.append(self.define("UMPIRE_DIR", self.spec["umpire"].prefix))
+
+        if spec.satisfies("+cuda"):
+            args.append(self.define("EXAWIND_ENABLE_CUDA", True))
+            args.append(self.define("CUDAToolkit_ROOT", self.spec["cuda"].prefix))
+            args.append(self.define("EXAWIND_CUDA_ARCH", self.spec.variants["cuda_arch"].value))
+
+        if spec.satisfies("+rocm"):
+            targets = self.spec.variants["amdgpu_target"].value
+            args.append(self.define("EXAWIND_ENABLE_ROCM", True))
+            args.append(self.define("CMAKE_CXX_COMPILER", self.spec["hip"].hipcc))
+            args.append(self.define("CMAKE_HIP_ARCHITECTURES", ";".join(str(x) for x in targets)))
+            args.append(self.define("AMDGPU_TARGETS", ";".join(str(x) for x in targets)))
+            args.append(self.define("GPU_TARGETS", ";".join(str(x) for x in targets)))
+
+        if spec.satisfies("^amr-wind+hdf5"):
+            args.append(self.define("H5Z_ZFP_USE_STATIC_LIBS", True))
+
+        if spec.satisfies("^amr-wind+ascent"):
+            args.append(self.define("CMAKE_EXE_LINKER_FLAGS", self.compiler.openmp_flag))
 
         return args
+
+    def setup_build_environment(self, env):
+        if "~stk_simd" in self.spec:
+            env.append_flags("CXXFLAGS", "-DUSE_STK_SIMD_NONE")
+        if "+rocm" in self.spec:
+            env.set("OMPI_CXX", self.spec["hip"].hipcc)
+            env.set("MPICH_CXX", self.spec["hip"].hipcc)
+            env.set("MPICXX_CXX", self.spec["hip"].hipcc)
diff --git a/var/spack/repos/builtin/packages/hypre/package.py b/var/spack/repos/builtin/packages/hypre/package.py
index f61cba9cc0..658259a2b5 100644
--- a/var/spack/repos/builtin/packages/hypre/package.py
+++ b/var/spack/repos/builtin/packages/hypre/package.py
@@ -72,6 +72,7 @@ class Hypre(AutotoolsPackage, CudaPackage, ROCmPackage):
     variant("int64", default=False, description="Use 64bit integers")
     variant("mixedint", default=False, description="Use 64bit integers while reducing memory use")
     variant("complex", default=False, description="Use complex values")
+    variant("gpu-aware-mpi", default=False, description="Use gpu-aware mpi")
     variant("mpi", default=True, description="Enable MPI support")
     variant("openmp", default=False, description="Enable OpenMP support")
     variant("debug", default=False, description="Build debug instead of optimized version")
@@ -300,6 +301,9 @@ class Hypre(AutotoolsPackage, CudaPackage, ROCmPackage):
             configure_args.append("--with-magma-lib=%s" % spec["magma"].libs)
             configure_args.append("--with-magma")
 
+        if "+gpu-aware-mpi" in spec:
+            options.append("--enable-gpu-aware-mpi")
+
         configure_args.extend(self.enable_or_disable("fortran"))
 
         return configure_args
diff --git a/var/spack/repos/builtin/packages/nalu-wind/package.py b/var/spack/repos/builtin/packages/nalu-wind/package.py
index 7d77c431d1..ba0d96e564 100644
--- a/var/spack/repos/builtin/packages/nalu-wind/package.py
+++ b/var/spack/repos/builtin/packages/nalu-wind/package.py
@@ -15,7 +15,7 @@ def _parse_float(val):
         return False
 
 
-class NaluWind(CMakePackage, CudaPackage):
+class NaluWind(CMakePackage, CudaPackage, ROCmPackage):
     """Nalu-Wind: Wind energy focused variant of Nalu."""
 
     homepage = "https://nalu-wind.readthedocs.io"
@@ -26,6 +26,7 @@ class NaluWind(CMakePackage, CudaPackage):
     tags = ["ecp", "ecp-apps"]
 
     version("master", branch="master")
+    version("2.0.0", tag="v2.0.0")
 
     variant("pic", default=True, description="Position independent code")
     variant(
@@ -45,12 +46,31 @@ class NaluWind(CMakePackage, CudaPackage):
     variant("hypre", default=True, description="Compile with Hypre support")
     variant("trilinos-solvers", default=True, description="Compile with Trilinos Solvers support")
     variant("catalyst", default=False, description="Compile with Catalyst support")
+    variant("shared", default=True, description="Build shared libraries")
     variant("fftw", default=False, description="Compile with FFTW support")
+    variant("fsi", default=False, description="Enable fluid-structure-interaction models")
     variant("boost", default=False, description="Enable Boost integration")
+    variant("gpu-aware-mpi", default=False, description="gpu-aware-mpi")
     variant("wind-utils", default=False, description="Build wind-utils")
+    variant("umpire", default=False, description="Enable Umpire")
+    conflicts(
+        "+shared",
+        when="+cuda",
+        msg="invalid device functions are generated with shared libs and cuda",
+    )
+    conflicts(
+        "+shared",
+        when="+rocm",
+        msg="invalid device functions are generated with shared libs and rocm",
+    )
+    conflicts("+cuda", when="+rocm")
+    conflicts("+rocm", when="+cuda")
 
     depends_on("mpi")
     depends_on("yaml-cpp@0.5.3:")
+    depends_on("openfast@4.0.0:+cxx+netcdf", when="+fsi")
+    depends_on("trilinos@13.4.1+exodus+zoltan+stk", when="@=2.0.0")
+    depends_on("hypre@2.29.0:", when="@2.0.0:+hypre")
     depends_on(
         "trilinos@13:+exodus+tpetra+zoltan+stk~superlu-dist~superlu+hdf5+shards~hypre+gtest"
     )
@@ -76,12 +96,42 @@ class NaluWind(CMakePackage, CudaPackage):
             "hypre@develop +mpi+cuda~int64~superlu-dist cuda_arch={0}".format(_arch),
             when="+hypre+cuda cuda_arch={0}".format(_arch),
         )
+    for _arch in ROCmPackage.amdgpu_targets:
+        depends_on(
+            "trilinos@13.4.0.2022.10.27: "
+            "~shared+exodus+tpetra+zoltan+stk~superlu-dist~superlu"
+            "+hdf5+shards~hypre+gtest+rocm amdgpu_target={0}".format(_arch),
+            when="+rocm amdgpu_target={0}".format(_arch),
+        )
+        depends_on(
+            "hypre+rocm amdgpu_target={0}".format(_arch),
+            when="+hypre+rocm amdgpu_target={0}".format(_arch),
+        )
+
     depends_on("trilinos-catalyst-ioss-adapter", when="+catalyst")
     depends_on("fftw+mpi", when="+fftw")
     depends_on("nccmp")
     # indirect dependency needed to make original concretizer work
     depends_on("netcdf-c+parallel-netcdf")
     depends_on("boost +filesystem +iostreams cxxstd=14", when="+boost")
+    supported_cxxstd = ["17"]
+    variant(
+        "cxxstd", default="17", values=supported_cxxstd, multi=False, description="cxx standard"
+    )
+    for std in supported_cxxstd:
+        depends_on("trilinos cxxstd=%s" % std, when="cxxstd=%s" % std)
+
+    def setup_build_environment(self, env):
+        if "~stk_simd" in self.spec:
+            env.append_flags("CXXFLAGS", "-DUSE_STK_SIMD_NONE")
+        if "+cuda" in self.spec:
+            env.set("CUDA_LAUNCH_BLOCKING", "1")
+            env.set("CUDA_MANAGED_FORCE_DEVICE_ALLOC", "1")
+            env.set("OMPI_CXX", self.spec["kokkos-nvcc-wrapper"].kokkos_cxx)
+            env.set("MPICH_CXX", self.spec["kokkos-nvcc-wrapper"].kokkos_cxx)
+            env.set("MPICXX_CXX", self.spec["kokkos-nvcc-wrapper"].kokkos_cxx)
+        if "+rocm" in self.spec:
+            env.append_flags("CXXFLAGS", "-fgpu-rdc")
 
     def cmake_args(self):
         spec = self.spec
@@ -95,22 +145,26 @@ class NaluWind(CMakePackage, CudaPackage):
             self.define_from_variant("ENABLE_CUDA", "cuda"),
             self.define_from_variant("ENABLE_WIND_UTILS", "wind-utils"),
             self.define_from_variant("ENABLE_BOOST", "boost"),
+            self.define_from_variant("CMAKE_CXX_STANDARD", "cxxstd"),
+            self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
+            self.define_from_variant("ENABLE_OPENFAST", "openfast"),
+            self.define_from_variant("ENABLE_TIOGA", "tioga"),
+            self.define_from_variant("ENABLE_HYPRE", "hypre"),
+            self.define_from_variant("ENABLE_TRILINOS_SOLVERS", "trilinos-solvers"),
+            self.define_from_variant("ENABLE_PARAVIEW_CATALYST", "catalyst"),
+            self.define_from_variant("ENABLE_FFTW", "fftw"),
+            self.define_from_variant("ENABLE_UMPIRE", "umpire"),
         ]
 
-        args.append(self.define_from_variant("ENABLE_OPENFAST", "openfast"))
         if "+openfast" in spec:
             args.append(self.define("OpenFAST_DIR", spec["openfast"].prefix))
 
-        args.append(self.define_from_variant("ENABLE_TIOGA", "tioga"))
         if "+tioga" in spec:
             args.append(self.define("TIOGA_DIR", spec["tioga"].prefix))
 
-        args.append(self.define_from_variant("ENABLE_HYPRE", "hypre"))
         if "+hypre" in spec:
             args.append(self.define("HYPRE_DIR", spec["hypre"].prefix))
 
-        args.append(self.define_from_variant("ENABLE_TRILINOS_SOLVERS", "trilinos-solvers"))
-        args.append(self.define_from_variant("ENABLE_PARAVIEW_CATALYST", "catalyst"))
         if "+catalyst" in spec:
             args.append(
                 self.define(
@@ -118,7 +172,6 @@ class NaluWind(CMakePackage, CudaPackage):
                 )
             )
 
-        args.append(self.define_from_variant("ENABLE_FFTW", "fftw"))
         if "+fftw" in spec:
             args.append(self.define("FFTW_DIR", spec["fftw"].prefix))
 
@@ -131,6 +184,15 @@ class NaluWind(CMakePackage, CudaPackage):
                 ]
             )
 
+        if "+umpire" in spec:
+            args.append(self.define("UMPIRE_DIR", spec["umpire"].prefix))
+
+        if "+rocm" in spec:
+            args.append(self.define("CMAKE_CXX_COMPILER", spec["hip"].hipcc))
+            args.append(self.define("ENABLE_ROCM", True))
+            targets = spec.variants["amdgpu_target"].value
+            args.append(self.define("GPU_TARGETS", ";".join(str(x) for x in targets)))
+
         if "darwin" in spec.architecture:
             args.append(self.define("CMAKE_MACOSX_RPATH", "ON"))
 
diff --git a/var/spack/repos/builtin/packages/openfast/hub_seg_fault.patch b/var/spack/repos/builtin/packages/openfast/hub_seg_fault.patch
new file mode 100644
index 0000000000..a312f04b8d
--- /dev/null
+++ b/var/spack/repos/builtin/packages/openfast/hub_seg_fault.patch
@@ -0,0 +1,17 @@
+diff --git a/modules/openfast-library/src/FAST_Solver.f90 b/modules/openfast-library/src/FAST_Solver.f90
+index 364d0b78..10056965 100644
+--- a/modules/openfast-library/src/FAST_Solver.f90
++++ b/modules/openfast-library/src/FAST_Solver.f90
+@@ -607,9 +607,9 @@ SUBROUTINE AD_InputSolve_IfW( p_FAST, u_AD, y_IfW, y_OpFM, ErrStat, ErrMsg )
+       end if
+
+       if (u_AD%rotors(1)%NacelleMotion%NNodes > 0) then
+-         u_AD%rotors(1)%InflowOnNacelle(1) = y_OpFM%u(node)
+-         u_AD%rotors(1)%InflowOnNacelle(2) = y_OpFM%v(node)
+-         u_AD%rotors(1)%InflowOnNacelle(3) = y_OpFM%w(node)
++         u_AD%rotors(1)%InflowOnNacelle(1) = y_OpFM%u(1)
++         u_AD%rotors(1)%InflowOnNacelle(2) = y_OpFM%v(1)
++         u_AD%rotors(1)%InflowOnNacelle(3) = y_OpFM%w(1)
+          node = node + 1
+       else
+          u_AD%rotors(1)%InflowOnNacelle = 0.0_ReKi
diff --git a/var/spack/repos/builtin/packages/openfast/package.py b/var/spack/repos/builtin/packages/openfast/package.py
index d00f560c60..b11038fc70 100644
--- a/var/spack/repos/builtin/packages/openfast/package.py
+++ b/var/spack/repos/builtin/packages/openfast/package.py
@@ -13,6 +13,7 @@ class Openfast(CMakePackage):
     git = "https://github.com/OpenFAST/openfast.git"
 
     maintainers("jrood-nrel")
+    patch("hub_seg_fault.patch", when="@2.7:3.2")
 
     license("Apache-2.0")
 
diff --git a/var/spack/repos/builtin/packages/tioga/package.py b/var/spack/repos/builtin/packages/tioga/package.py
index 8a453c39f9..6fdf41d141 100644
--- a/var/spack/repos/builtin/packages/tioga/package.py
+++ b/var/spack/repos/builtin/packages/tioga/package.py
@@ -20,6 +20,7 @@ class Tioga(CMakePackage):
     # The original TIOGA repo has possibly been abandoned,
     # so work on TIOGA has continued in the Exawind project
     version("develop", git="https://github.com/Exawind/tioga.git", branch="exawind")
+    version("1.0.0", git="https://github.com/Exawind/tioga.git", tag="v1.0.0")
     version("master", branch="master")
 
     variant("shared", default=sys.platform != "darwin", description="Build shared libraries")