xSDK examples v0.4.0 (#37295)

* [xsdk-examples] Initial commit for v0.4.0

* [xsdk-examples] v0.4.0 depends on xsdk@0.8.0

* add in missing xsdk dependencies

* [xsdk-examples] remove repeated 'depends_on' directive

* [xsdk-examples] simplify and extend a bit the package

[mfem] process more optional dependencies of HiOp

[strumpack, superlu-dist] add a workaround for an issue on Mac

* [mfem] fix the handling of the hiop dependency

* [@spackbot] updating style on behalf of v-dobrev

* [xsdk-examples] enable 'heffte' and 'tasmanian' if enabled in 'xsdk'

* [xsdk-examples] Add PUMI dependency

* [xsdk-examples] Add preCICE dependency

* [xsdk-examples] add +rocm

* heffte: add in a backport fix for building xsdk-examples with cuda

* [xsdk] Remove the explicit requirement for deal.II to be built +hdf5

* ENABLE_ROCM -> ENABLE_HIP

* [hiop] Workaround for CMake not finding Cray's BLAS (libsci)

[xsdk-examples] Set CUDA/HIP architectures; sync cuda/rocm variants with xsdk

* [@spackbot] updating style on behalf of v-dobrev

* [exago] Workaround for CMake not finding Cray's LAPACK/BLAS, libsci

[mfem] Tweaks for running tests under Flux and PBS

* [slate] Pass CUDA/HIP architectures to CMake

* [heffte] For newer CMake versions, set CMAKE_CUDA_ARCHITECTURES

* [hypre] Patch v2.26.0 to fix sequential compilation in 'src/seq_mv'

* [xsdk-examples] Some tweaks in dependencies and compilers used

* [xsdk] Make the 'trilinos' variant sticky

[xsdk-examples] Tweak dependencies

* [slate] Fix copy-paste error

* [xsdk-examples] Workaround for CMakePackage not having the legacy
   property 'build_directory'

* [xsdk-examples] Replace the testing branch used temporarily for v0.4.0 with
                the official release

---------

Co-authored-by: Satish Balay <balay@mcs.anl.gov>

11 files changed, 118 insertions, 46 deletions

diff --git a/var/spack/repos/builtin/packages/exago/package.py b/var/spack/repos/builtin/packages/exago/package.py
index 8fd28be25d..4534afecf6 100644
--- a/var/spack/repos/builtin/packages/exago/package.py
+++ b/var/spack/repos/builtin/packages/exago/package.py
@@ -179,6 +179,7 @@ class Exago(CMakePackage, CudaPackage, ROCmPackage):
                 self.define_from_variant("EXAGO_ENABLE_IPOPT", "ipopt"),
                 self.define_from_variant("EXAGO_ENABLE_PYTHON", "python"),
                 self.define_from_variant("EXAGO_ENABLE_LOGGING", "logging"),
+                self.define("LAPACK_LIBRARIES", spec["lapack"].libs + spec["blas"].libs),
             ]
         )
 
diff --git a/var/spack/repos/builtin/packages/heffte/cmake-magma-v230.patch b/var/spack/repos/builtin/packages/heffte/cmake-magma-v230.patch
new file mode 100644
index 0000000000..ec7413ec54
--- /dev/null
+++ b/var/spack/repos/builtin/packages/heffte/cmake-magma-v230.patch
@@ -0,0 +1,13 @@
+diff --git a/cmake/HeffteConfig.cmake b/cmake/HeffteConfig.cmake
+index 8bbe907..df1cf86 100644
+--- a/cmake/HeffteConfig.cmake
++++ b/cmake/HeffteConfig.cmake
+@@ -51,7 +51,7 @@ if (@Heffte_ENABLE_MAGMA@ AND NOT TARGET Heffte::MAGMA)
+     set_target_properties(Heffte::MAGMA PROPERTIES INTERFACE_INCLUDE_DIRECTORIES @HeffteMAGMA_INCLUDES@)
+     if (@Heffte_ENABLE_CUDA@)
+         list(FILTER CUDA_CUBLAS_LIBRARIES EXCLUDE REGEX "-NOTFOUND$") # work-around CMake 3.10 + CUDA 10
+-        target_link_libraries(Heffte Heffte::MAGMA INTERFACE ${CUDA_CUBLAS_LIBRARIES})
++        target_link_libraries(Heffte::MAGMA INTERFACE ${CUDA_CUBLAS_LIBRARIES})
+     endif()
+     if (@Heffte_ENABLE_ROCM@)
+         find_package(rocblas REQUIRED)
diff --git a/var/spack/repos/builtin/packages/heffte/package.py b/var/spack/repos/builtin/packages/heffte/package.py
index 700a01f9f0..c123251444 100644
--- a/var/spack/repos/builtin/packages/heffte/package.py
+++ b/var/spack/repos/builtin/packages/heffte/package.py
@@ -49,6 +49,7 @@ class Heffte(CMakePackage, CudaPackage, ROCmPackage):
 
     patch("threads10.patch", when="@1.0")
     patch("fortran200.patch", when="@2.0.0")
+    patch("cmake-magma-v230.patch", when="@2.3.0")
 
     depends_on("cmake@3.10:", type=("build", "run"))
     depends_on("cmake@3.19:", when="@develop", type=("build", "run"))
@@ -113,6 +114,8 @@ class Heffte(CMakePackage, CudaPackage, ROCmPackage):
                     nvcc_flags += "{0};".format(nvflag)
 
                 args.append("-DCUDA_NVCC_FLAGS={0}".format(nvcc_flags))
+                archs = ";".join(cuda_arch)
+                args.append("-DCMAKE_CUDA_ARCHITECTURES=%s" % archs)
 
         if "+rocm" in self.spec and self.spec.satisfies("@:2.3.0"):
             args.append("-DCMAKE_CXX_COMPILER={0}".format(self.spec["hip"].hipcc))
diff --git a/var/spack/repos/builtin/packages/hiop/package.py b/var/spack/repos/builtin/packages/hiop/package.py
index 82fab411a0..624a29859a 100644
--- a/var/spack/repos/builtin/packages/hiop/package.py
+++ b/var/spack/repos/builtin/packages/hiop/package.py
@@ -153,6 +153,8 @@ class Hiop(CMakePackage, CudaPackage, ROCmPackage):
         args.extend(
             [
                 self.define("HIOP_BUILD_STATIC", True),
+                self.define("BLAS_FOUND", True),
+                self.define("BLAS_LIBRARIES", spec["blas"].libs),
                 self.define("LAPACK_FOUND", True),
                 self.define("LAPACK_LIBRARIES", spec["lapack"].libs + spec["blas"].libs),
                 self.define_from_variant("HIOP_BUILD_SHARED", "shared"),
diff --git a/var/spack/repos/builtin/packages/hypre/package.py b/var/spack/repos/builtin/packages/hypre/package.py
index e3394a7a8c..a15db12e16 100644
--- a/var/spack/repos/builtin/packages/hypre/package.py
+++ b/var/spack/repos/builtin/packages/hypre/package.py
@@ -91,6 +91,10 @@ class Hypre(AutotoolsPackage, CudaPackage, ROCmPackage):
     # Patch to get config flags right
     patch("detect-compiler.patch", when="@2.15.0:2.20.0")
 
+    @when("@2.26.0")
+    def patch(self):  # fix sequential compilation in 'src/seq_mv'
+        filter_file("\tmake", "\t$(MAKE)", "src/seq_mv/Makefile")
+
     depends_on("mpi", when="+mpi")
     depends_on("blas")
     depends_on("lapack")
diff --git a/var/spack/repos/builtin/packages/mfem/package.py b/var/spack/repos/builtin/packages/mfem/package.py
index 24cf733d61..dbd917f0a9 100644
--- a/var/spack/repos/builtin/packages/mfem/package.py
+++ b/var/spack/repos/builtin/packages/mfem/package.py
@@ -537,7 +537,10 @@ class Mfem(Package, CudaPackage, ROCmPackage):
                 mfem_mpiexec = "jsrun"
                 mfem_mpiexec_np = "-p"
         elif "FLUX_EXEC_PATH" in os.environ:
-            mfem_mpiexec = "flux mini run"
+            mfem_mpiexec = "flux run"
+            mfem_mpiexec_np = "-n"
+        elif "PBS_JOBID" in os.environ:
+            mfem_mpiexec = "mpiexec"
             mfem_mpiexec_np = "-n"
 
         metis5_str = "NO"
@@ -1025,12 +1028,28 @@ class Mfem(Package, CudaPackage, ROCmPackage):
 
         if "+hiop" in spec:
             hiop = spec["hiop"]
+            hiop_hdrs = hiop.headers
             hiop_libs = hiop.libs
+            hiop_hdrs += spec["lapack"].headers + spec["blas"].headers
             hiop_libs += spec["lapack"].libs + spec["blas"].libs
-            if "^magma" in hiop:
-                hiop_libs += hiop["magma"].libs
+            hiop_opt_libs = ["magma", "umpire"]
+            for opt_lib in hiop_opt_libs:
+                if "^" + opt_lib in hiop:
+                    hiop_hdrs += hiop[opt_lib].headers
+                    hiop_libs += hiop[opt_lib].libs
+            # raja's libs property does not work
+            if "^raja" in hiop:
+                raja = hiop["raja"]
+                hiop_hdrs += raja.headers
+                hiop_libs += find_libraries(
+                    "libRAJA", raja.prefix, shared=("+shared" in raja), recursive=True
+                )
+                if raja.satisfies("^camp"):
+                    camp = raja["camp"]
+                    hiop_hdrs += camp.headers
+                    hiop_libs += find_optional_library("libcamp", camp.prefix)
             options += [
-                "HIOP_OPT=-I%s" % hiop.prefix.include,
+                "HIOP_OPT=%s" % hiop_hdrs.cpp_flags,
                 "HIOP_LIB=%s" % ld_flags_from_library_list(hiop_libs),
             ]
 
diff --git a/var/spack/repos/builtin/packages/slate/package.py b/var/spack/repos/builtin/packages/slate/package.py
index 40204e4632..72575a19d2 100644
--- a/var/spack/repos/builtin/packages/slate/package.py
+++ b/var/spack/repos/builtin/packages/slate/package.py
@@ -102,6 +102,13 @@ class Slate(CMakePackage, CudaPackage, ROCmPackage):
             backend_config,
             "-Duse_mpi=%s" % ("+mpi" in spec),
         ]
+        if "+cuda" in spec:
+            archs = ";".join(spec.variants["cuda_arch"].value)
+            config.append("-DCMAKE_CUDA_ARCHITECTURES=%s" % archs)
+        if "+rocm" in spec:
+            archs = ";".join(spec.variants["amdgpu_target"].value)
+            config.append("-DCMAKE_HIP_ARCHITECTURES=%s" % archs)
+
         if self.run_tests:
             config.append("-DSCALAPACK_LIBRARIES=%s" % spec["scalapack"].libs.joined(";"))
         return config
diff --git a/var/spack/repos/builtin/packages/strumpack/package.py b/var/spack/repos/builtin/packages/strumpack/package.py
index 145d520187..5f9853c729 100644
--- a/var/spack/repos/builtin/packages/strumpack/package.py
+++ b/var/spack/repos/builtin/packages/strumpack/package.py
@@ -139,6 +139,10 @@ class Strumpack(CMakePackage, CudaPackage, ROCmPackage):
                 )
             args.extend([self.define_from_variant("STRUMPACK_C_INTERFACE", "c_interface")])
 
+        # Workaround for linking issue on Mac:
+        if spec.satisfies("%apple-clang +mpi"):
+            args.append("-DCMAKE_Fortran_COMPILER=%s" % spec["mpi"].mpifc)
+
         if "+cuda" in spec:
             args.extend(
                 [
diff --git a/var/spack/repos/builtin/packages/superlu-dist/package.py b/var/spack/repos/builtin/packages/superlu-dist/package.py
index ad30dff284..b535c5fd12 100644
--- a/var/spack/repos/builtin/packages/superlu-dist/package.py
+++ b/var/spack/repos/builtin/packages/superlu-dist/package.py
@@ -123,6 +123,10 @@ class SuperluDist(CMakePackage, CudaPackage, ROCmPackage):
                     "HIP_HIPCC_FLAGS", "--amdgpu-target=" + ",".join(rocm_archs) + " -I/" + mpiinc
                 )
 
+        # Workaround for linking issue on Mac:
+        if spec.satisfies("%apple-clang"):
+            append_define("CMAKE_Fortran_COMPILER", spec["mpi"].mpifc)
+
         append_from_variant("BUILD_SHARED_LIBS", "shared")
         return cmake_args
 
diff --git a/var/spack/repos/builtin/packages/xsdk-examples/package.py b/var/spack/repos/builtin/packages/xsdk-examples/package.py
index a5684a0385..bf5ab4f02e 100644
--- a/var/spack/repos/builtin/packages/xsdk-examples/package.py
+++ b/var/spack/repos/builtin/packages/xsdk-examples/package.py
@@ -7,7 +7,7 @@
 from spack.package import *
 
 
-class XsdkExamples(CMakePackage, CudaPackage):
+class XsdkExamples(CMakePackage, CudaPackage, ROCmPackage):
     """xSDK Examples show usage of libraries in the xSDK package."""
 
     homepage = "http://xsdk.info"
@@ -17,6 +17,7 @@ class XsdkExamples(CMakePackage, CudaPackage):
     maintainers("balay", "luszczek", "balos1", "shuds13", "v-dobrev")
 
     version("develop", branch="master")
+    version("0.4.0", sha256="de54e02e0222420976a2f4cf0a6230e4bb625b443c66500fa1441032db206df9")
     version("0.3.0", sha256="e7444a403c0a69eeeb34a4068be4d6f4e5b54cbfd275629019b9236a538a739e")
     version(
         "0.2.0",
@@ -25,10 +26,25 @@ class XsdkExamples(CMakePackage, CudaPackage):
     )
 
     depends_on("xsdk+cuda", when="+cuda")
+    depends_on("xsdk~cuda", when="~cuda")
     for sm_ in CudaPackage.cuda_arch_values:
         depends_on("xsdk+cuda cuda_arch={0}".format(sm_), when="+cuda cuda_arch={0}".format(sm_))
+    depends_on("xsdk+rocm", when="+rocm")
+    depends_on("xsdk~rocm", when="~rocm")
+    for ac_ in ROCmPackage.amdgpu_targets:
+        depends_on(
+            "xsdk+rocm amdgpu_target={0}".format(ac_), when="+rocm amdgpu_target={0}".format(ac_)
+        )
 
     depends_on("xsdk@develop", when="@develop")
+    # Use ^dealii~hdf5 because of HDF5 linking issue in deal.II 9.4.0.
+    # Disable 'arborx' to remove the 'kokkos' dependency which conflicts with
+    # the internal Kokkos used by 'trilinos':
+    depends_on("xsdk@0.8.0 ~arborx ^mfem+pumi ^dealii~hdf5", when="@0.4.0")
+    depends_on("xsdk@0.8.0 ^mfem+strumpack", when="@0.4.0 ^xsdk+strumpack")
+    depends_on("xsdk@0.8.0 ^mfem+ginkgo", when="@0.4.0 ^xsdk+ginkgo")
+    depends_on("xsdk@0.8.0 ^mfem+hiop", when="@0.4.0 ^xsdk+hiop")
+    depends_on("xsdk@0.8.0 ^sundials+magma", when="@0.4.0 +cuda")
     depends_on("xsdk@0.7.0", when="@0.3.0")
     depends_on("xsdk@0.7.0 ^mfem+strumpack", when="@0.3.0 ^xsdk+strumpack")
     depends_on("xsdk@0.7.0 ^sundials+magma", when="@0.3.0 +cuda")
@@ -39,53 +55,52 @@ class XsdkExamples(CMakePackage, CudaPackage):
     def cmake_args(self):
         spec = self.spec
 
+        def enabled(pkg):
+            if type(pkg) is not list:
+                return "ON" if "^" + pkg in spec else "OFF"
+            else:
+                return "ON" if all([("^" + p in spec) for p in pkg]) else "OFF"
+
+        # Note: paths to the enabled packages are automatically added by Spack
+        # to the variable CMAKE_PREFIX_PATH.
         args = [
-            "-DCMAKE_C_COMPILER=%s" % spec["mpi"].mpicc,
-            "-DCMAKE_CXX_COMPILER=%s" % spec["mpi"].mpicxx,
+            # Using the MPI wrappers for C and C++ may cause linking issues
+            # when CUDA is enabled.
+            # "-DCMAKE_C_COMPILER=%s" % spec["mpi"].mpicc,
+            # "-DCMAKE_CXX_COMPILER=%s" % spec["mpi"].mpicxx,
+            # Use the Fortran MPI wrapper as a workaround for a linking issue
+            # with some versions of apple-clang on Mac.
+            "-DCMAKE_Fortran_COMPILER=%s" % spec["mpi"].mpifc,
+            "-DENABLE_AMREX=" + enabled("amrex"),
+            "-DENABLE_DEAL_II=" + enabled("dealii"),
+            "-DENABLE_GINKGO=" + enabled("ginkgo"),
+            "-DENABLE_HEFFTE=" + enabled("heffte"),
+            "-DENABLE_HIOP=" + enabled("hiop"),
             "-DENABLE_HYPRE=ON",
-            "-DHYPRE_DIR=%s" % spec["hypre"].prefix,
+            "-DENABLE_MAGMA=" + enabled("magma"),
             "-DENABLE_MFEM=ON",
-            "-DMETIS_DIR=%s" % spec["metis"].prefix,
-            "-DMFEM_DIR=%s" % spec["mfem"].prefix,
             "-DENABLE_PETSC=ON",
-            "-DPETSc_DIR=%s" % spec["petsc"].prefix,
-            "-DENABLE_PLASMA=ON",
-            "-DPLASMA_DIR=%s" % spec["plasma"].prefix,
+            # ENABLE_PLASMA also needs Slate:
+            "-DENABLE_PLASMA=" + enabled(["plasma", "slate"]),
+            "-DENABLE_PRECICE=" + enabled("precice"),
+            "-DENABLE_PUMI=ON",
+            "-DENABLE_STRUMPACK=" + enabled("strumpack"),
             "-DENABLE_SUNDIALS=ON",
-            "-DSUNDIALS_DIR=%s" % spec["sundials"].prefix,
             "-DENABLE_SUPERLU=ON",
-            "-DSUPERLUDIST_DIR=%s" % spec["superlu-dist"].prefix,
+            "-DENABLE_TASMANIAN=" + enabled("tasmanian"),
+            "-DENABLE_TRILINOS=" + enabled("trilinos"),
         ]
 
-        if "+cuda" in spec:  # if cuda variant was activated for xsdk
-            args.extend(
-                [
-                    "-DENABLE_CUDA=ON",
-                    "-DCMAKE_CUDA_ARCHITECTURES=%s" % spec.variants["cuda_arch"].value,
-                ]
-            )
-        if "+ginkgo" in spec:  # if ginkgo variant was activated for xsdk
-            args.extend(["-DENABLE_GINKGO=ON", "-DGinkgo_DIR=%s" % spec["ginkgo"].prefix])
-        if "+magma" in spec:  # if magma variant was activated for xsdk
-            args.extend(["-DENABLE_MAGMA=ON", "-DMAGMA_DIR=%s" % spec["magma"].prefix])
-        if "+strumpack" in spec:  # if magma variant was activated for xsdk
-            args.extend(["-DENABLE_STRUMPACK=ON", "-DSTRUMPACK_DIR=%s" % spec["strumpack"].prefix])
-        if "+slate" in spec:  # if slate variant was activated for xsdk
-            args.extend(
-                [
-                    "-DENABLE_SLATE=ON",
-                    "-DSLATE_DIR=%s" % spec["slate"].prefix,
-                    "-DBLASPP_DIR=%s" % spec["blaspp"].prefix,
-                    "-DLAPACKPP_DIR=%s" % spec["lapackpp"].prefix,
-                ]
-            )
-        if "trilinos" in spec:  # if trilinos variant was activated for xsdk
-            args.extend(["ENABLE_TRILINOS=ON", "-DTRILINOS_DIR_PATH=%s" % spec["trilinos"].prefix])
-        if "zlib" in spec:  # if zlib variant was activated for MFEM
-            args.append("-DZLIB_LIBRARY_DIR=%s" % spec["zlib"].prefix.lib)
+        if "+cuda" in spec:
+            archs = ";".join(spec.variants["cuda_arch"].value)
+            args.extend(["-DENABLE_CUDA=ON", "-DCMAKE_CUDA_ARCHITECTURES=%s" % archs])
+
+        if "+rocm" in spec:
+            archs = ";".join(spec.variants["amdgpu_target"].value)
+            args.extend(["-DENABLE_HIP=ON", "-DCMAKE_HIP_ARCHITECTURES=%s" % archs])
 
         return args
 
     def check(self):
-        with working_dir(self.build_directory):
-            ctest(parallel=False)
+        with working_dir(self.builder.build_directory):
+            ctest("--output-on-failure")
diff --git a/var/spack/repos/builtin/packages/xsdk/package.py b/var/spack/repos/builtin/packages/xsdk/package.py
index e3164c27f1..51a3c72ca1 100644
--- a/var/spack/repos/builtin/packages/xsdk/package.py
+++ b/var/spack/repos/builtin/packages/xsdk/package.py
@@ -89,7 +89,7 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
     version("0.7.0")
     version("0.6.0", deprecated=True)
 
-    variant("trilinos", default=True, description="Enable trilinos package build")
+    variant("trilinos", default=True, sticky=True, description="Enable trilinos package build")
     variant("datatransferkit", default=True, description="Enable datatransferkit package build")
     variant("omega-h", default=True, description="Enable omega-h package build")
     variant("strumpack", default=True, description="Enable strumpack package build")
@@ -205,12 +205,12 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
     xsdk_depends_on("dealii +trilinos~adol-c", when="+trilinos +dealii")
     xsdk_depends_on("dealii ~trilinos", when="~trilinos +dealii")
     xsdk_depends_on(
-        "dealii@master~assimp~python~doc~gmsh+petsc+slepc+mpi~int64+hdf5"
+        "dealii@master~assimp~python~doc~gmsh+petsc+slepc+mpi~int64"
         + "~netcdf+metis+sundials~ginkgo~symengine~nanoflann~simplex~arborx~cgal",
         when="@develop +dealii",
     )
     xsdk_depends_on(
-        "dealii@9.4.0~assimp~python~doc~gmsh+petsc+slepc+mpi~int64+hdf5"
+        "dealii@9.4.0~assimp~python~doc~gmsh+petsc+slepc+mpi~int64"
         + "~netcdf+metis+sundials~ginkgo~symengine~simplex~arborx~cgal",
         when="@0.8.0 +dealii",
     )