New MFEM version: 4.5 (#33480)

* New MFEM version: 4.5

Add new MFEM variants: ginkgo, hiop

* mfem: small tweaks

* mfem: tweak testing script

* mfem: try to resolve issue #30483

* mfem: fix style

* mfem: tweak for Spack-built hipsparse

Co-authored-by: eugeneswalker <38933153+eugeneswalker@users.noreply.github.com>

2 files changed, 124 insertions, 41 deletions

diff --git a/var/spack/repos/builtin/packages/mfem/package.py b/var/spack/repos/builtin/packages/mfem/package.py
index dab89f9550..d188c2fc88 100644
--- a/var/spack/repos/builtin/packages/mfem/package.py
+++ b/var/spack/repos/builtin/packages/mfem/package.py
@@ -49,6 +49,13 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     version("develop", branch="master")
 
     version(
+        "4.5.0",
+        sha256="4f201bec02fc5460a902596697b6c1deb7b15ac57c71f615b2ab4a8eb65665f7",
+        url="https://bit.ly/mfem-4-5",
+        extension="tar.gz",
+    )
+
+    version(
         "4.4.0",
         sha256="37250dbef6e97b16dc9ab50973e8d68bc165bb4afcdaf91b3b72c8972c87deef",
         url="https://bit.ly/mfem-4-4",
@@ -171,7 +178,9 @@ class Mfem(Package, CudaPackage, ROCmPackage):
         "libunwind", default=False, description="Enable backtrace on error support using Libunwind"
     )
     variant("fms", default=False, when="@4.3.0:", description="Enable FMS I/O support")
-    # TODO: SIMD, Ginkgo, ADIOS2, HiOp, MKL CPardiso, Axom/Sidre
+    variant("ginkgo", default=False, when="@4.3.0:", description="Enable Ginkgo support")
+    variant("hiop", default=False, when="@4.4.0:", description="Enable HiOp support")
+    # TODO: SIMD, ADIOS2, MKL CPardiso, Axom/Sidre
     variant(
         "timer",
         default="auto",
@@ -229,6 +238,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     conflicts("^mpich@4:", when="@:4.3+mpi")
 
     depends_on("mpi", when="+mpi")
+    depends_on("hipsparse", when="@4.4.0:+rocm")
     depends_on("hypre@2.10.0:2.13", when="@:3.3+mpi")
     depends_on("hypre@:2.20.0", when="@3.4:4.2+mpi")
     depends_on("hypre@:2.23.0", when="@4.3.0+mpi")
@@ -256,11 +266,13 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     depends_on("sundials@2.7.0+mpi+hypre", when="@:3.3.0+sundials+mpi")
     depends_on("sundials@2.7.0:", when="@3.3.2:+sundials~mpi")
     depends_on("sundials@2.7.0:+mpi+hypre", when="@3.3.2:+sundials+mpi")
-    depends_on("sundials@5.0.0:5", when="@4.0.1-xsdk:+sundials~mpi")
-    depends_on("sundials@5.0.0:5+mpi+hypre", when="@4.0.1-xsdk:+sundials+mpi")
+    depends_on("sundials@5.0.0:5", when="@4.0.1-xsdk:4.4+sundials~mpi")
+    depends_on("sundials@5.0.0:5+mpi+hypre", when="@4.0.1-xsdk:4.4+sundials+mpi")
+    depends_on("sundials@5.0.0:", when="@4.5.0:+sundials~mpi")
+    depends_on("sundials@5.0.0:+mpi+hypre", when="@4.5.0:+sundials+mpi")
     for sm_ in CudaPackage.cuda_arch_values:
         depends_on(
-            "sundials@5.4.0:5+cuda cuda_arch={0}".format(sm_),
+            "sundials@5.4.0:+cuda cuda_arch={0}".format(sm_),
             when="@4.2.0:+sundials+cuda cuda_arch={0}".format(sm_),
         )
     depends_on("pumi", when="+pumi~shared")
@@ -320,6 +332,29 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     depends_on("conduit@0.3.1:,master:", when="+conduit")
     depends_on("conduit+mpi", when="+conduit+mpi")
     depends_on("libfms@0.2.0:", when="+fms")
+    depends_on("ginkgo@1.4.0:", when="+ginkgo")
+    for sm_ in CudaPackage.cuda_arch_values:
+        depends_on(
+            "ginkgo+cuda cuda_arch={0}".format(sm_),
+            when="+ginkgo+cuda cuda_arch={0}".format(sm_),
+        )
+    for gfx in ROCmPackage.amdgpu_targets:
+        depends_on(
+            "ginkgo+rocm amdgpu_target={0}".format(gfx),
+            when="+ginkgo+rocm amdgpu_target={0}".format(gfx),
+        )
+    depends_on("hiop@0.4.6:~mpi", when="+hiop~mpi")
+    depends_on("hiop@0.4.6:+mpi", when="+hiop+mpi")
+    for sm_ in CudaPackage.cuda_arch_values:
+        depends_on(
+            "hiop+cuda cuda_arch={0}".format(sm_),
+            when="+hiop+cuda cuda_arch={0}".format(sm_),
+        )
+    for gfx in ROCmPackage.amdgpu_targets:
+        depends_on(
+            "hiop+rocm amdgpu_target={0}".format(gfx),
+            when="+hiop+rocm amdgpu_target={0}".format(gfx),
+        )
 
     # The MFEM 4.0.0 SuperLU interface fails when using hypre@2.16.0 and
     # superlu-dist@6.1.1. See https://github.com/mfem/mfem/issues/983.
@@ -488,6 +523,9 @@ class Mfem(Package, CudaPackage, ROCmPackage):
             else:
                 mfem_mpiexec = "jsrun"
                 mfem_mpiexec_np = "-p"
+        elif "FLUX_JOB_ID" in os.environ:
+            mfem_mpiexec = "flux mini run"
+            mfem_mpiexec_np = "-n"
 
         metis5_str = "NO"
         if ("+metis" in spec) and spec["metis"].satisfies("@5:"):
@@ -530,11 +568,32 @@ class Mfem(Package, CudaPackage, ROCmPackage):
             "MFEM_USE_CEED=%s" % yes_no("+libceed"),
             "MFEM_USE_UMPIRE=%s" % yes_no("+umpire"),
             "MFEM_USE_FMS=%s" % yes_no("+fms"),
+            "MFEM_USE_GINKGO=%s" % yes_no("+ginkgo"),
+            "MFEM_USE_HIOP=%s" % yes_no("+hiop"),
             "MFEM_MPIEXEC=%s" % mfem_mpiexec,
             "MFEM_MPIEXEC_NP=%s" % mfem_mpiexec_np,
             "MFEM_USE_EXCEPTIONS=%s" % yes_no("+exceptions"),
         ]
 
+        # Determine C++ standard to use:
+        cxxstd = None
+        if self.spec.satisfies("@4.0.0:"):
+            cxxstd = "11"
+        if self.spec.satisfies("^raja@2022.03.0:"):
+            cxxstd = "14"
+        if self.spec.satisfies("^umpire@2022.03.0:"):
+            cxxstd = "14"
+        if self.spec.satisfies("^sundials@6.4.0:"):
+            cxxstd = "14"
+        if self.spec.satisfies("^ginkgo"):
+            cxxstd = "14"
+        cxxstd_flag = None
+        if cxxstd:
+            if "+cuda" in spec:
+                cxxstd_flag = "-std=c++" + cxxstd
+            else:
+                cxxstd_flag = getattr(self.compiler, "cxx" + cxxstd + "_flag")
+
         cxxflags = spec.compiler_flags["cxxflags"]
 
         if cxxflags:
@@ -557,16 +616,16 @@ class Mfem(Package, CudaPackage, ROCmPackage):
                     "-x=cu --expt-extended-lambda -arch=sm_%s" % cuda_arch,
                     "-ccbin %s" % (spec["mpi"].mpicxx if "+mpi" in spec else env["CXX"]),
                 ]
-            if self.spec.satisfies("@4.0.0:"):
-                if "+cuda" in spec:
-                    cxxflags.append("-std=c++11")
-                else:
-                    cxxflags.append(self.compiler.cxx11_flag)
+            if cxxstd_flag:
+                cxxflags.append(cxxstd_flag)
             # The cxxflags are set by the spack c++ compiler wrapper. We also
             # set CXXFLAGS explicitly, for clarity, and to properly export the
             # cxxflags in the variable MFEM_CXXFLAGS in config.mk.
             options += ["CXXFLAGS=%s" % " ".join(cxxflags)]
 
+        elif cxxstd_flag:
+            options += ["BASE_FLAGS=%s" % cxxstd_flag]
+
         # Treat any 'CXXFLAGS' in the environment as extra c++ flags which are
         # handled through the 'CPPFLAGS' makefile variable in MFEM. Also, unset
         # 'CXXFLAGS' from the environment to prevent it from overriding the
@@ -589,9 +648,12 @@ class Mfem(Package, CudaPackage, ROCmPackage):
             if "+cuda" in hypre:
                 hypre_gpu_libs = " -lcusparse -lcurand"
             elif "+rocm" in hypre:
-                hypre_gpu_libs = " " + ld_flags_from_dirs(
-                    [env["ROCM_PATH"] + "/lib"], ["rocsparse", "rocrand"]
-                )
+                hypre_rocm_libs = LibraryList([])
+                if "^rocsparse" in hypre:
+                    hypre_rocm_libs += hypre["rocsparse"].libs
+                if "^rocrand" in hypre:
+                    hypre_rocm_libs += hypre["rocrand"].libs
+                hypre_gpu_libs = " " + ld_flags_from_library_list(hypre_rocm_libs)
             options += [
                 "HYPRE_OPT=-I%s" % hypre.prefix.include,
                 "HYPRE_LIB=%s%s" % (ld_flags_from_library_list(all_hypre_libs), hypre_gpu_libs),
@@ -804,6 +866,15 @@ class Mfem(Package, CudaPackage, ROCmPackage):
         if "+rocm" in spec:
             amdgpu_target = ",".join(spec.variants["amdgpu_target"].value)
             options += ["HIP_CXX=%s" % spec["hip"].hipcc, "HIP_ARCH=%s" % amdgpu_target]
+            if "^hipsparse" in spec:  # hipsparse is needed @4.4.0:+rocm
+                # Note: MFEM's defaults.mk want to find librocsparse.* in
+                # $(HIP_DIR)/lib, so we set HIP_DIR to be the prefix of
+                # rocsparse (which is a dependency of hipsparse).
+                options += [
+                    "HIP_DIR=%s" % spec["rocsparse"].prefix,
+                    "HIP_OPT=%s" % spec["hipsparse"].headers.cpp_flags,
+                    "HIP_LIB=%s" % ld_flags_from_library_list(spec["hipsparse"].libs),
+                ]
 
         if "+occa" in spec:
             options += [
@@ -895,6 +966,21 @@ class Mfem(Package, CudaPackage, ROCmPackage):
                 "FMS_LIB=%s" % ld_flags_from_library_list(libfms.libs),
             ]
 
+        if "+ginkgo" in spec:
+            ginkgo = spec["ginkgo"]
+            options += [
+                "GINKGO_DIR=%s" % ginkgo.prefix,
+                "GINKGO_BUILD_TYPE=%s" % ginkgo.variants["build_type"].value,
+            ]
+
+        if "+hiop" in spec:
+            hiop = spec["hiop"]
+            lapack_blas = spec["lapack"].libs + spec["blas"].libs
+            options += [
+                "HIOP_OPT=-I%s" % hiop.prefix.include,
+                "HIOP_LIB=%s" % ld_flags_from_library_list(hiop.libs + lapack_blas),
+            ]
+
         make("config", *options, parallel=False)
         make("info", parallel=False)
 
diff --git a/var/spack/repos/builtin/packages/mfem/test_builds.sh b/var/spack/repos/builtin/packages/mfem/test_builds.sh
index 9b534836ec..5c43f87512 100755
--- a/var/spack/repos/builtin/packages/mfem/test_builds.sh
+++ b/var/spack/repos/builtin/packages/mfem/test_builds.sh
@@ -8,7 +8,7 @@ rocm_arch="gfx908"
 spack_jobs=''
 # spack_jobs='-j 128'
 
-mfem='mfem@4.4.0'${compiler}
+mfem='mfem@4.5.0'${compiler}
 mfem_dev='mfem@develop'${compiler}
 
 backends='+occa+raja+libceed'
@@ -28,32 +28,29 @@ builds=(
     # preferred version:
     ${mfem}
     ${mfem}'~mpi~metis~zlib'
-    # TODO: add back '+gslib' when the gslib test is fixed.
-    # TODO: add back '+slepc' when its build is fixed.
-    ${mfem}"$backends"'+superlu-dist+strumpack+suite-sparse+petsc \
-        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
+    ${mfem}"$backends"'+superlu-dist+strumpack+suite-sparse+petsc+slepc+gslib \
+        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         '"$backends_specs $strumpack_spec $petsc_spec $hdf5_spec"
     ${mfem}'~mpi \
         '"$backends"'+suite-sparse+sundials+gslib+mpfr+netcdf \
-        +zlib+gnutls+libunwind+conduit \
+        +zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         '"$backends_specs $hdf5_spec"' ^sundials~mpi'
 
     # develop version, shared builds:
     ${mfem_dev}'+shared~static'
     ${mfem_dev}'+shared~static~mpi~metis~zlib'
     # NOTE: Shared build with +gslib works on mac but not on linux
-    # TODO: add back '+gslib' when the gslib test is fixed and the above NOTE
+    # TODO: add back '+gslib' when the above NOTE
     #       is addressed.
-    # TODO: add back '+slepc' when its build is fixed.
     ${mfem_dev}'+shared~static \
-        '"$backends"'+superlu-dist+strumpack+suite-sparse+petsc \
-        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
+        '"$backends"'+superlu-dist+strumpack+suite-sparse+petsc+slepc \
+        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         '"$backends_specs $strumpack_spec $petsc_spec $hdf5_spec"
     # NOTE: Shared build with +gslib works on mac but not on linux
     # TODO: add back '+gslib' when the above NOTE is addressed.
     ${mfem_dev}'+shared~static~mpi \
         '"$backends"'+suite-sparse+sundials+mpfr+netcdf \
-        +zlib+gnutls+libunwind+conduit \
+        +zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         '"$backends_specs $hdf5_spec"' ^sundials~mpi'
 )
 
@@ -67,8 +64,7 @@ builds2=(
     ${mfem}'+sundials~mpi ^sundials~mpi'
     ${mfem}'+sundials'
     ${mfem}'+pumi'
-    # TODO: uncomment the next line when the gslib test is fixed.
-    # ${mfem}'+gslib'
+    ${mfem}'+gslib'
     ${mfem}'+netcdf~mpi'
     ${mfem}'+netcdf'
     ${mfem}'+mpfr'
@@ -77,10 +73,11 @@ builds2=(
     ${mfem}'+conduit'
     ${mfem}'+umpire'
     ${mfem}'+petsc'" $petsc_spec"
-    # TODO: uncomment the next line when the slepc build is fixed.
-    # ${mfem}'+petsc+slepc'" $petsc_spec"
-    # TODO: uncomment the next line when the threadsafe build is fixed.
-    # ${mfem}'+threadsafe'
+    ${mfem}'+petsc+slepc'" $petsc_spec"
+    ${mfem}'+ginkgo'
+    ${mfem}'+hiop'
+    ${mfem}'+threadsafe'
+    #
     # develop version
     ${mfem_dev}"$backends $backends_specs"
     ${mfem_dev}'+superlu-dist'
@@ -90,8 +87,7 @@ builds2=(
     ${mfem_dev}'+sundials~mpi ^sundials~mpi'
     ${mfem_dev}'+sundials'
     ${mfem_dev}'+pumi'
-    # TODO: uncomment the next line when the gslib test is fixed.
-    # ${mfem_dev}'+gslib'
+    ${mfem_dev}'+gslib'
     ${mfem_dev}'+netcdf~mpi'
     ${mfem_dev}'+netcdf'
     ${mfem_dev}'+mpfr'
@@ -101,8 +97,9 @@ builds2=(
     ${mfem_dev}'+umpire'
     ${mfem_dev}'+petsc'" $petsc_spec"
     ${mfem_dev}'+petsc+slepc'" $petsc_spec"
-    # TODO: uncomment the next line when the threadsafe build is fixed.
-    # ${mfem_dev}'+threadsafe'
+    ${mfem_dev}'+ginkgo'
+    ${mfem_dev}'+hiop'
+    ${mfem_dev}'+threadsafe'
 )
 
 
@@ -125,7 +122,7 @@ builds_cuda=(
     # TODO: add back "+petsc+slepc $petsc_spec_cuda" when it works.
     ${mfem}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \
         +strumpack+suite-sparse \
-        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
+        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         ^raja+cuda+openmp'" $strumpack_cuda_spec"' \
         '"$hdf5_spec"
 
@@ -138,7 +135,7 @@ builds_cuda=(
     # TODO: add back "+sundials" when it's supported with '^hypre+cuda'.
     ${mfem}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \
         +suite-sparse \
-        +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
+        +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         ^raja+cuda+openmp ^hypre+cuda \
         '"$hdf5_spec"
 
@@ -164,7 +161,7 @@ builds_cuda=(
     # TODO: add back "+petsc+slepc $petsc_spec_cuda" when it works.
     ${mfem_dev}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \
         +strumpack+suite-sparse \
-        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
+        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         ^raja+cuda+openmp'" $strumpack_cuda_spec"' \
         '"$hdf5_spec"
 
@@ -177,7 +174,7 @@ builds_cuda=(
     # TODO: add back "+sundials" when it's supported with '^hypre+cuda'.
     ${mfem_dev}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \
         +suite-sparse \
-        +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
+        +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         ^raja+cuda+openmp ^hypre+cuda \
         '"$hdf5_spec"
 )
@@ -200,7 +197,7 @@ builds_rocm=(
     # TODO: add "+petsc+slepc $petsc_spec_rocm" when it is supported.
     ${mfem}'+rocm+openmp+raja+occa+libceed amdgpu_target='"${rocm_arch}"' \
         +strumpack+suite-sparse \
-        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
+        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         ^raja+rocm~openmp ^occa~cuda'" $strumpack_rocm_spec"' \
         '"$hdf5_spec"
 
@@ -212,7 +209,7 @@ builds_rocm=(
     # TODO: add back "+sundials" when it's supported with '^hypre+rocm'.
     ${mfem}'+rocm+openmp+raja+occa+libceed amdgpu_target='"${rocm_arch}"' \
         +suite-sparse \
-        +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
+        +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         ^raja+rocm~openmp ^occa~cuda ^hypre+rocm \
         '"$hdf5_spec"
 
@@ -238,9 +235,9 @@ for bld in "${run_builds[@]}"; do
     printf "    %s\n" "${bld}"
     printf "%s\n" "${SEP}"
     eval bbb="\"${bld}\""
-    spack spec -I $bbb || exit 1
+    spack spec --fresh -I $bbb || exit 1
     printf "%s\n" "${sep}"
-    spack install $spack_jobs --test=root $bbb || exit 2
+    spack install $spack_jobs --fresh --test=root $bbb || exit 2
 done
 
 # Uninstall all mfem builds: