MFEM: add new version v4.7 (#44010)

* Core change: logic for extracting RPATHs from modules may return
  `None`: filter this out of the set of RPATHs that is auto-generated
* Core change: `CachedCMakePackage` no longer adds ldflags to
  `CMAKE_STATIC_LINKER_FLAGS`: generally these flags are not appropriate
  for static linking (e.g. invocation of `ar`)
* [mfem] Add version 4.7
* [mfem] Add variant for precision (single/double). Enforce consistency
  for precision amongst mfem and hypre/petsc/mumps dependencies
* [mfem] Add cxxstd (and related constraints preventing use of
  old cxxstd values for newer versions of some dependencies)
* [hypre] In line with prior point, added support for specifying
  precision
* [petsc] Add config option to avoid error when building against
  `superlu-dist+rocm`
* [hiop] add proper `raja`/`umpire`/`camp` version constraints for
  `hiop` versions 0.3.99-0.4.x; require `+raja` for `+rocm`, and
  add dependency on `hiprand` for `+rocm`
* [butterflypack, mfem, strumpack, suite-sparse] Require
  `CRAYLIBS_{target-family}` env var to be defined
* [suite-sparse] versions `@7.4:` changed install location of headers:
  add symlink from old location to new location
* [zlib-ng] Fix error where shared libs were not successfully built for
  `%cce@17` (the build did not fail, but the finished `zlib-ng%cce@17`
  install did not have shared libs)

15 files changed, 478 insertions, 134 deletions

diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index acd15629a4..97e0ff6e0e 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -740,7 +740,9 @@ def get_rpaths(pkg):
     # Second module is our compiler mod name. We use that to get rpaths from
     # module show output.
     if pkg.compiler.modules and len(pkg.compiler.modules) > 1:
-        rpaths.append(path_from_modules([pkg.compiler.modules[1]]))
+        mod_rpath = path_from_modules([pkg.compiler.modules[1]])
+        if mod_rpath:
+            rpaths.append(mod_rpath)
     return list(dedupe(filter_system_paths(rpaths)))
 
 
diff --git a/lib/spack/spack/build_systems/cached_cmake.py b/lib/spack/spack/build_systems/cached_cmake.py
index 48c714f31a..aff54d7559 100644
--- a/lib/spack/spack/build_systems/cached_cmake.py
+++ b/lib/spack/spack/build_systems/cached_cmake.py
@@ -162,7 +162,9 @@ class CachedCMakeBuilder(CMakeBuilder):
             ld_flags = " ".join(flags["ldflags"])
             ld_format_string = "CMAKE_{0}_LINKER_FLAGS"
             # CMake has separate linker arguments for types of builds.
-            for ld_type in ["EXE", "MODULE", "SHARED", "STATIC"]:
+            # 'ldflags' should not be used with CMAKE_STATIC_LINKER_FLAGS which
+            # is used by the archiver, so don't include "STATIC" in this loop:
+            for ld_type in ["EXE", "MODULE", "SHARED"]:
                 ld_string = ld_format_string.format(ld_type)
                 entries.append(cmake_cache_string(ld_string, ld_flags))
 
diff --git a/var/spack/repos/builtin/packages/butterflypack/package.py b/var/spack/repos/builtin/packages/butterflypack/package.py
index 048b0a0243..09857510ff 100644
--- a/var/spack/repos/builtin/packages/butterflypack/package.py
+++ b/var/spack/repos/builtin/packages/butterflypack/package.py
@@ -3,8 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-from platform import machine
-
 from spack.package import *
 
 
@@ -79,7 +77,13 @@ class Butterflypack(CMakePackage):
         args.append("-Denable_openmp=%s" % ("ON" if "+openmp" in spec else "OFF"))
         if "%cce" in spec:
             # Assume the proper Cray CCE module (cce) is loaded:
-            craylibs_path = env["CRAYLIBS_" + machine().upper()]
+            craylibs_var = "CRAYLIBS_" + str(spec.target.family).upper()
+            craylibs_path = env.get(craylibs_var, None)
+            if not craylibs_path:
+                raise InstallError(
+                    f"The environment variable {craylibs_var} is not defined.\n"
+                    "\tMake sure the 'cce' module is in the compiler spec."
+                )
             env.setdefault("LDFLAGS", "")
             env["LDFLAGS"] += " -Wl,-rpath," + craylibs_path
 
diff --git a/var/spack/repos/builtin/packages/hiop/package.py b/var/spack/repos/builtin/packages/hiop/package.py
index 4364d0eeb3..188e51357c 100644
--- a/var/spack/repos/builtin/packages/hiop/package.py
+++ b/var/spack/repos/builtin/packages/hiop/package.py
@@ -75,7 +75,7 @@ class Hiop(CMakePackage, CudaPackage, ROCmPackage):
     variant("jsrun", default=False, description="Enable/Disable jsrun command for testing")
     variant("shared", default=False, description="Enable/Disable shared libraries")
     variant("mpi", default=True, description="Enable/Disable MPI")
-    variant("raja", default=False, description="Enable/Disable RAJA")
+    variant("raja", default=False, when="@0.3.99:", description="Enable/Disable RAJA")
     variant("kron", default=False, description="Enable/Disable Kron reduction")
     variant("sparse", default=False, description="Enable/Disable Sparse linear algebra")
     variant(
@@ -124,7 +124,11 @@ class Hiop(CMakePackage, CudaPackage, ROCmPackage):
     # 1.0.2 fixes bug with cuda 12 compatibility
     # hiop@0.6.0 requires cusolver API in cuda@11
     depends_on("cuda@11:11.9", when="@0.6.0:1.0.1+cuda")
-    depends_on("cuda@11:", when="@develop:+cuda")
+    # Version v0.7.0 of HiOp is the earliest version that uses
+    #    cusparseSpGEMMreuse_workEstimation
+    # which appears for the first time in the cuSPARSE version shipped with
+    # CUDA 11.3.1, at least according to the CUDA online documentation.
+    depends_on("cuda@11.3.1:", when="@0.7:+cuda")
     # Before hiop@0.6.0 only cuda requirement was magma
     depends_on("cuda", when="@:0.5.4+cuda")
 
@@ -134,9 +138,11 @@ class Hiop(CMakePackage, CudaPackage, ROCmPackage):
 
     # RAJA > 0.14 and Umpire > 6.0 require c++ std 14
     # We are working on supporting newer Umpire/RAJA versions
-    depends_on("raja@0.14.0:0.14", when="@0.5.0:+raja")
-    depends_on("umpire@6.0.0:6", when="@0.5.0:+raja")
-    depends_on("camp@0.2.3:0.2", when="@0.5.0:+raja")
+    depends_on("raja@0.14", when="@0.5:+raja")
+    depends_on("raja@:0.13", when="@0.3.99:0.4+raja")
+    depends_on("umpire@6", when="@0.5:+raja")
+    depends_on("umpire@:5", when="@0.3.99:0.4+raja")
+    depends_on("camp@0.2.3:0.2", when="@0.3.99:+raja")
 
     # This is no longer a requirement in RAJA > 0.14
     depends_on("umpire+cuda~shared", when="+raja+cuda ^raja@:0.14")
@@ -149,8 +155,10 @@ class Hiop(CMakePackage, CudaPackage, ROCmPackage):
 
     # We rely on RAJA / Umpire utilities when supporting CUDA backend
     conflicts("~raja", when="+cuda", msg="RAJA is required for CUDA support")
+    conflicts("~raja", when="+rocm", msg="RAJA is required for ROCm support")
 
     depends_on("hip", when="+rocm")
+    depends_on("hiprand", when="+rocm")
     depends_on("hipblas", when="+rocm")
     depends_on("hipsparse", when="+rocm")
 
diff --git a/var/spack/repos/builtin/packages/hypre/hypre-precision-fix.patch b/var/spack/repos/builtin/packages/hypre/hypre-precision-fix.patch
new file mode 100644
index 0000000000..d40b705820
--- /dev/null
+++ b/var/spack/repos/builtin/packages/hypre/hypre-precision-fix.patch
@@ -0,0 +1,27 @@
+diff --git a/src/distributed_ls/ParaSails/ConjGrad.c b/src/distributed_ls/ParaSails/ConjGrad.c
+index 0ef71b36f..7abbc38aa 100644
+--- a/src/distributed_ls/ParaSails/ConjGrad.c
++++ b/src/distributed_ls/ParaSails/ConjGrad.c
+@@ -33,19 +33,19 @@ static HYPRE_Real InnerProd(HYPRE_Int n, HYPRE_Real *x, HYPRE_Real *y, MPI_Comm
+ static void CopyVector(HYPRE_Int n, HYPRE_Real *x, HYPRE_Real *y)
+ {
+     HYPRE_Int one = 1;
+-    hypre_F90_NAME_BLAS(dcopy, DCOPY)(&n, x, &one, y, &one);
++    hypre_dcopy(&n, x, &one, y, &one);
+ }
+ 
+ static void ScaleVector(HYPRE_Int n, HYPRE_Real alpha, HYPRE_Real *x)
+ {
+     HYPRE_Int one = 1;
+-    hypre_F90_NAME_BLAS(dscal, DSCAL)(&n, &alpha, x, &one);
++    hypre_dscal(&n, &alpha, x, &one);
+ }
+ 
+ static void Axpy(HYPRE_Int n, HYPRE_Real alpha, HYPRE_Real *x, HYPRE_Real *y)
+ {
+     HYPRE_Int one = 1;
+-    hypre_F90_NAME_BLAS(daxpy, DAXPY)(&n, &alpha, x, &one, y, &one);
++    hypre_daxpy(&n, &alpha, x, &one, y, &one);
+ }
+ 
+ 
diff --git a/var/spack/repos/builtin/packages/hypre/package.py b/var/spack/repos/builtin/packages/hypre/package.py
index 60eaa2bbb1..a4a60d2d8b 100644
--- a/var/spack/repos/builtin/packages/hypre/package.py
+++ b/var/spack/repos/builtin/packages/hypre/package.py
@@ -85,6 +85,14 @@ class Hypre(AutotoolsPackage, CudaPackage, ROCmPackage):
     variant("caliper", default=False, description="Enable Caliper support")
     variant("rocblas", default=False, description="Enable rocBLAS")
     variant("cublas", default=False, description="Enable cuBLAS")
+    variant(
+        "precision",
+        default="double",
+        values=("single", "double", "longdouble"),
+        multi=False,
+        description="Floating point precision",
+        when="@2.12.1:",
+    )
 
     # Patch to add gptune hookup codes
     patch("ij_gptune.patch", when="+gptune@2.19.0")
@@ -100,6 +108,10 @@ class Hypre(AutotoolsPackage, CudaPackage, ROCmPackage):
     patch("hypre21800-compat.patch", when="@2.18.0")
     # Patch to get config flags right
     patch("detect-compiler.patch", when="@2.15.0:2.20.0")
+    # The following patch may not work for all versions, so apply it only when
+    # it is needed:
+    patch("hypre-precision-fix.patch", when="precision=single")
+    patch("hypre-precision-fix.patch", when="precision=longdouble")
 
     @when("@2.26.0")
     def patch(self):  # fix sequential compilation in 'src/seq_mv'
@@ -220,6 +232,11 @@ class Hypre(AutotoolsPackage, CudaPackage, ROCmPackage):
 
         configure_args.extend(self.enable_or_disable("complex"))
 
+        if spec.satisfies("precision=single"):
+            configure_args.append("--enable-single")
+        elif spec.satisfies("precision=longdouble"):
+            configure_args.append("--enable-longdouble")
+
         if spec.satisfies("+shared"):
             configure_args.append("--enable-shared")
 
diff --git a/var/spack/repos/builtin/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py
index c77ef4b345..6493747fd3 100644
--- a/var/spack/repos/builtin/packages/metis/package.py
+++ b/var/spack/repos/builtin/packages/metis/package.py
@@ -98,12 +98,32 @@ class SetupEnvironment:
 
 class MakefileBuilder(spack.build_systems.makefile.MakefileBuilder, SetupEnvironment):
     @property
-    def build_targets(self):
+    def compile_options(self):
         options = []
         if "+shared" in self.spec:
-            options.append("COPTIONS={0}".format(self.pkg.compiler.cc_pic_flag))
+            options.append(self.pkg.compiler.cc_pic_flag)
+        if self.spec.satisfies("%cce@17:"):
+            options.append("-std=c89")
+        return options
+
+    @property
+    def optimize_options(self):
+        options = []
         if "+debug" in self.spec:
-            options.append("OPTFLAGS=-g -O0")
+            options.extend(["-g", "-O0"])
+        else:
+            options.append("-O2")  # default in Makefile.in
+        return options
+
+    @property
+    def build_targets(self):
+        options = []
+        copts = self.compile_options
+        oopts = self.optimize_options
+        if copts:
+            options.append("COPTIONS={0}".format(" ".join(copts)))
+        if oopts:
+            options.append("OPTFLAGS={0}".format(" ".join(oopts)))
         return options
 
     def install(self, pkg, spec, prefix):
@@ -156,6 +176,8 @@ class MakefileBuilder(spack.build_systems.makefile.MakefileBuilder, SetupEnviron
 
         # Set up and run tests on installation
         ccompile(
+            *self.compile_options,
+            *self.optimize_options,
             "-I%s" % prefix.include,
             "-L%s" % prefix.lib,
             (pkg.compiler.cc_rpath_arg + prefix.lib if "+shared" in spec else ""),
diff --git a/var/spack/repos/builtin/packages/mfem/mfem-4.7.patch b/var/spack/repos/builtin/packages/mfem/mfem-4.7.patch
new file mode 100644
index 0000000000..6e0d3c7ef5
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mfem/mfem-4.7.patch
@@ -0,0 +1,102 @@
+diff --git a/examples/hiop/ex9p.cpp b/examples/hiop/ex9p.cpp
+index 4facbb3c0b..f783b97a3b 100644
+--- a/examples/hiop/ex9p.cpp
++++ b/examples/hiop/ex9p.cpp
+@@ -96,6 +96,7 @@ public:
+    {
+       Vector w_glob(width);
+       pfes.Dof_TrueDof_Matrix()->MultTranspose(w, w_glob);
++      w_glob.HostReadWrite(); // read+write -> can use w_glob(i) (non-const)
+       for (int i = 0; i < width; i++) { grad(0, i) = w_glob(i); }
+    }
+ 
+diff --git a/linalg/sparsemat.cpp b/linalg/sparsemat.cpp
+index 0b5334d2a6..efe471d416 100644
+--- a/linalg/sparsemat.cpp
++++ b/linalg/sparsemat.cpp
+@@ -1267,24 +1267,32 @@ real_t SparseMatrix::InnerProduct(const Vector &x, const Vector &y) const
+ 
+ void SparseMatrix::GetRowSums(Vector &x) const
+ {
+-   for (int i = 0; i < height; i++)
++   if (Finalized())
+    {
+-      real_t a = 0.0;
+-      if (A)
++      auto d_I = ReadI();
++      auto d_A = ReadData();
++      auto d_x = x.Write();
++      mfem::forall(height, [=] MFEM_HOST_DEVICE (int i)
+       {
+-         for (int j = I[i], end = I[i+1]; j < end; j++)
++         real_t sum = 0.0;
++         for (int j = d_I[i], end = d_I[i+1]; j < end; j++)
+          {
+-            a += A[j];
++            sum += d_A[j];
+          }
+-      }
+-      else
++         d_x[i] = sum;
++      });
++   }
++   else
++   {
++      for (int i = 0; i < height; i++)
+       {
++         real_t a = 0.0;
+          for (RowNode *np = Rows[i]; np != NULL; np = np->Prev)
+          {
+             a += np->Value;
+          }
++         x(i) = a;
+       }
+-      x(i) = a;
+    }
+ }
+ 
+diff --git a/linalg/sparsemat.hpp b/linalg/sparsemat.hpp
+index 7042279663..dc2d773bc4 100644
+--- a/linalg/sparsemat.hpp
++++ b/linalg/sparsemat.hpp
+@@ -216,7 +216,7 @@ public:
+    void ClearCuSparse() { ClearGPUSparse(); }
+ 
+    /// Check if the SparseMatrix is empty.
+-   bool Empty() const { return (A == NULL) && (Rows == NULL); }
++   bool Empty() const { return A.Empty() && (Rows == NULL); }
+ 
+    /// Return the array #I.
+    inline int *GetI() { return I; }
+diff --git a/tests/unit/general/test_umpire_mem.cpp b/tests/unit/general/test_umpire_mem.cpp
+index 84457669ec..d4a7b85093 100644
+--- a/tests/unit/general/test_umpire_mem.cpp
++++ b/tests/unit/general/test_umpire_mem.cpp
+@@ -18,12 +18,13 @@
+ #include <unistd.h>
+ #include <stdio.h>
+ #include "umpire/Umpire.hpp"
++#include <umpire/strategy/QuickPool.hpp>
+ 
+ #ifdef MFEM_USE_CUDA
+ #include <cuda.h>
+ constexpr const char * device_name = "cuda";
+ #elif defined(MFEM_USE_HIP)
+-constexpr const char * device_name = "raja-hip";
++constexpr const char * device_name = "hip";
+ #endif
+ 
+ using namespace mfem;
+@@ -45,10 +46,12 @@ static bool is_pinned_host(void * h_p)
+    unsigned flags;
+ #ifdef MFEM_USE_CUDA
+    auto err = cudaHostGetFlags(&flags, h_p);
++   cudaGetLastError(); // also resets last error
+    if (err == cudaSuccess) { return true; }
+    else if (err == cudaErrorInvalidValue) { return false; }
+ #elif defined(MFEM_USE_HIP)
+    auto err = hipHostGetFlags(&flags, h_p);
++   hipGetLastError(); // also resets last error
+    if (err == hipSuccess) { return true; }
+    else if (err == hipErrorInvalidValue) { return false; }
+ #endif
diff --git a/var/spack/repos/builtin/packages/mfem/mfem-hip.patch b/var/spack/repos/builtin/packages/mfem/mfem-hip.patch
deleted file mode 100644
index 565bae348c..0000000000
--- a/var/spack/repos/builtin/packages/mfem/mfem-hip.patch
+++ /dev/null
@@ -1,24 +0,0 @@
-From 93ab69cac72cc2d13cfd4b7efcc235bdbca2b9f5 Mon Sep 17 00:00:00 2001

-From: Afzal Patel <afzal.patel@amd.com>

-Date: Wed, 17 Jan 2024 11:44:18 -0800

-Subject: [PATCH] Add hip library path to ghv flags so libamdhip64 can be found

-

----

- config/makefile | 2 +-

- 1 file changed, 1 insertion(+), 1 deletion(-)

-

-diff --git a/config/makefile b/config/makefile

-index 627d117..a453865 100644

---- a/config/makefile

-+++ b/config/makefile

-@@ -38,7 +38,7 @@ all: header config-mk

- MPI = $(MFEM_USE_MPI:NO=)

- GHV_CXX ?= $(MFEM_CXX)

- GHV = get_hypre_version

--GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))

-+GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT)) $(HIP_LIB)

- SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)

- SMX_PATH = $(PUMI_DIR)/include/gmi_sim.h

- SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))

---

-2.25.1
\ No newline at end of file
diff --git a/var/spack/repos/builtin/packages/mfem/package.py b/var/spack/repos/builtin/packages/mfem/package.py
index 0f339e11e7..6ed1214d8a 100644
--- a/var/spack/repos/builtin/packages/mfem/package.py
+++ b/var/spack/repos/builtin/packages/mfem/package.py
@@ -6,7 +6,6 @@
 import os
 import shutil
 import sys
-from platform import machine
 
 from spack.package import *
 
@@ -52,6 +51,13 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     version("develop", branch="master")
 
     version(
+        "4.7.0",
+        sha256="5e889493f5f79848f7b2d16afaae307c59880ac2a7ff2315551c60ca54717751",
+        url="https://bit.ly/mfem-4-7",
+        extension="tar.gz",
+    )
+
+    version(
         "4.6.0",
         sha256="5fa9465b5bec56bfb777a4d2826fba48d85fbace4aed8b64a2fd4059bf075b15",
         url="https://bit.ly/mfem-4-6",
@@ -212,6 +218,21 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     variant("examples", default=False, description="Build and install examples")
     variant("miniapps", default=False, description="Build and install miniapps")
     variant("exceptions", default=False, description="Enable the use of exceptions")
+    variant(
+        "precision",
+        default="double",
+        values=("single", "double"),
+        multi=False,
+        description="Floating point precision",
+        when="@4.7.0:",
+    )
+    variant(
+        "cxxstd",
+        default="auto",
+        values=("auto", conditional("98", when="@:3"), "11", "14", "17"),
+        multi=False,
+        description="C++ language standard",
+    )
 
     conflicts("+shared", when="@:3.3.2")
     conflicts("~static~shared")
@@ -290,6 +311,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     depends_on("sundials@5.0.0:5+mpi+hypre", when="@4.0.1-xsdk:4.4+sundials+mpi")
     depends_on("sundials@5.0.0:6.7.0", when="@4.5.0:+sundials~mpi")
     depends_on("sundials@5.0.0:6.7.0+mpi+hypre", when="@4.5.0:+sundials+mpi")
+    conflicts("cxxstd=11", when="^sundials@6.4.0:")
     for sm_ in CudaPackage.cuda_arch_values:
         depends_on(
             "sundials@5.4.0:+cuda cuda_arch={0}".format(sm_),
@@ -337,7 +359,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     # The PETSc tests in MFEM will fail if PETSc is not configured with
     # MUMPS (and SuiteSparse in older versions). On the other hand, PETSc built
     # with MUMPS is not strictly required, so we do not require it here.
-    depends_on("petsc@3.8:+mpi+double+hypre", when="+petsc")
+    depends_on("petsc@3.8:+mpi+hypre", when="+petsc")
     depends_on("slepc@3.8.0:", when="+slepc")
     # If petsc is built with +cuda, propagate cuda_arch to petsc and slepc
     for sm_ in CudaPackage.cuda_arch_values:
@@ -361,6 +383,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     depends_on("conduit+mpi", when="+conduit+mpi")
     depends_on("libfms@0.2.0:", when="+fms")
     depends_on("ginkgo@1.4.0:", when="+ginkgo")
+    conflicts("cxxstd=11", when="^ginkgo")
     for sm_ in CudaPackage.cuda_arch_values:
         depends_on(
             "ginkgo+cuda cuda_arch={0}".format(sm_), when="+ginkgo+cuda cuda_arch={0}".format(sm_)
@@ -402,6 +425,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     depends_on("raja@0.13.0", when="@4.3.0+raja")
     depends_on("raja@0.14.0:2022.03", when="@4.4.0:4.5.0+raja")
     depends_on("raja@2022.10.3:", when="@4.5.2:+raja")
+    conflicts("cxxstd=11", when="^raja@2022.03.0:")
     for sm_ in CudaPackage.cuda_arch_values:
         depends_on(
             "raja+cuda cuda_arch={0}".format(sm_), when="+raja+cuda cuda_arch={0}".format(sm_)
@@ -429,6 +453,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
 
     depends_on("umpire@2.0.0:2.1.0", when="@:4.3.0+umpire")
     depends_on("umpire@3.0.0:", when="@4.4.0:+umpire")
+    conflicts("cxxstd=11", when="^umpire@2022.03.0:")
     for sm_ in CudaPackage.cuda_arch_values:
         depends_on(
             "umpire+cuda cuda_arch={0}".format(sm_), when="+umpire+cuda cuda_arch={0}".format(sm_)
@@ -448,6 +473,20 @@ class Mfem(Package, CudaPackage, ROCmPackage):
             "amgx~mpi cuda_arch={0}".format(sm_), when="+amgx~mpi cuda_arch={0}".format(sm_)
         )
 
+    for using_double_cond in ["@:4.6", "precision=double"]:
+        with when(using_double_cond):
+            # May need to enforce precision consistency on other packages in the
+            # future.
+            depends_on("hypre precision=double", when="+mpi")
+            depends_on("petsc+double", when="+petsc")
+            depends_on("mumps+double", when="+mumps")
+    with when("precision=single"):
+        # May need to enforce precision consistency on other packages in the
+        # future.
+        depends_on("hypre precision=single", when="+mpi")
+        depends_on("petsc~double", when="+petsc")
+        depends_on("mumps+float", when="+mumps")
+
     patch("mfem_ppc_build.patch", when="@3.2:3.3.0 arch=ppc64le")
     patch("mfem-3.4.patch", when="@3.4.0")
     patch("mfem-3.3-3.4-petsc-3.9.patch", when="@3.3.0:3.4.0 +petsc ^petsc@3.9.0:")
@@ -468,7 +507,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
         when="@4.6.0 +gslib+shared+miniapps",
         sha256="2a31682d876626529e2778a216d403648b83b90997873659a505d982d0e65beb",
     )
-    patch("mfem-hip.patch", when="+rocm ^hip@6.0:")
+    patch("mfem-4.7.patch", when="@4.7.0")
 
     phases = ["configure", "build", "install"]
 
@@ -489,56 +528,16 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     # likely to be up to date in supporting *all* of MFEM's
     # configuration options. So, don't use CMake
     #
-    def configure(self, spec, prefix):
+    def get_make_config_options(self, spec, prefix):
         def yes_no(varstr):
             return "YES" if varstr in self.spec else "NO"
 
-        # See also find_system_libraries in lib/spack/llnl/util/filesystem.py
-        # where the same list of paths is used.
-        sys_lib_paths = [
-            "/lib64",
-            "/lib",
-            "/usr/lib64",
-            "/usr/lib",
-            "/usr/local/lib64",
-            "/usr/local/lib",
-            "/usr/lib/x86_64-linux-gnu",
-        ]
-
-        def is_sys_lib_path(dir):
-            return dir in sys_lib_paths
-
-        xcompiler = ""
-        xlinker = "-Wl,"
-        if "+cuda" in spec:
-            xcompiler = "-Xcompiler="
-            xlinker = "-Xlinker="
-        cuda_arch = None if "~cuda" in spec else spec.variants["cuda_arch"].value
+        xcompiler = "" if "~cuda" in spec else "-Xcompiler="
 
         # We need to add rpaths explicitly to allow proper export of link flags
-        # from within MFEM.
-
-        # Similar to spec[pkg].libs.ld_flags but prepends rpath flags too.
-        # Also does not add system library paths as defined by 'sys_lib_paths'
-        # above -- this is done to avoid issues like this:
-        # https://github.com/mfem/mfem/issues/1088.
-        def ld_flags_from_library_list(libs_list):
-            flags = [
-                "%s-rpath,%s" % (xlinker, dir)
-                for dir in libs_list.directories
-                if not is_sys_lib_path(dir)
-            ]
-            flags += ["-L%s" % dir for dir in libs_list.directories if not is_sys_lib_path(dir)]
-            flags += [libs_list.link_flags]
-            return " ".join(flags)
-
-        def ld_flags_from_dirs(pkg_dirs_list, pkg_libs_list):
-            flags = [
-                "%s-rpath,%s" % (xlinker, dir) for dir in pkg_dirs_list if not is_sys_lib_path(dir)
-            ]
-            flags += ["-L%s" % dir for dir in pkg_dirs_list if not is_sys_lib_path(dir)]
-            flags += ["-l%s" % lib for lib in pkg_libs_list]
-            return " ".join(flags)
+        # from within MFEM. We use the following two functions to do that.
+        ld_flags_from_library_list = self.ld_flags_from_library_list
+        ld_flags_from_dirs = self.ld_flags_from_dirs
 
         def find_optional_library(name, prefix):
             for shared in [True, False]:
@@ -619,6 +618,8 @@ class Mfem(Package, CudaPackage, ROCmPackage):
             "MFEM_USE_EXCEPTIONS=%s" % yes_no("+exceptions"),
             "MFEM_USE_MUMPS=%s" % yes_no("+mumps"),
         ]
+        if spec.satisfies("@4.7.0:"):
+            options += ["MFEM_PRECISION=%s" % spec.variants["precision"].value]
 
         # Determine C++ standard to use:
         cxxstd = None
@@ -632,6 +633,11 @@ class Mfem(Package, CudaPackage, ROCmPackage):
             cxxstd = "14"
         if self.spec.satisfies("^ginkgo"):
             cxxstd = "14"
+        cxxstd_req = spec.variants["cxxstd"].value
+        if cxxstd_req != "auto":
+            # Constraints for valid standard level should be imposed during
+            # concretization based on 'conflicts' or other directives.
+            cxxstd = cxxstd_req
         cxxstd_flag = None
         if cxxstd:
             if "+cuda" in spec:
@@ -639,6 +645,8 @@ class Mfem(Package, CudaPackage, ROCmPackage):
             else:
                 cxxstd_flag = getattr(self.compiler, "cxx" + cxxstd + "_flag")
 
+        cuda_arch = None if "~cuda" in spec else spec.variants["cuda_arch"].value
+
         cxxflags = spec.compiler_flags["cxxflags"].copy()
 
         if cxxflags:
@@ -944,7 +952,6 @@ class Mfem(Package, CudaPackage, ROCmPackage):
             options += ["HIP_CXX=%s" % spec["hip"].hipcc, "HIP_ARCH=%s" % amdgpu_target]
             hip_headers = HeaderList([])
             hip_libs = LibraryList([])
-            hip_libs += find_libraries("libamdhip64", spec["hip"].prefix.lib)
             # To use a C++ compiler that supports -xhip flag one can use
             # something like this:
             #   options += [
@@ -972,9 +979,27 @@ class Mfem(Package, CudaPackage, ROCmPackage):
                 hip_headers += spec["hipblas"].headers
             if "%cce" in spec:
                 # We assume the proper Cray CCE module (cce) is loaded:
-                craylibs_path = env["CRAYLIBS_" + machine().upper()]
-                craylibs = ["libmodules", "libfi", "libcraymath", "libf", "libu", "libcsup"]
+                proc = str(spec.target.family)
+                craylibs_var = "CRAYLIBS_" + proc.upper()
+                craylibs_path = env.get(craylibs_var, None)
+                if not craylibs_path:
+                    raise InstallError(
+                        f"The environment variable {craylibs_var} is not defined.\n"
+                        "\tMake sure the 'cce' module is in the compiler spec."
+                    )
+                craylibs = [
+                    "libmodules",
+                    "libfi",
+                    "libcraymath",
+                    "libf",
+                    "libu",
+                    "libcsup",
+                    "libpgas-shmem",
+                ]
                 hip_libs += find_libraries(craylibs, craylibs_path)
+                craylibs_path2 = join_path(craylibs_path, "../../../cce-clang", proc, "lib")
+                hip_libs += find_libraries("libunwind", craylibs_path2)
+
             if hip_headers:
                 options += ["HIP_OPT=%s" % hip_headers.cpp_flags]
             if hip_libs:
@@ -1018,9 +1043,17 @@ class Mfem(Package, CudaPackage, ROCmPackage):
             ]
 
         if "+umpire" in spec:
+            umpire = spec["umpire"]
+            umpire_opts = umpire.headers
+            umpire_libs = umpire.libs
+            if "^camp" in umpire:
+                umpire_opts += umpire["camp"].headers
+            if "^fmt" in umpire:
+                umpire_opts += umpire["fmt"].headers
+                umpire_libs += umpire["fmt"].libs
             options += [
-                "UMPIRE_OPT=-I%s" % spec["umpire"].prefix.include,
-                "UMPIRE_LIB=%s" % ld_flags_from_library_list(spec["umpire"].libs),
+                "UMPIRE_OPT=%s" % umpire_opts.cpp_flags,
+                "UMPIRE_LIB=%s" % ld_flags_from_library_list(umpire_libs),
             ]
 
         timer_ids = {"std": "0", "posix": "2", "mac": "4", "mpi": "6"}
@@ -1089,7 +1122,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
             hiop_libs = hiop.libs
             hiop_hdrs += spec["lapack"].headers + spec["blas"].headers
             hiop_libs += spec["lapack"].libs + spec["blas"].libs
-            hiop_opt_libs = ["magma", "umpire"]
+            hiop_opt_libs = ["magma", "umpire", "hipblas", "hiprand"]
             for opt_lib in hiop_opt_libs:
                 if "^" + opt_lib in hiop:
                     hiop_hdrs += hiop[opt_lib].headers
@@ -1105,6 +1138,8 @@ class Mfem(Package, CudaPackage, ROCmPackage):
                     camp = raja["camp"]
                     hiop_hdrs += camp.headers
                     hiop_libs += find_optional_library("libcamp", camp.prefix)
+            if hiop.satisfies("@0.6:+cuda"):
+                hiop_libs += LibraryList(["cublas", "curand"])
             options += [
                 "HIOP_OPT=%s" % hiop_hdrs.cpp_flags,
                 "HIOP_LIB=%s" % ld_flags_from_library_list(hiop_libs),
@@ -1121,6 +1156,10 @@ class Mfem(Package, CudaPackage, ROCmPackage):
                 "MUMPS_LIB=%s" % ld_flags_from_library_list(mumps.libs),
             ]
 
+        return options
+
+    def configure(self, spec, prefix):
+        options = self.get_make_config_options(spec, prefix)
         make("config", *options, parallel=False)
         make("info", parallel=False)
 
@@ -1281,3 +1320,46 @@ class Mfem(Package, CudaPackage, ROCmPackage):
             if os.access(f, os.R_OK):
                 return FileList(f)
         return FileList(find(self.prefix, "test.mk", recursive=True))
+
+    # See also find_system_libraries in lib/spack/llnl/util/filesystem.py
+    # where the similar list of paths is used.
+    sys_lib_paths = [
+        "/lib64",
+        "/lib",
+        "/usr/lib64",
+        "/usr/lib",
+        "/usr/local/lib64",
+        "/usr/local/lib",
+        "/usr/lib/x86_64-linux-gnu",
+    ]
+
+    def is_sys_lib_path(self, dir):
+        return dir in self.sys_lib_paths
+
+    @property
+    def xlinker(self):
+        return "-Wl," if "~cuda" in self.spec else "-Xlinker="
+
+    # Similar to spec[pkg].libs.ld_flags but prepends rpath flags too.
+    # Also does not add system library paths as defined by 'sys_lib_paths'
+    # above -- this is done to avoid issues like this:
+    # https://github.com/mfem/mfem/issues/1088.
+    def ld_flags_from_library_list(self, libs_list):
+        flags = [
+            "%s-rpath,%s" % (self.xlinker, dir)
+            for dir in libs_list.directories
+            if not self.is_sys_lib_path(dir)
+        ]
+        flags += ["-L%s" % dir for dir in libs_list.directories if not self.is_sys_lib_path(dir)]
+        flags += [libs_list.link_flags]
+        return " ".join(flags)
+
+    def ld_flags_from_dirs(self, pkg_dirs_list, pkg_libs_list):
+        flags = [
+            "%s-rpath,%s" % (self.xlinker, dir)
+            for dir in pkg_dirs_list
+            if not self.is_sys_lib_path(dir)
+        ]
+        flags += ["-L%s" % dir for dir in pkg_dirs_list if not self.is_sys_lib_path(dir)]
+        flags += ["-l%s" % lib for lib in pkg_libs_list]
+        return " ".join(flags)
diff --git a/var/spack/repos/builtin/packages/mfem/test_builds.sh b/var/spack/repos/builtin/packages/mfem/test_builds.sh
index cb658dd59c..be0d27bc0f 100755
--- a/var/spack/repos/builtin/packages/mfem/test_builds.sh
+++ b/var/spack/repos/builtin/packages/mfem/test_builds.sh
@@ -14,9 +14,9 @@ rocm_arch="gfx908"
 spack_jobs=''
 # spack_jobs='-j 128'
 
-mfem='mfem@4.6.0'${compiler}
+mfem='mfem@4.7.0'${compiler}
 # mfem_dev='mfem@develop'${compiler}
-mfem_dev='mfem@4.6.0'${compiler}
+mfem_dev='mfem@4.7.0'${compiler}
 
 backends='+occa+raja+libceed'
 backends_specs='^occa~cuda ^raja~openmp'
@@ -31,44 +31,60 @@ petsc_spec_rocm='^petsc+rocm+mumps'
 strumpack_spec='^strumpack~slate~openmp~cuda'
 strumpack_cuda_spec='^strumpack+cuda~slate~openmp'
 strumpack_rocm_spec='^strumpack+rocm~slate~openmp~cuda'
-# superlu specs with cuda and rocm
-superlu_cuda_spec='^superlu-dist+cuda'
-superlu_rocm_spec='^superlu-dist+rocm'
+# superlu specs with cpu, cuda and rocm
+# - v8.2.1 on CPU and GPU stalls in ex11p; works when superlu::PARMETIS is
+#   replaced with superlu::METIS_AT_PLUS_A, at least on CPU
+superlu_spec='^superlu-dist@8.1.2'
+superlu_cuda_spec='^superlu-dist@8.1.2+cuda'
+superlu_rocm_spec='^superlu-dist@8.1.2+rocm'
+# FMS spec
+fms_spec='^libfms+conduit'
 
 builds=(
     # preferred version:
     ${mfem}
     ${mfem}'~mpi~metis~zlib'
-    ${mfem}"$backends"'+superlu-dist+strumpack+suite-sparse+petsc+slepc+gslib \
-        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
-        '"$backends_specs $strumpack_spec $petsc_spec $conduit_spec"
+    # TODO: add back "+fms $fms_spec" when the FMS unit test is fixed
+    ${mfem}"$backends"'+superlu-dist+strumpack+mumps+suite-sparse+petsc+slepc \
+        +gslib+sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo \
+        +hiop \
+        '"$backends_specs $superlu_spec $strumpack_spec $petsc_spec"' \
+        '"$conduit_spec"
+    # TODO: add back "+fms $fms_spec" when the FMS unit test is fixed
     ${mfem}'~mpi \
         '"$backends"'+suite-sparse+sundials+gslib+mpfr+netcdf \
         +zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         '"$backends_specs $conduit_spec"' ^sundials~mpi'
+    ${mfem}' precision=single +mumps+petsc '"$petsc_spec"
 
     # develop version, shared builds:
     ${mfem_dev}'+shared~static'
     ${mfem_dev}'+shared~static~mpi~metis~zlib'
     # NOTE: Shared build with +gslib works on mac but not on linux
     # TODO: add back '+gslib' when the above NOTE is addressed.
+    # TODO: add back "+fms $fms_spec" when the FMS unit test is fixed
     ${mfem_dev}'+shared~static \
-        '"$backends"'+superlu-dist+strumpack+suite-sparse+petsc+slepc \
+        '"$backends"'+superlu-dist+strumpack+mumps+suite-sparse+petsc+slepc \
         +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
-        '"$backends_specs $strumpack_spec $petsc_spec $conduit_spec"
+        '"$backends_specs $superlu_spec $strumpack_spec $petsc_spec"' \
+        '"$conduit_spec"
     # NOTE: Shared build with +gslib works on mac but not on linux
     # TODO: add back '+gslib' when the above NOTE is addressed.
+    # TODO: add back "+fms $fms_spec" when the FMS unit test is fixed
     ${mfem_dev}'+shared~static~mpi \
         '"$backends"'+suite-sparse+sundials+mpfr+netcdf \
         +zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         '"$backends_specs $conduit_spec"' ^sundials~mpi'
+    ${mfem_dev}'+shared~static precision=single +mumps+petsc '"$petsc_spec"
 )
 
 builds2=(
     # preferred version
     ${mfem}"$backends $backends_specs"
-    ${mfem}'+superlu-dist'
+    ${mfem}' precision=single'
+    ${mfem}'+superlu-dist'" $superlu_spec"
     ${mfem}'+strumpack'" $strumpack_spec"
+    ${mfem}'+mumps'
     ${mfem}'+suite-sparse~mpi'
     ${mfem}'+suite-sparse'
     ${mfem}'+sundials~mpi ^sundials~mpi'
@@ -81,6 +97,8 @@ builds2=(
     ${mfem}'+gnutls'
     ${mfem}'+conduit~mpi'" $conduit_spec"
     ${mfem}'+conduit'" $conduit_spec"
+    # TODO: uncomment next line when the FMS unit test is fixed
+    # ${mfem}'+fms'" $fms_spec"
     ${mfem}'+umpire'
     ${mfem}'+petsc'" $petsc_spec"
     ${mfem}'+petsc+slepc'" $petsc_spec"
@@ -93,8 +111,10 @@ builds2=(
     #
     # develop version
     ${mfem_dev}"$backends $backends_specs"
-    ${mfem_dev}'+superlu-dist'
+    ${mfem_dev}' precision=single'
+    ${mfem_dev}'+superlu-dist'" $superlu_spec"
     ${mfem_dev}'+strumpack'" $strumpack_spec"
+    ${mfem_dev}'+mumps'
     ${mfem_dev}'+suite-sparse~mpi'
     ${mfem_dev}'+suite-sparse'
     ${mfem_dev}'+sundials~mpi ^sundials~mpi'
@@ -107,6 +127,8 @@ builds2=(
     ${mfem_dev}'+gnutls'
     ${mfem_dev}'+conduit~mpi'" $conduit_spec"
     ${mfem_dev}'+conduit'" $conduit_spec"
+    # TODO: uncomment next line when the FMS unit test is fixed
+    # ${mfem_dev}'+fms'" $fms_spec"
     ${mfem_dev}'+umpire'
     ${mfem_dev}'+petsc'" $petsc_spec"
     ${mfem_dev}'+petsc+slepc'" $petsc_spec"
@@ -134,25 +156,37 @@ builds_cuda=(
     # hypre without cuda:
     # NOTE: PETSc tests may need PETSC_OPTIONS="-use_gpu_aware_mpi 0"
     # TODO: restore '+libceed' when the libCEED CUDA unit tests take less time.
-    # TODO: remove "^hiop+shared" when the default static build is fixed.
     ${mfem}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \
         +superlu-dist+strumpack+suite-sparse+gslib+petsc+slepc \
-        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
-        ^raja+cuda+openmp ^hiop+shared'" $strumpack_cuda_spec"' \
+        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo \
+        ^raja+cuda+openmp'" $strumpack_cuda_spec"' \
         '"$superlu_cuda_spec $petsc_spec_cuda $conduit_spec"
 
+    ${mfem}'+cuda cuda_arch='"${cuda_arch}"' +raja+umpire'
+
+    # hiop needs older versions of raja, umpire, etc
+    # TODO: combine this spec with the above spec when the combined spec works.
+    ${mfem}'+cuda cuda_arch='"${cuda_arch}"' +hiop'
+
     # hypre with cuda:
     # TODO: restore '+libceed' when the libCEED CUDA unit tests take less time.
     # TODO: add back "+petsc+slepc $petsc_spec_cuda" when it works.
     # NOTE: PETSc tests may need PETSC_OPTIONS="-use_gpu_aware_mpi 0"
     # TODO: add back "+sundials" when it's supported with '^hypre+cuda'.
-    # TODO: remove "^hiop+shared" when the default static build is fixed.
     ${mfem}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \
         +superlu-dist+strumpack+suite-sparse+gslib \
-        +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
-        ^raja+cuda+openmp ^hiop+shared ^hypre+cuda \
+        +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo \
+        ^raja+cuda+openmp ^hypre+cuda \
         '" $strumpack_cuda_spec $superlu_cuda_spec $conduit_spec"
 
+    ${mfem}'+cuda cuda_arch='"${cuda_arch}"' +raja+umpire ^hypre+cuda'
+
+    # hiop needs older versions of raja, umpire, etc
+    # TODO: combine this spec with the above spec when the combined spec works.
+    ${mfem}'+cuda cuda_arch='"${cuda_arch}"' +hiop ^hypre+cuda'
+
+    ${mfem}' precision=single +cuda cuda_arch='"${cuda_arch}"' ^hypre+cuda'
+
     #
     # same builds as above with ${mfem_dev}
     #
@@ -171,24 +205,36 @@ builds_cuda=(
     # hypre without cuda:
     # NOTE: PETSc tests may need PETSC_OPTIONS="-use_gpu_aware_mpi 0"
     # TODO: restore '+libceed' when the libCEED CUDA unit tests take less time.
-    # TODO: remove "^hiop+shared" when the default static build is fixed.
     ${mfem_dev}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \
         +superlu-dist+strumpack+suite-sparse+gslib+petsc+slepc \
-        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
-        ^raja+cuda+openmp ^hiop+shared'" $strumpack_cuda_spec"' \
+        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo \
+        ^raja+cuda+openmp'" $strumpack_cuda_spec"' \
         '"$superlu_cuda_spec $petsc_spec_cuda $conduit_spec"
 
+    ${mfem_dev}'+cuda cuda_arch='"${cuda_arch}"' +raja+umpire'
+
+    # hiop needs older versions of raja, umpire, etc
+    # TODO: combine this spec with the above spec when the combined spec works.
+    ${mfem_dev}'+cuda cuda_arch='"${cuda_arch}"' +hiop'
+
     # hypre with cuda:
     # TODO: restore '+libceed' when the libCEED CUDA unit tests take less time.
     # TODO: add back "+petsc+slepc $petsc_spec_cuda" when it works.
     # NOTE: PETSc tests may need PETSC_OPTIONS="-use_gpu_aware_mpi 0"
     # TODO: add back "+sundials" when it's supported with '^hypre+cuda'.
-    # TODO: remove "^hiop+shared" when the default static build is fixed.
     ${mfem_dev}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \
         +superlu-dist+strumpack+suite-sparse+gslib \
-        +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
-        ^raja+cuda+openmp ^hiop+shared ^hypre+cuda \
+        +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo \
+        ^raja+cuda+openmp ^hypre+cuda \
         '"$strumpack_cuda_spec $superlu_cuda_spec $conduit_spec"
+
+    ${mfem_dev}'+cuda cuda_arch='"${cuda_arch}"' +raja+umpire ^hypre+cuda'
+
+    # hiop needs older versions of raja, umpire, etc
+    # TODO: combine this spec with the above spec when the combined spec works.
+    ${mfem_dev}'+cuda cuda_arch='"${cuda_arch}"' +hiop ^hypre+cuda'
+
+    ${mfem_dev}' precision=single +cuda cuda_arch='"${cuda_arch}"' ^hypre+cuda'
 )
 
 
@@ -204,27 +250,35 @@ builds_rocm=(
         ^raja+rocm~openmp ^occa~cuda~openmp ^hypre+rocm'
 
     # hypre without rocm:
-    # TODO: add back '+hiop' when it is no longer linked with tcmalloc* through
-    #       its magma dependency.
-    # TODO: add back '+ginkgo' when the Ginkgo example works.
     ${mfem}'+rocm+openmp+raja+occa+libceed amdgpu_target='"${rocm_arch}"' \
         +superlu-dist+strumpack+suite-sparse+gslib+petsc+slepc \
-        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
+        +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo \
         ^raja+rocm~openmp ^occa~cuda'" $strumpack_rocm_spec"' \
         '"$superlu_rocm_spec $petsc_spec_rocm $conduit_spec"
 
+    ${mfem}'+rocm amdgpu_target='"${rocm_arch}"' +raja+umpire'
+
+    # hiop needs older versions of raja, umpire, etc
+    # TODO: combine this spec with the above spec when the combined spec works.
+    ${mfem}'+rocm amdgpu_target='"${rocm_arch}"' +hiop'
+
     # hypre with rocm:
     # TODO: add back "+petsc+slepc $petsc_spec_rocm" when it works.
-    # TODO: add back '+hiop' when it is no longer linked with tcmalloc* through
-    #       its magma dependency.
-    # TODO: add back '+ginkgo' when the Ginkgo example works.
     # TODO: add back "+sundials" when it's supported with '^hypre+rocm'.
     ${mfem}'+rocm+openmp+raja+occa+libceed amdgpu_target='"${rocm_arch}"' \
         +superlu-dist+strumpack+suite-sparse+gslib \
-        +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
+        +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo \
         ^raja+rocm~openmp ^occa~cuda ^hypre+rocm \
         '"$strumpack_rocm_spec $superlu_rocm_spec $conduit_spec"
 
+    ${mfem}'+rocm amdgpu_target='"${rocm_arch}"' +raja+umpire ^hypre+rocm'
+
+    # hiop needs older versions of raja, umpire, etc
+    # TODO: combine this spec with the above spec when the combined spec works.
+    ${mfem}'+rocm amdgpu_target='"${rocm_arch}"' +hiop ^hypre+rocm'
+
+    ${mfem}' precision=single +rocm amdgpu_target='"${rocm_arch}"' ^hypre+rocm'
+
     #
     # same builds as above with ${mfem_dev}
     #
@@ -244,6 +298,8 @@ run_builds=("${builds[@]}" "${builds2[@]}")
 
 # PETSc CUDA tests on Lassen need this:
 # export PETSC_OPTIONS="-use_gpu_aware_mpi 0"
+# STRUMPACK forces "^openblas threads=openmp" when using openblas:
+export OMP_NUM_THREADS=1
 
 # spack files to clean in "$mfem_src_dir" when using 'dev-build'
 clean_files=(
diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py
index e1c5b1530e..2564d7f1ce 100644
--- a/var/spack/repos/builtin/packages/petsc/package.py
+++ b/var/spack/repos/builtin/packages/petsc/package.py
@@ -613,6 +613,10 @@ class Petsc(Package, CudaPackage, ROCmPackage):
         if "superlu-dist" in spec:
             if spec.satisfies("@3.10.3:3.15"):
                 options.append("--with-cxx-dialect=C++11")
+            if spec["superlu-dist"].satisfies("+rocm"):
+                # Suppress HIP header warning message, otherwise the PETSc
+                # configuration fails:
+                options.append("CXXPPFLAGS=-DROCM_NO_WRAPPER_HEADER_WARNING")
 
         if "+mkl-pardiso" in spec:
             options.append("--with-mkl_pardiso-dir=%s" % spec["mkl"].prefix)
diff --git a/var/spack/repos/builtin/packages/strumpack/package.py b/var/spack/repos/builtin/packages/strumpack/package.py
index 348a74947e..93b062eff1 100644
--- a/var/spack/repos/builtin/packages/strumpack/package.py
+++ b/var/spack/repos/builtin/packages/strumpack/package.py
@@ -3,8 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-from platform import machine
-
 from spack.package import *
 from spack.util.environment import set_env
 
@@ -177,7 +175,13 @@ class Strumpack(CMakePackage, CudaPackage, ROCmPackage):
 
         if "%cce" in spec:
             # Assume the proper Cray CCE module (cce) is loaded:
-            craylibs_path = env["CRAYLIBS_" + machine().upper()]
+            craylibs_var = "CRAYLIBS_" + str(spec.target.family).upper()
+            craylibs_path = env.get(craylibs_var, None)
+            if not craylibs_path:
+                raise InstallError(
+                    f"The environment variable {craylibs_var} is not defined.\n"
+                    "\tMake sure the 'cce' module is in the compiler spec."
+                )
             env.setdefault("LDFLAGS", "")
             env["LDFLAGS"] += " -Wl,-rpath," + craylibs_path
 
diff --git a/var/spack/repos/builtin/packages/suite-sparse/package.py b/var/spack/repos/builtin/packages/suite-sparse/package.py
index 3effe71cd9..70a1f92032 100644
--- a/var/spack/repos/builtin/packages/suite-sparse/package.py
+++ b/var/spack/repos/builtin/packages/suite-sparse/package.py
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import os.path
+
 from spack.package import *
 
 
@@ -189,6 +191,7 @@ class SuiteSparse(Package):
             # even though this fix is ugly
             f"BLAS={spec['blas'].libs.ld_flags + (' -lstdc++' if '@4.5.1' in spec else '')}",
             f"LAPACK={spec['lapack'].libs.ld_flags}",
+            f"JOBS={make_jobs}",
         ]
 
         # Recent versions require c11 but some demos do not get the c11 from
@@ -229,21 +232,45 @@ class SuiteSparse(Package):
             # Without CMAKE_LIBRARY_PATH defined, the CMake file in the
             # Mongoose directory finds libsuitesparseconfig.so in system
             # directories like /usr/lib.
-            make_args += [
-                f"CMAKE_OPTIONS=-DCMAKE_INSTALL_PREFIX={prefix}"
-                + f" -DCMAKE_LIBRARY_PATH={prefix.lib}"
-                + f" -DBLAS_ROOT={spec['blas'].prefix}"
-                + f" -DLAPACK_ROOT={spec['lapack'].prefix}"
+            cmake_args = [
+                f"-DCMAKE_INSTALL_PREFIX={prefix}",
+                f"-DCMAKE_LIBRARY_PATH={prefix.lib}",
+                f"-DBLAS_ROOT={spec['blas'].prefix}",
+                f"-DLAPACK_ROOT={spec['lapack'].prefix}",
                 # *_LIBRARIES is critical to pick up static
                 # libraries (if intended) and also to avoid
                 # unintentional system blas/lapack packages
-                + f" -DBLAS_LIBRARIES={spec['blas'].libs}"
-                + f" -DLAPACK_LIBRARIES={spec['lapack'].libs}"
+                f'-DBLAS_LIBRARIES="{";".join(spec["blas"].libs)}"',
+                f'-DLAPACK_LIBRARIES="{";".join(spec["lapack"].libs)}"',
+                "-DCMAKE_VERBOSE_MAKEFILE=ON",
             ]
+            if spec.satisfies("@:7.3"):
+                cmake_args += [
+                    f"-DNOPENMP={'OFF' if '+openmp' in spec else 'ON'}",
+                    f"-DENABLE_CUDA={'ON' if '+cuda' in spec else 'OFF'}",
+                ]
+            else:
+                cmake_args += [
+                    f"-DSUITESPARSE_USE_OPENMP={'ON' if '+openmp' in spec else 'OFF'}",
+                    f"-DSUITESPARSE_USE_CUDA={'ON' if '+cuda' in spec else 'OFF'}",
+                ]
+            make_args += [f"CMAKE_OPTIONS={' '.join(cmake_args)}"]
 
         if spec.satisfies("%gcc platform=darwin"):
             make_args += ["LDLIBS=-lm"]
 
+        if "%cce" in spec:
+            # Assume the proper Cray CCE module (cce) is loaded:
+            craylibs_var = "CRAYLIBS_" + str(spec.target.family).upper()
+            craylibs_path = env.get(craylibs_var, None)
+            if not craylibs_path:
+                raise InstallError(
+                    f"The environment variable {craylibs_var} is not defined.\n"
+                    "\tMake sure the 'cce' module is in the compiler spec."
+                )
+            env.setdefault("LDFLAGS", "")
+            env["LDFLAGS"] += " -Wl,-rpath," + craylibs_path
+
         make_args.append(f"INSTALL={prefix}")
 
         # Filter the targets we're interested in
@@ -278,6 +305,16 @@ class SuiteSparse(Package):
             make("-C", target, "library", *make_args)
             make("-C", target, "install", *make_args)
 
+        # Starting with v7.4.0 headers are installed in a subdirectory called
+        # 'suitesparse' by default. For backward compatibility, after
+        # installation, we create links for all files from 'suitesparse' in the
+        # containing directory, '<prefix>/include':
+        if spec.satisfies("@7.4:"):
+            with working_dir(prefix.include):
+                for f in find("suitesparse", "*", recursive=False):
+                    sf = os.path.basename(f)
+                    symlink(join_path("suitesparse", sf), sf)
+
     @run_after("install")
     def fix_darwin_install(self):
         # The shared libraries are not installed correctly on Darwin:
diff --git a/var/spack/repos/builtin/packages/zlib-ng/package.py b/var/spack/repos/builtin/packages/zlib-ng/package.py
index 6322d97696..576729c2ce 100644
--- a/var/spack/repos/builtin/packages/zlib-ng/package.py
+++ b/var/spack/repos/builtin/packages/zlib-ng/package.py
@@ -76,11 +76,12 @@ class ZlibNg(AutotoolsPackage, CMakePackage):
     def flag_handler(self, name, flags):
         if name == "cflags" and self.spec.satisfies("+pic build_system=autotools"):
             flags.append(self.compiler.cc_pic_flag)
+        if name == "ldflags" and self.spec.satisfies("%cce@17"):
+            flags.append("-Wl,--undefined-version")
         return (flags, None, None)
 
 
 class AutotoolsBuilder(autotools.AutotoolsBuilder):
-
     @run_before("configure")
     def pretend_gcc(self):
         # All nice things (PIC flags, symbol versioning) that happen to the compilers that are