mfem: patch @4.3.0 to support hypre up to v2.23.0 (#26640)

2 files changed, 577 insertions, 1 deletions

diff --git a/var/spack/repos/builtin/packages/mfem/mfem-4.3-hypre-2.23.0.patch b/var/spack/repos/builtin/packages/mfem/mfem-4.3-hypre-2.23.0.patch
new file mode 100644
index 0000000000..743e0c42dc
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mfem/mfem-4.3-hypre-2.23.0.patch
@@ -0,0 +1,575 @@
+diff --git a/fem/pfespace.cpp b/fem/pfespace.cpp
+index 1e2a47a5b..53964328e 100644
+--- a/fem/pfespace.cpp
++++ b/fem/pfespace.cpp
+@@ -1278,12 +1278,17 @@ const FiniteElement *ParFiniteElementSpace::GetFaceNbrFaceFE(int i) const
+ 
+ void ParFiniteElementSpace::Lose_Dof_TrueDof_Matrix()
+ {
++   P -> StealData();
++#if MFEM_HYPRE_VERSION <= 22200
+    hypre_ParCSRMatrix *csrP = (hypre_ParCSRMatrix*)(*P);
+    hypre_ParCSRMatrixOwnsRowStarts(csrP) = 1;
+    hypre_ParCSRMatrixOwnsColStarts(csrP) = 1;
+-   P -> StealData();
+    dof_offsets.LoseData();
+    tdof_offsets.LoseData();
++#else
++   dof_offsets.DeleteAll();
++   tdof_offsets.DeleteAll();
++#endif
+ }
+ 
+ void ParFiniteElementSpace::ConstructTrueDofs()
+diff --git a/linalg/complex_operator.cpp b/linalg/complex_operator.cpp
+index 1f42ab425..8a2f8e580 100644
+--- a/linalg/complex_operator.cpp
++++ b/linalg/complex_operator.cpp
+@@ -787,10 +787,16 @@ HypreParMatrix * ComplexHypreParMatrix::GetSystemMatrix() const
+                                            2 * num_cols_offd, cmap,
+                                            true);
+ 
++#if MFEM_HYPRE_VERSION <= 22200
+    // Give the new matrix ownership of row_starts and col_starts
+    hypre_ParCSRMatrix *hA = (hypre_ParCSRMatrix*)(*A);
++
+    hypre_ParCSRMatrixSetRowStartsOwner(hA,1);
+    hypre_ParCSRMatrixSetColStartsOwner(hA,1);
++#else
++   mfem_hypre_TFree_host(row_starts);
++   mfem_hypre_TFree_host(col_starts);
++#endif
+ 
+    return A;
+ }
+diff --git a/linalg/hypre.cpp b/linalg/hypre.cpp
+index 3f393e39a..6ca80e987 100644
+--- a/linalg/hypre.cpp
++++ b/linalg/hypre.cpp
+@@ -88,7 +88,9 @@ HypreParVector::HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size,
+ {
+    x = hypre_ParVectorCreate(comm,glob_size,col);
+    hypre_ParVectorInitialize(x);
++#if MFEM_HYPRE_VERSION <= 22200
+    hypre_ParVectorSetPartitioningOwner(x,0);
++#endif
+    // The data will be destroyed by hypre (this is the default)
+    hypre_ParVectorSetDataOwner(x,1);
+    hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1);
+@@ -105,7 +107,9 @@ HypreParVector::HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size,
+    hypre_ParVectorSetDataOwner(x,1); // owns the seq vector
+    hypre_Vector *x_loc = hypre_ParVectorLocalVector(x);
+    hypre_SeqVectorSetDataOwner(x_loc,0);
++#if MFEM_HYPRE_VERSION <= 22200
+    hypre_ParVectorSetPartitioningOwner(x,0);
++#endif
+    double tmp = 0.0;
+    hypre_VectorData(x_loc) = &tmp;
+ #ifdef HYPRE_USING_CUDA
+@@ -128,7 +132,9 @@ HypreParVector::HypreParVector(const HypreParVector &y) : Vector()
+    x = hypre_ParVectorCreate(y.x -> comm, y.x -> global_size,
+                              y.x -> partitioning);
+    hypre_ParVectorInitialize(x);
++#if MFEM_HYPRE_VERSION <= 22200
+    hypre_ParVectorSetPartitioningOwner(x,0);
++#endif
+    hypre_ParVectorSetDataOwner(x,1);
+    hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1);
+    _SetDataAndSize_();
+@@ -162,7 +168,9 @@ HypreParVector::HypreParVector(ParFiniteElementSpace *pfes)
+    x = hypre_ParVectorCreate(pfes->GetComm(), pfes->GlobalTrueVSize(),
+                              pfes->GetTrueDofOffsets());
+    hypre_ParVectorInitialize(x);
++#if MFEM_HYPRE_VERSION <= 22200
+    hypre_ParVectorSetPartitioningOwner(x,0);
++#endif
+    // The data will be destroyed by hypre (this is the default)
+    hypre_ParVectorSetDataOwner(x,1);
+    hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1);
+@@ -683,8 +691,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, HYPRE_BigInt glob_size,
+    A = hypre_ParCSRMatrixCreate(comm, glob_size, glob_size, row_starts,
+                                 row_starts, 0, diag->NumNonZeroElems(), 0);
+    hypre_ParCSRMatrixSetDataOwner(A,1);
++#if MFEM_HYPRE_VERSION <= 22200
+    hypre_ParCSRMatrixSetRowStartsOwner(A,0);
+    hypre_ParCSRMatrixSetColStartsOwner(A,0);
++#endif
+ 
+    hypre_CSRMatrixSetDataOwner(A->diag,0);
+    diagOwner = CopyCSR(diag, mem_diag, A->diag, false);
+@@ -726,8 +736,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
+                                 row_starts, col_starts,
+                                 0, diag->NumNonZeroElems(), 0);
+    hypre_ParCSRMatrixSetDataOwner(A,1);
++#if MFEM_HYPRE_VERSION <= 22200
+    hypre_ParCSRMatrixSetRowStartsOwner(A,0);
+    hypre_ParCSRMatrixSetColStartsOwner(A,0);
++#endif
+ 
+    hypre_CSRMatrixSetDataOwner(A->diag,0);
+    diagOwner = CopyCSR(diag, mem_diag, A->diag, false);
+@@ -770,8 +782,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
+                                 offd->Width(), diag->NumNonZeroElems(),
+                                 offd->NumNonZeroElems());
+    hypre_ParCSRMatrixSetDataOwner(A,1);
++#if MFEM_HYPRE_VERSION <= 22200
+    hypre_ParCSRMatrixSetRowStartsOwner(A,0);
+    hypre_ParCSRMatrixSetColStartsOwner(A,0);
++#endif
+ 
+    hypre_CSRMatrixSetDataOwner(A->diag,0);
+    diagOwner = CopyCSR(diag, mem_diag, A->diag, own_diag_offd);
+@@ -817,8 +831,10 @@ HypreParMatrix::HypreParMatrix(
+    A = hypre_ParCSRMatrixCreate(comm, global_num_rows, global_num_cols,
+                                 row_starts, col_starts, offd_num_cols, 0, 0);
+    hypre_ParCSRMatrixSetDataOwner(A,1);
++#if MFEM_HYPRE_VERSION <= 22200
+    hypre_ParCSRMatrixSetRowStartsOwner(A,0);
+    hypre_ParCSRMatrixSetColStartsOwner(A,0);
++#endif
+ 
+    HYPRE_Int local_num_rows = hypre_CSRMatrixNumRows(A->diag);
+ 
+@@ -931,8 +947,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
+    A = hypre_ParCSRMatrixCreate(comm, global_num_rows, global_num_cols,
+                                 row_starts, col_starts, 0, nnz, 0);
+    hypre_ParCSRMatrixSetDataOwner(A,1);
++#if MFEM_HYPRE_VERSION <= 22200
+    hypre_ParCSRMatrixSetRowStartsOwner(A,0);
+    hypre_ParCSRMatrixSetColStartsOwner(A,0);
++#endif
+ 
+    hypre_CSRMatrixSetDataOwner(A->diag,0);
+    diagOwner = CopyBoolCSR(diag, mem_diag, A->diag);
+@@ -989,8 +1007,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int id, int np,
+    }
+ 
+    hypre_ParCSRMatrixSetDataOwner(A,1);
++#if MFEM_HYPRE_VERSION <= 22200
+    hypre_ParCSRMatrixSetRowStartsOwner(A,0);
+    hypre_ParCSRMatrixSetColStartsOwner(A,0);
++#endif
+ 
+    mem_diag.data.New(diag_nnz);
+    for (HYPRE_Int i = 0; i < diag_nnz; i++)
+@@ -1177,6 +1197,13 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int nrows,
+    {
+       hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A));
+    }
++#if MFEM_HYPRE_VERSION > 22200
++   mfem_hypre_TFree_host(row_starts);
++   if (rows != cols)
++   {
++      mfem_hypre_TFree_host(col_starts);
++   }
++#endif
+    hypre_MatvecCommPkgCreate(A);
+ 
+    height = GetNumRows();
+@@ -1263,6 +1290,7 @@ void HypreParMatrix::SetOwnerFlags(signed char diag, signed char offd,
+ 
+ void HypreParMatrix::CopyRowStarts()
+ {
++#if MFEM_HYPRE_VERSION <= 22200
+    if (!A || hypre_ParCSRMatrixOwnsRowStarts(A) ||
+        (hypre_ParCSRMatrixRowStarts(A) == hypre_ParCSRMatrixColStarts(A) &&
+         hypre_ParCSRMatrixOwnsColStarts(A)))
+@@ -1297,10 +1325,12 @@ void HypreParMatrix::CopyRowStarts()
+       hypre_ParCSRMatrixColStarts(A) = new_row_starts;
+       hypre_ParCSRMatrixOwnsColStarts(A) = 0;
+    }
++#endif
+ }
+ 
+ void HypreParMatrix::CopyColStarts()
+ {
++#if MFEM_HYPRE_VERSION <= 22200
+    if (!A || hypre_ParCSRMatrixOwnsColStarts(A) ||
+        (hypre_ParCSRMatrixRowStarts(A) == hypre_ParCSRMatrixColStarts(A) &&
+         hypre_ParCSRMatrixOwnsRowStarts(A)))
+@@ -1339,6 +1369,7 @@ void HypreParMatrix::CopyColStarts()
+    {
+       hypre_ParCSRMatrixOwnsColStarts(A) = 1;
+    }
++#endif
+ }
+ 
+ void HypreParMatrix::GetDiag(Vector &diag) const
+@@ -1490,6 +1521,10 @@ HypreParMatrix * HypreParMatrix::Transpose() const
+ HypreParMatrix *HypreParMatrix::ExtractSubmatrix(const Array<int> &indices,
+                                                  double threshold) const
+ {
++   // hypre_ParCSRMatrixExtractSubmatrixFC works on host only, so we move this
++   // matrix to host, temporarily:
++   HostRead();
++
+    if (!(A->comm))
+    {
+       hypre_MatvecCommPkgCreate(A);
+@@ -1501,15 +1536,28 @@ HypreParMatrix *HypreParMatrix::ExtractSubmatrix(const Array<int> &indices,
+    int local_num_vars = hypre_CSRMatrixNumRows(hypre_ParCSRMatrixDiag(A));
+ 
+    // Form hypre CF-splitting array designating submatrix as F-points (-1)
++#ifdef hypre_IntArrayData
++   // hypre_BoomerAMGCoarseParms needs CF_marker to be hypre_IntArray *
++   hypre_IntArray *CF_marker;
++
++   CF_marker = hypre_IntArrayCreate(local_num_vars);
++   hypre_IntArrayInitialize_v2(CF_marker, HYPRE_MEMORY_HOST);
++   hypre_IntArraySetConstantValues(CF_marker, 1);
++#else
+    Array<HYPRE_Int> CF_marker(local_num_vars);
+    CF_marker = 1;
++#endif
+    for (int j=0; j<indices.Size(); j++)
+    {
+       if (indices[j] > local_num_vars)
+       {
+          MFEM_WARNING("WARNING : " << indices[j] << " > " << local_num_vars);
+       }
++#ifdef hypre_IntArrayData
++      hypre_IntArrayData(CF_marker)[indices[j]] = -1;
++#else
+       CF_marker[indices[j]] = -1;
++#endif
+    }
+ 
+    // Construct cpts_global array on hypre matrix structure
+@@ -1518,10 +1566,22 @@ HypreParMatrix *HypreParMatrix::ExtractSubmatrix(const Array<int> &indices,
+                               CF_marker, NULL, &cpts_global);
+ 
+    // Extract submatrix into *submat
++#ifdef hypre_IntArrayData
++   hypre_ParCSRMatrixExtractSubmatrixFC(A, hypre_IntArrayData(CF_marker),
++                                        cpts_global, "FF", &submat,
++                                        threshold);
++#else
+    hypre_ParCSRMatrixExtractSubmatrixFC(A, CF_marker, cpts_global,
+                                         "FF", &submat, threshold);
++#endif
+ 
+    mfem_hypre_TFree(cpts_global);
++#ifdef hypre_IntArrayData
++   hypre_IntArrayDestroy(CF_marker);
++#endif
++
++   HypreRead(); // restore the matrix location to the default hypre location
++
+    return new HypreParMatrix(submat);
+ }
+ #endif
+@@ -1791,9 +1851,14 @@ HypreParMatrix* HypreParMatrix::LeftDiagMult(const SparseMatrix &D,
+                          DA_diag, DA_offd, new_col_map_offd,
+                          own_diag_offd);
+ 
++#if MFEM_HYPRE_VERSION <= 22200
+    // Give ownership of row_starts, col_starts, and col_map_offd to DA
+    hypre_ParCSRMatrixSetRowStartsOwner(DA->A, 1);
+    hypre_ParCSRMatrixSetColStartsOwner(DA->A, 1);
++#else
++   mfem_hypre_TFree_host(new_row_starts);
++   mfem_hypre_TFree_host(new_col_starts);
++#endif
+    DA->colMapOwner = 1;
+ 
+    return DA;
+@@ -1948,18 +2013,22 @@ void HypreParMatrix::Threshold(double threshold)
+    row_starts = hypre_ParCSRMatrixRowStarts(A);
+    col_starts = hypre_ParCSRMatrixColStarts(A);
+ 
++#if MFEM_HYPRE_VERSION <= 22200
+    bool old_owns_row = hypre_ParCSRMatrixOwnsRowStarts(A);
+    bool old_owns_col = hypre_ParCSRMatrixOwnsColStarts(A);
++#endif
+    HYPRE_BigInt global_num_rows = hypre_ParCSRMatrixGlobalNumRows(A);
+    HYPRE_BigInt global_num_cols = hypre_ParCSRMatrixGlobalNumCols(A);
+    parcsr_A_ptr = hypre_ParCSRMatrixCreate(comm, global_num_rows,
+                                            global_num_cols,
+                                            row_starts, col_starts,
+                                            0, 0, 0);
++#if MFEM_HYPRE_VERSION <= 22200
+    hypre_ParCSRMatrixOwnsRowStarts(parcsr_A_ptr) = old_owns_row;
+    hypre_ParCSRMatrixOwnsColStarts(parcsr_A_ptr) = old_owns_col;
+    hypre_ParCSRMatrixOwnsRowStarts(A) = 0;
+    hypre_ParCSRMatrixOwnsColStarts(A) = 0;
++#endif
+ 
+    csr_A = hypre_MergeDiagAndOffd(A);
+ 
+@@ -1994,7 +2063,12 @@ void HypreParMatrix::Threshold(double threshold)
+ 
+    hypre_ParCSRMatrixSetNumNonzeros(A);
+    /* Make sure that the first entry in each row is the diagonal one. */
++#if MFEM_HYPRE_VERSION <= 22200
+    if (row_starts == col_starts)
++#else
++   if ((row_starts[0] == col_starts[0]) &&
++       (row_starts[1] == col_starts[1]))
++#endif
+    {
+       hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A));
+    }
+@@ -2310,7 +2384,7 @@ inline void delete_hypre_ParCSRMatrixColMapOffd(hypre_ParCSRMatrix *A)
+ {
+    HYPRE_BigInt  *A_col_map_offd = hypre_ParCSRMatrixColMapOffd(A);
+    int size = hypre_CSRMatrixNumCols(hypre_ParCSRMatrixOffd(A));
+-   Memory<HYPRE_Int>(A_col_map_offd, size, true).Delete();
++   Memory<HYPRE_BigInt>(A_col_map_offd, size, true).Delete();
+ }
+ 
+ void HypreParMatrix::Destroy()
+@@ -2503,11 +2577,14 @@ HypreParMatrix * RAP(const HypreParMatrix *A, const HypreParMatrix *P)
+       // hypre_ParCSRMatrixRAPKT
+    }
+ #else
++#if MFEM_HYPRE_VERSION <= 22200
+    HYPRE_Int P_owns_its_col_starts =
+       hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*P));
++#endif
+ 
+    hypre_BoomerAMGBuildCoarseOperator(*P,*A,*P,&rap);
+ 
++#if MFEM_HYPRE_VERSION <= 22200
+    /* Warning: hypre_BoomerAMGBuildCoarseOperator steals the col_starts
+       from P (even if it does not own them)! */
+    hypre_ParCSRMatrixSetRowStartsOwner(rap,0);
+@@ -2516,6 +2593,7 @@ HypreParMatrix * RAP(const HypreParMatrix *A, const HypreParMatrix *P)
+    {
+       hypre_ParCSRMatrixSetColStartsOwner(*P, 1);
+    }
++#endif
+ #endif
+ 
+    hypre_ParCSRMatrixSetNumNonzeros(rap);
+@@ -2536,13 +2614,16 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A,
+       hypre_ParCSRMatrixDestroy(Q);
+    }
+ #else
++#if MFEM_HYPRE_VERSION <= 22200
+    HYPRE_Int P_owns_its_col_starts =
+       hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*P));
+    HYPRE_Int Rt_owns_its_col_starts =
+       hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*Rt));
++#endif
+ 
+    hypre_BoomerAMGBuildCoarseOperator(*Rt,*A,*P,&rap);
+ 
++#if MFEM_HYPRE_VERSION <= 22200
+    /* Warning: hypre_BoomerAMGBuildCoarseOperator steals the col_starts
+       from Rt and P (even if they do not own them)! */
+    hypre_ParCSRMatrixSetRowStartsOwner(rap,0);
+@@ -2555,6 +2636,7 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A,
+    {
+       hypre_ParCSRMatrixSetColStartsOwner(*Rt, 1);
+    }
++#endif
+ #endif
+ 
+    hypre_ParCSRMatrixSetNumNonzeros(rap);
+@@ -3144,8 +3226,12 @@ void HypreSmoother::SetOperator(const Operator &op)
+       }
+       else
+       {
++#if MFEM_HYPRE_VERSION <= 22200
+          min_eig_est = 0;
+          hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est);
++#else
++         hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est, &min_eig_est);
++#endif
+       }
+       Z = new HypreParVector(*A);
+    }
+@@ -3159,8 +3245,12 @@ void HypreSmoother::SetOperator(const Operator &op)
+       }
+       else
+       {
++#if MFEM_HYPRE_VERSION <= 22200
+          min_eig_est = 0;
+          hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est);
++#else
++         hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est, &min_eig_est);
++#endif
+       }
+ 
+       // The Taubin and FIR polynomials are defined on [0, 2]
+@@ -4472,9 +4562,8 @@ void HypreBoomerAMG::SetSystemsOptions(int dim, bool order_bynodes)
+    // HYPRE_BoomerAMGSetDofFunc as in the following code.
+    if (order_bynodes)
+    {
+-      // hypre actually deletes the following pointer in HYPRE_BoomerAMGDestroy,
+-      // so we don't need to track it
+-      HYPRE_Int *mapping = mfem_hypre_CTAlloc_host(HYPRE_Int, height);
++      // Generate DofFunc mapping on the host
++      HYPRE_Int *h_mapping = mfem_hypre_CTAlloc_host(HYPRE_Int, height);
+       int h_nnodes = height / dim; // nodes owned in linear algebra (not fem)
+       MFEM_VERIFY(height % dim == 0, "Ordering does not work as claimed!");
+       int k = 0;
+@@ -4482,9 +4571,24 @@ void HypreBoomerAMG::SetSystemsOptions(int dim, bool order_bynodes)
+       {
+          for (int j = 0; j < h_nnodes; ++j)
+          {
+-            mapping[k++] = i;
++            h_mapping[k++] = i;
+          }
+       }
++
++      // After the addition of hypre_IntArray, mapping is assumed
++      // to be a device pointer. Previously, it was assumed to be
++      // a host pointer.
++#if defined(hypre_IntArrayData) && defined(HYPRE_USING_GPU)
++      HYPRE_Int *mapping = mfem_hypre_CTAlloc(HYPRE_Int, height);
++      hypre_TMemcpy(mapping, h_mapping, HYPRE_Int, height,
++                    HYPRE_MEMORY_DEVICE, HYPRE_MEMORY_HOST);
++      mfem_hypre_TFree_host(h_mapping);
++#else
++      HYPRE_Int *mapping = h_mapping;
++#endif
++
++      // hypre actually deletes the mapping pointer in HYPRE_BoomerAMGDestroy,
++      // so we don't need to track it
+       HYPRE_BoomerAMGSetDofFunc(amg_precond, mapping);
+    }
+ 
+@@ -4529,7 +4633,8 @@ void HypreBoomerAMG::RecomputeRBMs()
+ 
+       rbms.SetSize(nrbms);
+       gf_rbms.SetSize(nrbms);
+-      gf_rbms[0] = rbms_rxy.ParallelAverage();
++      gf_rbms[0] = fespace->NewTrueDofVector();
++      rbms_rxy.GetTrueDofs(*gf_rbms[0]);
+    }
+    else if (dim == 3)
+    {
+@@ -4548,9 +4653,12 @@ void HypreBoomerAMG::RecomputeRBMs()
+ 
+       rbms.SetSize(nrbms);
+       gf_rbms.SetSize(nrbms);
+-      gf_rbms[0] = rbms_rxy.ParallelAverage();
+-      gf_rbms[1] = rbms_ryz.ParallelAverage();
+-      gf_rbms[2] = rbms_rzx.ParallelAverage();
++      gf_rbms[0] = fespace->NewTrueDofVector();
++      gf_rbms[1] = fespace->NewTrueDofVector();
++      gf_rbms[2] = fespace->NewTrueDofVector();
++      rbms_rxy.GetTrueDofs(*gf_rbms[0]);
++      rbms_ryz.GetTrueDofs(*gf_rbms[1]);
++      rbms_rzx.GetTrueDofs(*gf_rbms[2]);
+    }
+    else
+    {
+@@ -4997,11 +5105,11 @@ HypreADS::HypreADS(const HypreParMatrix &A, ParFiniteElementSpace *face_fespace)
+ void HypreADS::Init(ParFiniteElementSpace *face_fespace)
+ {
+    int cycle_type       = 11;
+-   int rlx_type         = 2;
+    int rlx_sweeps       = 1;
+    double rlx_weight    = 1.0;
+    double rlx_omega     = 1.0;
+ #ifndef HYPRE_USING_CUDA
++   int rlx_type         = 2;
+    int amg_coarsen_type = 10;
+    int amg_agg_levels   = 1;
+    int amg_rlx_type     = 8;
+@@ -5009,6 +5117,7 @@ void HypreADS::Init(ParFiniteElementSpace *face_fespace)
+    int amg_interp_type  = 6;
+    int amg_Pmax         = 4;
+ #else
++   int rlx_type         = 1;
+    int amg_coarsen_type = 8;
+    int amg_agg_levels   = 0;
+    int amg_rlx_type     = 18;
+@@ -5592,16 +5701,18 @@ HypreLOBPCG::OperatorMatvec( void *matvec_data,
+ 
+    Operator *Aop = (Operator*)A;
+ 
+-   int width = Aop->Width();
+-
+    hypre_ParVector * xPar = (hypre_ParVector *)x;
+    hypre_ParVector * yPar = (hypre_ParVector *)y;
+ 
+-   Vector xVec(xPar->local_vector->data, width);
+-   Vector yVec(yPar->local_vector->data, width);
++   HypreParVector xVec(xPar);
++   HypreParVector yVec(yPar);
+ 
+    Aop->Mult( xVec, yVec );
+ 
++   // Move data back to hypre's device memory location in case the above Mult
++   // operation moved it to host.
++   yVec.HypreReadWrite();
++
+    return 0;
+ }
+ 
+@@ -5617,19 +5728,20 @@ HypreLOBPCG::PrecondSolve(void *solver,
+                           void *b,
+                           void *x)
+ {
+-   Solver   *PC = (Solver*)solver;
+-   Operator *OP = (Operator*)A;
+-
+-   int width = OP->Width();
++   Solver *PC = (Solver*)solver;
+ 
+    hypre_ParVector * bPar = (hypre_ParVector *)b;
+    hypre_ParVector * xPar = (hypre_ParVector *)x;
+ 
+-   Vector bVec(bPar->local_vector->data, width);
+-   Vector xVec(xPar->local_vector->data, width);
++   HypreParVector bVec(bPar);
++   HypreParVector xVec(xPar);
+ 
+    PC->Mult( bVec, xVec );
+ 
++   // Move data back to hypre's device memory location in case the above Mult
++   // operation moved it to host.
++   xVec.HypreReadWrite();
++
+    return 0;
+ }
+ 
+diff --git a/linalg/hypre_parcsr.cpp b/linalg/hypre_parcsr.cpp
+index 5f1f50f3f..1c1ba1640 100644
+--- a/linalg/hypre_parcsr.cpp
++++ b/linalg/hypre_parcsr.cpp
+@@ -508,8 +508,10 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
+                                   hypre_ParCSRMatrixColStarts(A),
+                                   0, 0, 0);
+ 
++#if MFEM_HYPRE_VERSION <= 22200
+    hypre_ParCSRMatrixSetRowStartsOwner(*Ae, 0);
+    hypre_ParCSRMatrixSetColStartsOwner(*Ae, 0);
++#endif
+ 
+    hypre_CSRMatrix *Ae_diag = hypre_ParCSRMatrixDiag(*Ae);
+    hypre_CSRMatrix *Ae_offd = hypre_ParCSRMatrixOffd(*Ae);
+@@ -1002,10 +1004,17 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,
+ 
+       hypre_ParCSRMatrixOwnsData(blocks[i]) = 1;
+ 
++#if MFEM_HYPRE_VERSION <= 22200
+       /* only the first block will own the row/col_starts */
+       hypre_ParCSRMatrixOwnsRowStarts(blocks[i]) = !i;
+       hypre_ParCSRMatrixOwnsColStarts(blocks[i]) = !i;
++#endif
+    }
++
++#if MFEM_HYPRE_VERSION > 22200
++   mfem_hypre_TFree_host(row_starts);
++   mfem_hypre_TFree_host(col_starts);
++#endif
+ }
+ 
+ /* Based on hypre_CSRMatrixMatvec in hypre's csr_matvec.c */
+@@ -1916,9 +1925,12 @@ hypre_ParCSRMatrixAdd(hypre_ParCSRMatrix *A,
+ 
+    /* C owns diag, offd, and cmap. */
+    hypre_ParCSRMatrixSetDataOwner(C, 1);
++
++#if MFEM_HYPRE_VERSION <= 22200
+    /* C does not own row and column starts. */
+    hypre_ParCSRMatrixSetRowStartsOwner(C, 0);
+    hypre_ParCSRMatrixSetColStartsOwner(C, 0);
++#endif
+ 
+    return C;
+ }
diff --git a/var/spack/repos/builtin/packages/mfem/package.py b/var/spack/repos/builtin/packages/mfem/package.py
index f1acb85db8..fff1564140 100644
--- a/var/spack/repos/builtin/packages/mfem/package.py
+++ b/var/spack/repos/builtin/packages/mfem/package.py
@@ -200,7 +200,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     depends_on('mpi', when='+mpi')
     depends_on('hypre@2.10.0:2.13', when='@:3.3+mpi')
     depends_on('hypre@:2.20.0', when='@3.4:4.2+mpi')
-    depends_on('hypre@:2.22.0', when='@4.3.0+mpi')
+    depends_on('hypre@:2.23.0', when='@4.3.0+mpi')
     depends_on('hypre', when='+mpi')
 
     depends_on('metis', when='+metis')
@@ -300,6 +300,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     patch('mfem-4.2-umpire.patch', when='@4.2.0+umpire')
     patch('mfem-4.2-slepc.patch', when='@4.2.0+slepc')
     patch('mfem-4.2-petsc-3.15.0.patch', when='@4.2.0+petsc ^petsc@3.15.0:')
+    patch('mfem-4.3-hypre-2.23.0.patch', when='@4.3.0')
 
     # Patch to fix MFEM makefile syntax error. See
     # https://github.com/mfem/mfem/issues/1042 for the bug report and