diff --git a/fem/pfespace.cpp b/fem/pfespace.cpp index 1e2a47a5b..53964328e 100644 --- a/fem/pfespace.cpp +++ b/fem/pfespace.cpp @@ -1278,12 +1278,17 @@ const FiniteElement *ParFiniteElementSpace::GetFaceNbrFaceFE(int i) const void ParFiniteElementSpace::Lose_Dof_TrueDof_Matrix() { + P -> StealData(); +#if MFEM_HYPRE_VERSION <= 22200 hypre_ParCSRMatrix *csrP = (hypre_ParCSRMatrix*)(*P); hypre_ParCSRMatrixOwnsRowStarts(csrP) = 1; hypre_ParCSRMatrixOwnsColStarts(csrP) = 1; - P -> StealData(); dof_offsets.LoseData(); tdof_offsets.LoseData(); +#else + dof_offsets.DeleteAll(); + tdof_offsets.DeleteAll(); +#endif } void ParFiniteElementSpace::ConstructTrueDofs() diff --git a/linalg/complex_operator.cpp b/linalg/complex_operator.cpp index 1f42ab425..8a2f8e580 100644 --- a/linalg/complex_operator.cpp +++ b/linalg/complex_operator.cpp @@ -787,10 +787,16 @@ HypreParMatrix * ComplexHypreParMatrix::GetSystemMatrix() const 2 * num_cols_offd, cmap, true); +#if MFEM_HYPRE_VERSION <= 22200 // Give the new matrix ownership of row_starts and col_starts hypre_ParCSRMatrix *hA = (hypre_ParCSRMatrix*)(*A); + hypre_ParCSRMatrixSetRowStartsOwner(hA,1); hypre_ParCSRMatrixSetColStartsOwner(hA,1); +#else + mfem_hypre_TFree_host(row_starts); + mfem_hypre_TFree_host(col_starts); +#endif return A; } diff --git a/linalg/hypre.cpp b/linalg/hypre.cpp index 3f393e39a..6ca80e987 100644 --- a/linalg/hypre.cpp +++ b/linalg/hypre.cpp @@ -88,7 +88,9 @@ HypreParVector::HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size, { x = hypre_ParVectorCreate(comm,glob_size,col); hypre_ParVectorInitialize(x); +#if MFEM_HYPRE_VERSION <= 22200 hypre_ParVectorSetPartitioningOwner(x,0); +#endif // The data will be destroyed by hypre (this is the default) hypre_ParVectorSetDataOwner(x,1); hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1); @@ -105,7 +107,9 @@ HypreParVector::HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size, hypre_ParVectorSetDataOwner(x,1); // owns the seq vector hypre_Vector *x_loc = hypre_ParVectorLocalVector(x); hypre_SeqVectorSetDataOwner(x_loc,0); +#if MFEM_HYPRE_VERSION <= 22200 hypre_ParVectorSetPartitioningOwner(x,0); +#endif double tmp = 0.0; hypre_VectorData(x_loc) = &tmp; #ifdef HYPRE_USING_CUDA @@ -128,7 +132,9 @@ HypreParVector::HypreParVector(const HypreParVector &y) : Vector() x = hypre_ParVectorCreate(y.x -> comm, y.x -> global_size, y.x -> partitioning); hypre_ParVectorInitialize(x); +#if MFEM_HYPRE_VERSION <= 22200 hypre_ParVectorSetPartitioningOwner(x,0); +#endif hypre_ParVectorSetDataOwner(x,1); hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1); _SetDataAndSize_(); @@ -162,7 +168,9 @@ HypreParVector::HypreParVector(ParFiniteElementSpace *pfes) x = hypre_ParVectorCreate(pfes->GetComm(), pfes->GlobalTrueVSize(), pfes->GetTrueDofOffsets()); hypre_ParVectorInitialize(x); +#if MFEM_HYPRE_VERSION <= 22200 hypre_ParVectorSetPartitioningOwner(x,0); +#endif // The data will be destroyed by hypre (this is the default) hypre_ParVectorSetDataOwner(x,1); hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1); @@ -683,8 +691,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, HYPRE_BigInt glob_size, A = hypre_ParCSRMatrixCreate(comm, glob_size, glob_size, row_starts, row_starts, 0, diag->NumNonZeroElems(), 0); hypre_ParCSRMatrixSetDataOwner(A,1); +#if MFEM_HYPRE_VERSION <= 22200 hypre_ParCSRMatrixSetRowStartsOwner(A,0); hypre_ParCSRMatrixSetColStartsOwner(A,0); +#endif hypre_CSRMatrixSetDataOwner(A->diag,0); diagOwner = CopyCSR(diag, mem_diag, A->diag, false); @@ -726,8 +736,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, row_starts, col_starts, 0, diag->NumNonZeroElems(), 0); hypre_ParCSRMatrixSetDataOwner(A,1); +#if MFEM_HYPRE_VERSION <= 22200 hypre_ParCSRMatrixSetRowStartsOwner(A,0); hypre_ParCSRMatrixSetColStartsOwner(A,0); +#endif hypre_CSRMatrixSetDataOwner(A->diag,0); diagOwner = CopyCSR(diag, mem_diag, A->diag, false); @@ -770,8 +782,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, offd->Width(), diag->NumNonZeroElems(), offd->NumNonZeroElems()); hypre_ParCSRMatrixSetDataOwner(A,1); +#if MFEM_HYPRE_VERSION <= 22200 hypre_ParCSRMatrixSetRowStartsOwner(A,0); hypre_ParCSRMatrixSetColStartsOwner(A,0); +#endif hypre_CSRMatrixSetDataOwner(A->diag,0); diagOwner = CopyCSR(diag, mem_diag, A->diag, own_diag_offd); @@ -817,8 +831,10 @@ HypreParMatrix::HypreParMatrix( A = hypre_ParCSRMatrixCreate(comm, global_num_rows, global_num_cols, row_starts, col_starts, offd_num_cols, 0, 0); hypre_ParCSRMatrixSetDataOwner(A,1); +#if MFEM_HYPRE_VERSION <= 22200 hypre_ParCSRMatrixSetRowStartsOwner(A,0); hypre_ParCSRMatrixSetColStartsOwner(A,0); +#endif HYPRE_Int local_num_rows = hypre_CSRMatrixNumRows(A->diag); @@ -931,8 +947,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, A = hypre_ParCSRMatrixCreate(comm, global_num_rows, global_num_cols, row_starts, col_starts, 0, nnz, 0); hypre_ParCSRMatrixSetDataOwner(A,1); +#if MFEM_HYPRE_VERSION <= 22200 hypre_ParCSRMatrixSetRowStartsOwner(A,0); hypre_ParCSRMatrixSetColStartsOwner(A,0); +#endif hypre_CSRMatrixSetDataOwner(A->diag,0); diagOwner = CopyBoolCSR(diag, mem_diag, A->diag); @@ -989,8 +1007,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int id, int np, } hypre_ParCSRMatrixSetDataOwner(A,1); +#if MFEM_HYPRE_VERSION <= 22200 hypre_ParCSRMatrixSetRowStartsOwner(A,0); hypre_ParCSRMatrixSetColStartsOwner(A,0); +#endif mem_diag.data.New(diag_nnz); for (HYPRE_Int i = 0; i < diag_nnz; i++) @@ -1177,6 +1197,13 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int nrows, { hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A)); } +#if MFEM_HYPRE_VERSION > 22200 + mfem_hypre_TFree_host(row_starts); + if (rows != cols) + { + mfem_hypre_TFree_host(col_starts); + } +#endif hypre_MatvecCommPkgCreate(A); height = GetNumRows(); @@ -1263,6 +1290,7 @@ void HypreParMatrix::SetOwnerFlags(signed char diag, signed char offd, void HypreParMatrix::CopyRowStarts() { +#if MFEM_HYPRE_VERSION <= 22200 if (!A || hypre_ParCSRMatrixOwnsRowStarts(A) || (hypre_ParCSRMatrixRowStarts(A) == hypre_ParCSRMatrixColStarts(A) && hypre_ParCSRMatrixOwnsColStarts(A))) @@ -1297,10 +1325,12 @@ void HypreParMatrix::CopyRowStarts() hypre_ParCSRMatrixColStarts(A) = new_row_starts; hypre_ParCSRMatrixOwnsColStarts(A) = 0; } +#endif } void HypreParMatrix::CopyColStarts() { +#if MFEM_HYPRE_VERSION <= 22200 if (!A || hypre_ParCSRMatrixOwnsColStarts(A) || (hypre_ParCSRMatrixRowStarts(A) == hypre_ParCSRMatrixColStarts(A) && hypre_ParCSRMatrixOwnsRowStarts(A))) @@ -1339,6 +1369,7 @@ void HypreParMatrix::CopyColStarts() { hypre_ParCSRMatrixOwnsColStarts(A) = 1; } +#endif } void HypreParMatrix::GetDiag(Vector &diag) const @@ -1490,6 +1521,10 @@ HypreParMatrix * HypreParMatrix::Transpose() const HypreParMatrix *HypreParMatrix::ExtractSubmatrix(const Array &indices, double threshold) const { + // hypre_ParCSRMatrixExtractSubmatrixFC works on host only, so we move this + // matrix to host, temporarily: + HostRead(); + if (!(A->comm)) { hypre_MatvecCommPkgCreate(A); @@ -1501,15 +1536,28 @@ HypreParMatrix *HypreParMatrix::ExtractSubmatrix(const Array &indices, int local_num_vars = hypre_CSRMatrixNumRows(hypre_ParCSRMatrixDiag(A)); // Form hypre CF-splitting array designating submatrix as F-points (-1) +#ifdef hypre_IntArrayData + // hypre_BoomerAMGCoarseParms needs CF_marker to be hypre_IntArray * + hypre_IntArray *CF_marker; + + CF_marker = hypre_IntArrayCreate(local_num_vars); + hypre_IntArrayInitialize_v2(CF_marker, HYPRE_MEMORY_HOST); + hypre_IntArraySetConstantValues(CF_marker, 1); +#else Array CF_marker(local_num_vars); CF_marker = 1; +#endif for (int j=0; j local_num_vars) { MFEM_WARNING("WARNING : " << indices[j] << " > " << local_num_vars); } +#ifdef hypre_IntArrayData + hypre_IntArrayData(CF_marker)[indices[j]] = -1; +#else CF_marker[indices[j]] = -1; +#endif } // Construct cpts_global array on hypre matrix structure @@ -1518,10 +1566,22 @@ HypreParMatrix *HypreParMatrix::ExtractSubmatrix(const Array &indices, CF_marker, NULL, &cpts_global); // Extract submatrix into *submat +#ifdef hypre_IntArrayData + hypre_ParCSRMatrixExtractSubmatrixFC(A, hypre_IntArrayData(CF_marker), + cpts_global, "FF", &submat, + threshold); +#else hypre_ParCSRMatrixExtractSubmatrixFC(A, CF_marker, cpts_global, "FF", &submat, threshold); +#endif mfem_hypre_TFree(cpts_global); +#ifdef hypre_IntArrayData + hypre_IntArrayDestroy(CF_marker); +#endif + + HypreRead(); // restore the matrix location to the default hypre location + return new HypreParMatrix(submat); } #endif @@ -1791,9 +1851,14 @@ HypreParMatrix* HypreParMatrix::LeftDiagMult(const SparseMatrix &D, DA_diag, DA_offd, new_col_map_offd, own_diag_offd); +#if MFEM_HYPRE_VERSION <= 22200 // Give ownership of row_starts, col_starts, and col_map_offd to DA hypre_ParCSRMatrixSetRowStartsOwner(DA->A, 1); hypre_ParCSRMatrixSetColStartsOwner(DA->A, 1); +#else + mfem_hypre_TFree_host(new_row_starts); + mfem_hypre_TFree_host(new_col_starts); +#endif DA->colMapOwner = 1; return DA; @@ -1948,18 +2013,22 @@ void HypreParMatrix::Threshold(double threshold) row_starts = hypre_ParCSRMatrixRowStarts(A); col_starts = hypre_ParCSRMatrixColStarts(A); +#if MFEM_HYPRE_VERSION <= 22200 bool old_owns_row = hypre_ParCSRMatrixOwnsRowStarts(A); bool old_owns_col = hypre_ParCSRMatrixOwnsColStarts(A); +#endif HYPRE_BigInt global_num_rows = hypre_ParCSRMatrixGlobalNumRows(A); HYPRE_BigInt global_num_cols = hypre_ParCSRMatrixGlobalNumCols(A); parcsr_A_ptr = hypre_ParCSRMatrixCreate(comm, global_num_rows, global_num_cols, row_starts, col_starts, 0, 0, 0); +#if MFEM_HYPRE_VERSION <= 22200 hypre_ParCSRMatrixOwnsRowStarts(parcsr_A_ptr) = old_owns_row; hypre_ParCSRMatrixOwnsColStarts(parcsr_A_ptr) = old_owns_col; hypre_ParCSRMatrixOwnsRowStarts(A) = 0; hypre_ParCSRMatrixOwnsColStarts(A) = 0; +#endif csr_A = hypre_MergeDiagAndOffd(A); @@ -1994,7 +2063,12 @@ void HypreParMatrix::Threshold(double threshold) hypre_ParCSRMatrixSetNumNonzeros(A); /* Make sure that the first entry in each row is the diagonal one. */ +#if MFEM_HYPRE_VERSION <= 22200 if (row_starts == col_starts) +#else + if ((row_starts[0] == col_starts[0]) && + (row_starts[1] == col_starts[1])) +#endif { hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A)); } @@ -2310,7 +2384,7 @@ inline void delete_hypre_ParCSRMatrixColMapOffd(hypre_ParCSRMatrix *A) { HYPRE_BigInt *A_col_map_offd = hypre_ParCSRMatrixColMapOffd(A); int size = hypre_CSRMatrixNumCols(hypre_ParCSRMatrixOffd(A)); - Memory(A_col_map_offd, size, true).Delete(); + Memory(A_col_map_offd, size, true).Delete(); } void HypreParMatrix::Destroy() @@ -2503,11 +2577,14 @@ HypreParMatrix * RAP(const HypreParMatrix *A, const HypreParMatrix *P) // hypre_ParCSRMatrixRAPKT } #else +#if MFEM_HYPRE_VERSION <= 22200 HYPRE_Int P_owns_its_col_starts = hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*P)); +#endif hypre_BoomerAMGBuildCoarseOperator(*P,*A,*P,&rap); +#if MFEM_HYPRE_VERSION <= 22200 /* Warning: hypre_BoomerAMGBuildCoarseOperator steals the col_starts from P (even if it does not own them)! */ hypre_ParCSRMatrixSetRowStartsOwner(rap,0); @@ -2516,6 +2593,7 @@ HypreParMatrix * RAP(const HypreParMatrix *A, const HypreParMatrix *P) { hypre_ParCSRMatrixSetColStartsOwner(*P, 1); } +#endif #endif hypre_ParCSRMatrixSetNumNonzeros(rap); @@ -2536,13 +2614,16 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A, hypre_ParCSRMatrixDestroy(Q); } #else +#if MFEM_HYPRE_VERSION <= 22200 HYPRE_Int P_owns_its_col_starts = hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*P)); HYPRE_Int Rt_owns_its_col_starts = hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*Rt)); +#endif hypre_BoomerAMGBuildCoarseOperator(*Rt,*A,*P,&rap); +#if MFEM_HYPRE_VERSION <= 22200 /* Warning: hypre_BoomerAMGBuildCoarseOperator steals the col_starts from Rt and P (even if they do not own them)! */ hypre_ParCSRMatrixSetRowStartsOwner(rap,0); @@ -2555,6 +2636,7 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A, { hypre_ParCSRMatrixSetColStartsOwner(*Rt, 1); } +#endif #endif hypre_ParCSRMatrixSetNumNonzeros(rap); @@ -3144,8 +3226,12 @@ void HypreSmoother::SetOperator(const Operator &op) } else { +#if MFEM_HYPRE_VERSION <= 22200 min_eig_est = 0; hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est); +#else + hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est, &min_eig_est); +#endif } Z = new HypreParVector(*A); } @@ -3159,8 +3245,12 @@ void HypreSmoother::SetOperator(const Operator &op) } else { +#if MFEM_HYPRE_VERSION <= 22200 min_eig_est = 0; hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est); +#else + hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est, &min_eig_est); +#endif } // The Taubin and FIR polynomials are defined on [0, 2] @@ -4472,9 +4562,8 @@ void HypreBoomerAMG::SetSystemsOptions(int dim, bool order_bynodes) // HYPRE_BoomerAMGSetDofFunc as in the following code. if (order_bynodes) { - // hypre actually deletes the following pointer in HYPRE_BoomerAMGDestroy, - // so we don't need to track it - HYPRE_Int *mapping = mfem_hypre_CTAlloc_host(HYPRE_Int, height); + // Generate DofFunc mapping on the host + HYPRE_Int *h_mapping = mfem_hypre_CTAlloc_host(HYPRE_Int, height); int h_nnodes = height / dim; // nodes owned in linear algebra (not fem) MFEM_VERIFY(height % dim == 0, "Ordering does not work as claimed!"); int k = 0; @@ -4482,9 +4571,24 @@ void HypreBoomerAMG::SetSystemsOptions(int dim, bool order_bynodes) { for (int j = 0; j < h_nnodes; ++j) { - mapping[k++] = i; + h_mapping[k++] = i; } } + + // After the addition of hypre_IntArray, mapping is assumed + // to be a device pointer. Previously, it was assumed to be + // a host pointer. +#if defined(hypre_IntArrayData) && defined(HYPRE_USING_GPU) + HYPRE_Int *mapping = mfem_hypre_CTAlloc(HYPRE_Int, height); + hypre_TMemcpy(mapping, h_mapping, HYPRE_Int, height, + HYPRE_MEMORY_DEVICE, HYPRE_MEMORY_HOST); + mfem_hypre_TFree_host(h_mapping); +#else + HYPRE_Int *mapping = h_mapping; +#endif + + // hypre actually deletes the mapping pointer in HYPRE_BoomerAMGDestroy, + // so we don't need to track it HYPRE_BoomerAMGSetDofFunc(amg_precond, mapping); } @@ -4529,7 +4633,8 @@ void HypreBoomerAMG::RecomputeRBMs() rbms.SetSize(nrbms); gf_rbms.SetSize(nrbms); - gf_rbms[0] = rbms_rxy.ParallelAverage(); + gf_rbms[0] = fespace->NewTrueDofVector(); + rbms_rxy.GetTrueDofs(*gf_rbms[0]); } else if (dim == 3) { @@ -4548,9 +4653,12 @@ void HypreBoomerAMG::RecomputeRBMs() rbms.SetSize(nrbms); gf_rbms.SetSize(nrbms); - gf_rbms[0] = rbms_rxy.ParallelAverage(); - gf_rbms[1] = rbms_ryz.ParallelAverage(); - gf_rbms[2] = rbms_rzx.ParallelAverage(); + gf_rbms[0] = fespace->NewTrueDofVector(); + gf_rbms[1] = fespace->NewTrueDofVector(); + gf_rbms[2] = fespace->NewTrueDofVector(); + rbms_rxy.GetTrueDofs(*gf_rbms[0]); + rbms_ryz.GetTrueDofs(*gf_rbms[1]); + rbms_rzx.GetTrueDofs(*gf_rbms[2]); } else { @@ -4997,11 +5105,11 @@ HypreADS::HypreADS(const HypreParMatrix &A, ParFiniteElementSpace *face_fespace) void HypreADS::Init(ParFiniteElementSpace *face_fespace) { int cycle_type = 11; - int rlx_type = 2; int rlx_sweeps = 1; double rlx_weight = 1.0; double rlx_omega = 1.0; #ifndef HYPRE_USING_CUDA + int rlx_type = 2; int amg_coarsen_type = 10; int amg_agg_levels = 1; int amg_rlx_type = 8; @@ -5009,6 +5117,7 @@ void HypreADS::Init(ParFiniteElementSpace *face_fespace) int amg_interp_type = 6; int amg_Pmax = 4; #else + int rlx_type = 1; int amg_coarsen_type = 8; int amg_agg_levels = 0; int amg_rlx_type = 18; @@ -5592,16 +5701,18 @@ HypreLOBPCG::OperatorMatvec( void *matvec_data, Operator *Aop = (Operator*)A; - int width = Aop->Width(); - hypre_ParVector * xPar = (hypre_ParVector *)x; hypre_ParVector * yPar = (hypre_ParVector *)y; - Vector xVec(xPar->local_vector->data, width); - Vector yVec(yPar->local_vector->data, width); + HypreParVector xVec(xPar); + HypreParVector yVec(yPar); Aop->Mult( xVec, yVec ); + // Move data back to hypre's device memory location in case the above Mult + // operation moved it to host. + yVec.HypreReadWrite(); + return 0; } @@ -5617,19 +5728,20 @@ HypreLOBPCG::PrecondSolve(void *solver, void *b, void *x) { - Solver *PC = (Solver*)solver; - Operator *OP = (Operator*)A; - - int width = OP->Width(); + Solver *PC = (Solver*)solver; hypre_ParVector * bPar = (hypre_ParVector *)b; hypre_ParVector * xPar = (hypre_ParVector *)x; - Vector bVec(bPar->local_vector->data, width); - Vector xVec(xPar->local_vector->data, width); + HypreParVector bVec(bPar); + HypreParVector xVec(xPar); PC->Mult( bVec, xVec ); + // Move data back to hypre's device memory location in case the above Mult + // operation moved it to host. + xVec.HypreReadWrite(); + return 0; } diff --git a/linalg/hypre_parcsr.cpp b/linalg/hypre_parcsr.cpp index 5f1f50f3f..1c1ba1640 100644 --- a/linalg/hypre_parcsr.cpp +++ b/linalg/hypre_parcsr.cpp @@ -508,8 +508,10 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A, hypre_ParCSRMatrixColStarts(A), 0, 0, 0); +#if MFEM_HYPRE_VERSION <= 22200 hypre_ParCSRMatrixSetRowStartsOwner(*Ae, 0); hypre_ParCSRMatrixSetColStartsOwner(*Ae, 0); +#endif hypre_CSRMatrix *Ae_diag = hypre_ParCSRMatrixDiag(*Ae); hypre_CSRMatrix *Ae_offd = hypre_ParCSRMatrixOffd(*Ae); @@ -1002,10 +1004,17 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A, hypre_ParCSRMatrixOwnsData(blocks[i]) = 1; +#if MFEM_HYPRE_VERSION <= 22200 /* only the first block will own the row/col_starts */ hypre_ParCSRMatrixOwnsRowStarts(blocks[i]) = !i; hypre_ParCSRMatrixOwnsColStarts(blocks[i]) = !i; +#endif } + +#if MFEM_HYPRE_VERSION > 22200 + mfem_hypre_TFree_host(row_starts); + mfem_hypre_TFree_host(col_starts); +#endif } /* Based on hypre_CSRMatrixMatvec in hypre's csr_matvec.c */ @@ -1916,9 +1925,12 @@ hypre_ParCSRMatrixAdd(hypre_ParCSRMatrix *A, /* C owns diag, offd, and cmap. */ hypre_ParCSRMatrixSetDataOwner(C, 1); + +#if MFEM_HYPRE_VERSION <= 22200 /* C does not own row and column starts. */ hypre_ParCSRMatrixSetRowStartsOwner(C, 0); hypre_ParCSRMatrixSetColStartsOwner(C, 0); +#endif return C; }