diff options
-rw-r--r-- | var/spack/repos/builtin/packages/mfem/mfem-4.3-hypre-2.23.0.patch | 575 | ||||
-rw-r--r-- | var/spack/repos/builtin/packages/mfem/package.py | 3 |
2 files changed, 577 insertions, 1 deletions
diff --git a/var/spack/repos/builtin/packages/mfem/mfem-4.3-hypre-2.23.0.patch b/var/spack/repos/builtin/packages/mfem/mfem-4.3-hypre-2.23.0.patch new file mode 100644 index 0000000000..743e0c42dc --- /dev/null +++ b/var/spack/repos/builtin/packages/mfem/mfem-4.3-hypre-2.23.0.patch @@ -0,0 +1,575 @@ +diff --git a/fem/pfespace.cpp b/fem/pfespace.cpp +index 1e2a47a5b..53964328e 100644 +--- a/fem/pfespace.cpp ++++ b/fem/pfespace.cpp +@@ -1278,12 +1278,17 @@ const FiniteElement *ParFiniteElementSpace::GetFaceNbrFaceFE(int i) const + + void ParFiniteElementSpace::Lose_Dof_TrueDof_Matrix() + { ++ P -> StealData(); ++#if MFEM_HYPRE_VERSION <= 22200 + hypre_ParCSRMatrix *csrP = (hypre_ParCSRMatrix*)(*P); + hypre_ParCSRMatrixOwnsRowStarts(csrP) = 1; + hypre_ParCSRMatrixOwnsColStarts(csrP) = 1; +- P -> StealData(); + dof_offsets.LoseData(); + tdof_offsets.LoseData(); ++#else ++ dof_offsets.DeleteAll(); ++ tdof_offsets.DeleteAll(); ++#endif + } + + void ParFiniteElementSpace::ConstructTrueDofs() +diff --git a/linalg/complex_operator.cpp b/linalg/complex_operator.cpp +index 1f42ab425..8a2f8e580 100644 +--- a/linalg/complex_operator.cpp ++++ b/linalg/complex_operator.cpp +@@ -787,10 +787,16 @@ HypreParMatrix * ComplexHypreParMatrix::GetSystemMatrix() const + 2 * num_cols_offd, cmap, + true); + ++#if MFEM_HYPRE_VERSION <= 22200 + // Give the new matrix ownership of row_starts and col_starts + hypre_ParCSRMatrix *hA = (hypre_ParCSRMatrix*)(*A); ++ + hypre_ParCSRMatrixSetRowStartsOwner(hA,1); + hypre_ParCSRMatrixSetColStartsOwner(hA,1); ++#else ++ mfem_hypre_TFree_host(row_starts); ++ mfem_hypre_TFree_host(col_starts); ++#endif + + return A; + } +diff --git a/linalg/hypre.cpp b/linalg/hypre.cpp +index 3f393e39a..6ca80e987 100644 +--- a/linalg/hypre.cpp ++++ b/linalg/hypre.cpp +@@ -88,7 +88,9 @@ HypreParVector::HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size, + { + x = hypre_ParVectorCreate(comm,glob_size,col); + hypre_ParVectorInitialize(x); ++#if MFEM_HYPRE_VERSION <= 22200 + hypre_ParVectorSetPartitioningOwner(x,0); ++#endif + // The data will be destroyed by hypre (this is the default) + hypre_ParVectorSetDataOwner(x,1); + hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1); +@@ -105,7 +107,9 @@ HypreParVector::HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size, + hypre_ParVectorSetDataOwner(x,1); // owns the seq vector + hypre_Vector *x_loc = hypre_ParVectorLocalVector(x); + hypre_SeqVectorSetDataOwner(x_loc,0); ++#if MFEM_HYPRE_VERSION <= 22200 + hypre_ParVectorSetPartitioningOwner(x,0); ++#endif + double tmp = 0.0; + hypre_VectorData(x_loc) = &tmp; + #ifdef HYPRE_USING_CUDA +@@ -128,7 +132,9 @@ HypreParVector::HypreParVector(const HypreParVector &y) : Vector() + x = hypre_ParVectorCreate(y.x -> comm, y.x -> global_size, + y.x -> partitioning); + hypre_ParVectorInitialize(x); ++#if MFEM_HYPRE_VERSION <= 22200 + hypre_ParVectorSetPartitioningOwner(x,0); ++#endif + hypre_ParVectorSetDataOwner(x,1); + hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1); + _SetDataAndSize_(); +@@ -162,7 +168,9 @@ HypreParVector::HypreParVector(ParFiniteElementSpace *pfes) + x = hypre_ParVectorCreate(pfes->GetComm(), pfes->GlobalTrueVSize(), + pfes->GetTrueDofOffsets()); + hypre_ParVectorInitialize(x); ++#if MFEM_HYPRE_VERSION <= 22200 + hypre_ParVectorSetPartitioningOwner(x,0); ++#endif + // The data will be destroyed by hypre (this is the default) + hypre_ParVectorSetDataOwner(x,1); + hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1); +@@ -683,8 +691,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, HYPRE_BigInt glob_size, + A = hypre_ParCSRMatrixCreate(comm, glob_size, glob_size, row_starts, + row_starts, 0, diag->NumNonZeroElems(), 0); + hypre_ParCSRMatrixSetDataOwner(A,1); ++#if MFEM_HYPRE_VERSION <= 22200 + hypre_ParCSRMatrixSetRowStartsOwner(A,0); + hypre_ParCSRMatrixSetColStartsOwner(A,0); ++#endif + + hypre_CSRMatrixSetDataOwner(A->diag,0); + diagOwner = CopyCSR(diag, mem_diag, A->diag, false); +@@ -726,8 +736,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, + row_starts, col_starts, + 0, diag->NumNonZeroElems(), 0); + hypre_ParCSRMatrixSetDataOwner(A,1); ++#if MFEM_HYPRE_VERSION <= 22200 + hypre_ParCSRMatrixSetRowStartsOwner(A,0); + hypre_ParCSRMatrixSetColStartsOwner(A,0); ++#endif + + hypre_CSRMatrixSetDataOwner(A->diag,0); + diagOwner = CopyCSR(diag, mem_diag, A->diag, false); +@@ -770,8 +782,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, + offd->Width(), diag->NumNonZeroElems(), + offd->NumNonZeroElems()); + hypre_ParCSRMatrixSetDataOwner(A,1); ++#if MFEM_HYPRE_VERSION <= 22200 + hypre_ParCSRMatrixSetRowStartsOwner(A,0); + hypre_ParCSRMatrixSetColStartsOwner(A,0); ++#endif + + hypre_CSRMatrixSetDataOwner(A->diag,0); + diagOwner = CopyCSR(diag, mem_diag, A->diag, own_diag_offd); +@@ -817,8 +831,10 @@ HypreParMatrix::HypreParMatrix( + A = hypre_ParCSRMatrixCreate(comm, global_num_rows, global_num_cols, + row_starts, col_starts, offd_num_cols, 0, 0); + hypre_ParCSRMatrixSetDataOwner(A,1); ++#if MFEM_HYPRE_VERSION <= 22200 + hypre_ParCSRMatrixSetRowStartsOwner(A,0); + hypre_ParCSRMatrixSetColStartsOwner(A,0); ++#endif + + HYPRE_Int local_num_rows = hypre_CSRMatrixNumRows(A->diag); + +@@ -931,8 +947,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, + A = hypre_ParCSRMatrixCreate(comm, global_num_rows, global_num_cols, + row_starts, col_starts, 0, nnz, 0); + hypre_ParCSRMatrixSetDataOwner(A,1); ++#if MFEM_HYPRE_VERSION <= 22200 + hypre_ParCSRMatrixSetRowStartsOwner(A,0); + hypre_ParCSRMatrixSetColStartsOwner(A,0); ++#endif + + hypre_CSRMatrixSetDataOwner(A->diag,0); + diagOwner = CopyBoolCSR(diag, mem_diag, A->diag); +@@ -989,8 +1007,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int id, int np, + } + + hypre_ParCSRMatrixSetDataOwner(A,1); ++#if MFEM_HYPRE_VERSION <= 22200 + hypre_ParCSRMatrixSetRowStartsOwner(A,0); + hypre_ParCSRMatrixSetColStartsOwner(A,0); ++#endif + + mem_diag.data.New(diag_nnz); + for (HYPRE_Int i = 0; i < diag_nnz; i++) +@@ -1177,6 +1197,13 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int nrows, + { + hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A)); + } ++#if MFEM_HYPRE_VERSION > 22200 ++ mfem_hypre_TFree_host(row_starts); ++ if (rows != cols) ++ { ++ mfem_hypre_TFree_host(col_starts); ++ } ++#endif + hypre_MatvecCommPkgCreate(A); + + height = GetNumRows(); +@@ -1263,6 +1290,7 @@ void HypreParMatrix::SetOwnerFlags(signed char diag, signed char offd, + + void HypreParMatrix::CopyRowStarts() + { ++#if MFEM_HYPRE_VERSION <= 22200 + if (!A || hypre_ParCSRMatrixOwnsRowStarts(A) || + (hypre_ParCSRMatrixRowStarts(A) == hypre_ParCSRMatrixColStarts(A) && + hypre_ParCSRMatrixOwnsColStarts(A))) +@@ -1297,10 +1325,12 @@ void HypreParMatrix::CopyRowStarts() + hypre_ParCSRMatrixColStarts(A) = new_row_starts; + hypre_ParCSRMatrixOwnsColStarts(A) = 0; + } ++#endif + } + + void HypreParMatrix::CopyColStarts() + { ++#if MFEM_HYPRE_VERSION <= 22200 + if (!A || hypre_ParCSRMatrixOwnsColStarts(A) || + (hypre_ParCSRMatrixRowStarts(A) == hypre_ParCSRMatrixColStarts(A) && + hypre_ParCSRMatrixOwnsRowStarts(A))) +@@ -1339,6 +1369,7 @@ void HypreParMatrix::CopyColStarts() + { + hypre_ParCSRMatrixOwnsColStarts(A) = 1; + } ++#endif + } + + void HypreParMatrix::GetDiag(Vector &diag) const +@@ -1490,6 +1521,10 @@ HypreParMatrix * HypreParMatrix::Transpose() const + HypreParMatrix *HypreParMatrix::ExtractSubmatrix(const Array<int> &indices, + double threshold) const + { ++ // hypre_ParCSRMatrixExtractSubmatrixFC works on host only, so we move this ++ // matrix to host, temporarily: ++ HostRead(); ++ + if (!(A->comm)) + { + hypre_MatvecCommPkgCreate(A); +@@ -1501,15 +1536,28 @@ HypreParMatrix *HypreParMatrix::ExtractSubmatrix(const Array<int> &indices, + int local_num_vars = hypre_CSRMatrixNumRows(hypre_ParCSRMatrixDiag(A)); + + // Form hypre CF-splitting array designating submatrix as F-points (-1) ++#ifdef hypre_IntArrayData ++ // hypre_BoomerAMGCoarseParms needs CF_marker to be hypre_IntArray * ++ hypre_IntArray *CF_marker; ++ ++ CF_marker = hypre_IntArrayCreate(local_num_vars); ++ hypre_IntArrayInitialize_v2(CF_marker, HYPRE_MEMORY_HOST); ++ hypre_IntArraySetConstantValues(CF_marker, 1); ++#else + Array<HYPRE_Int> CF_marker(local_num_vars); + CF_marker = 1; ++#endif + for (int j=0; j<indices.Size(); j++) + { + if (indices[j] > local_num_vars) + { + MFEM_WARNING("WARNING : " << indices[j] << " > " << local_num_vars); + } ++#ifdef hypre_IntArrayData ++ hypre_IntArrayData(CF_marker)[indices[j]] = -1; ++#else + CF_marker[indices[j]] = -1; ++#endif + } + + // Construct cpts_global array on hypre matrix structure +@@ -1518,10 +1566,22 @@ HypreParMatrix *HypreParMatrix::ExtractSubmatrix(const Array<int> &indices, + CF_marker, NULL, &cpts_global); + + // Extract submatrix into *submat ++#ifdef hypre_IntArrayData ++ hypre_ParCSRMatrixExtractSubmatrixFC(A, hypre_IntArrayData(CF_marker), ++ cpts_global, "FF", &submat, ++ threshold); ++#else + hypre_ParCSRMatrixExtractSubmatrixFC(A, CF_marker, cpts_global, + "FF", &submat, threshold); ++#endif + + mfem_hypre_TFree(cpts_global); ++#ifdef hypre_IntArrayData ++ hypre_IntArrayDestroy(CF_marker); ++#endif ++ ++ HypreRead(); // restore the matrix location to the default hypre location ++ + return new HypreParMatrix(submat); + } + #endif +@@ -1791,9 +1851,14 @@ HypreParMatrix* HypreParMatrix::LeftDiagMult(const SparseMatrix &D, + DA_diag, DA_offd, new_col_map_offd, + own_diag_offd); + ++#if MFEM_HYPRE_VERSION <= 22200 + // Give ownership of row_starts, col_starts, and col_map_offd to DA + hypre_ParCSRMatrixSetRowStartsOwner(DA->A, 1); + hypre_ParCSRMatrixSetColStartsOwner(DA->A, 1); ++#else ++ mfem_hypre_TFree_host(new_row_starts); ++ mfem_hypre_TFree_host(new_col_starts); ++#endif + DA->colMapOwner = 1; + + return DA; +@@ -1948,18 +2013,22 @@ void HypreParMatrix::Threshold(double threshold) + row_starts = hypre_ParCSRMatrixRowStarts(A); + col_starts = hypre_ParCSRMatrixColStarts(A); + ++#if MFEM_HYPRE_VERSION <= 22200 + bool old_owns_row = hypre_ParCSRMatrixOwnsRowStarts(A); + bool old_owns_col = hypre_ParCSRMatrixOwnsColStarts(A); ++#endif + HYPRE_BigInt global_num_rows = hypre_ParCSRMatrixGlobalNumRows(A); + HYPRE_BigInt global_num_cols = hypre_ParCSRMatrixGlobalNumCols(A); + parcsr_A_ptr = hypre_ParCSRMatrixCreate(comm, global_num_rows, + global_num_cols, + row_starts, col_starts, + 0, 0, 0); ++#if MFEM_HYPRE_VERSION <= 22200 + hypre_ParCSRMatrixOwnsRowStarts(parcsr_A_ptr) = old_owns_row; + hypre_ParCSRMatrixOwnsColStarts(parcsr_A_ptr) = old_owns_col; + hypre_ParCSRMatrixOwnsRowStarts(A) = 0; + hypre_ParCSRMatrixOwnsColStarts(A) = 0; ++#endif + + csr_A = hypre_MergeDiagAndOffd(A); + +@@ -1994,7 +2063,12 @@ void HypreParMatrix::Threshold(double threshold) + + hypre_ParCSRMatrixSetNumNonzeros(A); + /* Make sure that the first entry in each row is the diagonal one. */ ++#if MFEM_HYPRE_VERSION <= 22200 + if (row_starts == col_starts) ++#else ++ if ((row_starts[0] == col_starts[0]) && ++ (row_starts[1] == col_starts[1])) ++#endif + { + hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A)); + } +@@ -2310,7 +2384,7 @@ inline void delete_hypre_ParCSRMatrixColMapOffd(hypre_ParCSRMatrix *A) + { + HYPRE_BigInt *A_col_map_offd = hypre_ParCSRMatrixColMapOffd(A); + int size = hypre_CSRMatrixNumCols(hypre_ParCSRMatrixOffd(A)); +- Memory<HYPRE_Int>(A_col_map_offd, size, true).Delete(); ++ Memory<HYPRE_BigInt>(A_col_map_offd, size, true).Delete(); + } + + void HypreParMatrix::Destroy() +@@ -2503,11 +2577,14 @@ HypreParMatrix * RAP(const HypreParMatrix *A, const HypreParMatrix *P) + // hypre_ParCSRMatrixRAPKT + } + #else ++#if MFEM_HYPRE_VERSION <= 22200 + HYPRE_Int P_owns_its_col_starts = + hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*P)); ++#endif + + hypre_BoomerAMGBuildCoarseOperator(*P,*A,*P,&rap); + ++#if MFEM_HYPRE_VERSION <= 22200 + /* Warning: hypre_BoomerAMGBuildCoarseOperator steals the col_starts + from P (even if it does not own them)! */ + hypre_ParCSRMatrixSetRowStartsOwner(rap,0); +@@ -2516,6 +2593,7 @@ HypreParMatrix * RAP(const HypreParMatrix *A, const HypreParMatrix *P) + { + hypre_ParCSRMatrixSetColStartsOwner(*P, 1); + } ++#endif + #endif + + hypre_ParCSRMatrixSetNumNonzeros(rap); +@@ -2536,13 +2614,16 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A, + hypre_ParCSRMatrixDestroy(Q); + } + #else ++#if MFEM_HYPRE_VERSION <= 22200 + HYPRE_Int P_owns_its_col_starts = + hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*P)); + HYPRE_Int Rt_owns_its_col_starts = + hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*Rt)); ++#endif + + hypre_BoomerAMGBuildCoarseOperator(*Rt,*A,*P,&rap); + ++#if MFEM_HYPRE_VERSION <= 22200 + /* Warning: hypre_BoomerAMGBuildCoarseOperator steals the col_starts + from Rt and P (even if they do not own them)! */ + hypre_ParCSRMatrixSetRowStartsOwner(rap,0); +@@ -2555,6 +2636,7 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A, + { + hypre_ParCSRMatrixSetColStartsOwner(*Rt, 1); + } ++#endif + #endif + + hypre_ParCSRMatrixSetNumNonzeros(rap); +@@ -3144,8 +3226,12 @@ void HypreSmoother::SetOperator(const Operator &op) + } + else + { ++#if MFEM_HYPRE_VERSION <= 22200 + min_eig_est = 0; + hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est); ++#else ++ hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est, &min_eig_est); ++#endif + } + Z = new HypreParVector(*A); + } +@@ -3159,8 +3245,12 @@ void HypreSmoother::SetOperator(const Operator &op) + } + else + { ++#if MFEM_HYPRE_VERSION <= 22200 + min_eig_est = 0; + hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est); ++#else ++ hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est, &min_eig_est); ++#endif + } + + // The Taubin and FIR polynomials are defined on [0, 2] +@@ -4472,9 +4562,8 @@ void HypreBoomerAMG::SetSystemsOptions(int dim, bool order_bynodes) + // HYPRE_BoomerAMGSetDofFunc as in the following code. + if (order_bynodes) + { +- // hypre actually deletes the following pointer in HYPRE_BoomerAMGDestroy, +- // so we don't need to track it +- HYPRE_Int *mapping = mfem_hypre_CTAlloc_host(HYPRE_Int, height); ++ // Generate DofFunc mapping on the host ++ HYPRE_Int *h_mapping = mfem_hypre_CTAlloc_host(HYPRE_Int, height); + int h_nnodes = height / dim; // nodes owned in linear algebra (not fem) + MFEM_VERIFY(height % dim == 0, "Ordering does not work as claimed!"); + int k = 0; +@@ -4482,9 +4571,24 @@ void HypreBoomerAMG::SetSystemsOptions(int dim, bool order_bynodes) + { + for (int j = 0; j < h_nnodes; ++j) + { +- mapping[k++] = i; ++ h_mapping[k++] = i; + } + } ++ ++ // After the addition of hypre_IntArray, mapping is assumed ++ // to be a device pointer. Previously, it was assumed to be ++ // a host pointer. ++#if defined(hypre_IntArrayData) && defined(HYPRE_USING_GPU) ++ HYPRE_Int *mapping = mfem_hypre_CTAlloc(HYPRE_Int, height); ++ hypre_TMemcpy(mapping, h_mapping, HYPRE_Int, height, ++ HYPRE_MEMORY_DEVICE, HYPRE_MEMORY_HOST); ++ mfem_hypre_TFree_host(h_mapping); ++#else ++ HYPRE_Int *mapping = h_mapping; ++#endif ++ ++ // hypre actually deletes the mapping pointer in HYPRE_BoomerAMGDestroy, ++ // so we don't need to track it + HYPRE_BoomerAMGSetDofFunc(amg_precond, mapping); + } + +@@ -4529,7 +4633,8 @@ void HypreBoomerAMG::RecomputeRBMs() + + rbms.SetSize(nrbms); + gf_rbms.SetSize(nrbms); +- gf_rbms[0] = rbms_rxy.ParallelAverage(); ++ gf_rbms[0] = fespace->NewTrueDofVector(); ++ rbms_rxy.GetTrueDofs(*gf_rbms[0]); + } + else if (dim == 3) + { +@@ -4548,9 +4653,12 @@ void HypreBoomerAMG::RecomputeRBMs() + + rbms.SetSize(nrbms); + gf_rbms.SetSize(nrbms); +- gf_rbms[0] = rbms_rxy.ParallelAverage(); +- gf_rbms[1] = rbms_ryz.ParallelAverage(); +- gf_rbms[2] = rbms_rzx.ParallelAverage(); ++ gf_rbms[0] = fespace->NewTrueDofVector(); ++ gf_rbms[1] = fespace->NewTrueDofVector(); ++ gf_rbms[2] = fespace->NewTrueDofVector(); ++ rbms_rxy.GetTrueDofs(*gf_rbms[0]); ++ rbms_ryz.GetTrueDofs(*gf_rbms[1]); ++ rbms_rzx.GetTrueDofs(*gf_rbms[2]); + } + else + { +@@ -4997,11 +5105,11 @@ HypreADS::HypreADS(const HypreParMatrix &A, ParFiniteElementSpace *face_fespace) + void HypreADS::Init(ParFiniteElementSpace *face_fespace) + { + int cycle_type = 11; +- int rlx_type = 2; + int rlx_sweeps = 1; + double rlx_weight = 1.0; + double rlx_omega = 1.0; + #ifndef HYPRE_USING_CUDA ++ int rlx_type = 2; + int amg_coarsen_type = 10; + int amg_agg_levels = 1; + int amg_rlx_type = 8; +@@ -5009,6 +5117,7 @@ void HypreADS::Init(ParFiniteElementSpace *face_fespace) + int amg_interp_type = 6; + int amg_Pmax = 4; + #else ++ int rlx_type = 1; + int amg_coarsen_type = 8; + int amg_agg_levels = 0; + int amg_rlx_type = 18; +@@ -5592,16 +5701,18 @@ HypreLOBPCG::OperatorMatvec( void *matvec_data, + + Operator *Aop = (Operator*)A; + +- int width = Aop->Width(); +- + hypre_ParVector * xPar = (hypre_ParVector *)x; + hypre_ParVector * yPar = (hypre_ParVector *)y; + +- Vector xVec(xPar->local_vector->data, width); +- Vector yVec(yPar->local_vector->data, width); ++ HypreParVector xVec(xPar); ++ HypreParVector yVec(yPar); + + Aop->Mult( xVec, yVec ); + ++ // Move data back to hypre's device memory location in case the above Mult ++ // operation moved it to host. ++ yVec.HypreReadWrite(); ++ + return 0; + } + +@@ -5617,19 +5728,20 @@ HypreLOBPCG::PrecondSolve(void *solver, + void *b, + void *x) + { +- Solver *PC = (Solver*)solver; +- Operator *OP = (Operator*)A; +- +- int width = OP->Width(); ++ Solver *PC = (Solver*)solver; + + hypre_ParVector * bPar = (hypre_ParVector *)b; + hypre_ParVector * xPar = (hypre_ParVector *)x; + +- Vector bVec(bPar->local_vector->data, width); +- Vector xVec(xPar->local_vector->data, width); ++ HypreParVector bVec(bPar); ++ HypreParVector xVec(xPar); + + PC->Mult( bVec, xVec ); + ++ // Move data back to hypre's device memory location in case the above Mult ++ // operation moved it to host. ++ xVec.HypreReadWrite(); ++ + return 0; + } + +diff --git a/linalg/hypre_parcsr.cpp b/linalg/hypre_parcsr.cpp +index 5f1f50f3f..1c1ba1640 100644 +--- a/linalg/hypre_parcsr.cpp ++++ b/linalg/hypre_parcsr.cpp +@@ -508,8 +508,10 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A, + hypre_ParCSRMatrixColStarts(A), + 0, 0, 0); + ++#if MFEM_HYPRE_VERSION <= 22200 + hypre_ParCSRMatrixSetRowStartsOwner(*Ae, 0); + hypre_ParCSRMatrixSetColStartsOwner(*Ae, 0); ++#endif + + hypre_CSRMatrix *Ae_diag = hypre_ParCSRMatrixDiag(*Ae); + hypre_CSRMatrix *Ae_offd = hypre_ParCSRMatrixOffd(*Ae); +@@ -1002,10 +1004,17 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A, + + hypre_ParCSRMatrixOwnsData(blocks[i]) = 1; + ++#if MFEM_HYPRE_VERSION <= 22200 + /* only the first block will own the row/col_starts */ + hypre_ParCSRMatrixOwnsRowStarts(blocks[i]) = !i; + hypre_ParCSRMatrixOwnsColStarts(blocks[i]) = !i; ++#endif + } ++ ++#if MFEM_HYPRE_VERSION > 22200 ++ mfem_hypre_TFree_host(row_starts); ++ mfem_hypre_TFree_host(col_starts); ++#endif + } + + /* Based on hypre_CSRMatrixMatvec in hypre's csr_matvec.c */ +@@ -1916,9 +1925,12 @@ hypre_ParCSRMatrixAdd(hypre_ParCSRMatrix *A, + + /* C owns diag, offd, and cmap. */ + hypre_ParCSRMatrixSetDataOwner(C, 1); ++ ++#if MFEM_HYPRE_VERSION <= 22200 + /* C does not own row and column starts. */ + hypre_ParCSRMatrixSetRowStartsOwner(C, 0); + hypre_ParCSRMatrixSetColStartsOwner(C, 0); ++#endif + + return C; + } diff --git a/var/spack/repos/builtin/packages/mfem/package.py b/var/spack/repos/builtin/packages/mfem/package.py index f1acb85db8..fff1564140 100644 --- a/var/spack/repos/builtin/packages/mfem/package.py +++ b/var/spack/repos/builtin/packages/mfem/package.py @@ -200,7 +200,7 @@ class Mfem(Package, CudaPackage, ROCmPackage): depends_on('mpi', when='+mpi') depends_on('hypre@2.10.0:2.13', when='@:3.3+mpi') depends_on('hypre@:2.20.0', when='@3.4:4.2+mpi') - depends_on('hypre@:2.22.0', when='@4.3.0+mpi') + depends_on('hypre@:2.23.0', when='@4.3.0+mpi') depends_on('hypre', when='+mpi') depends_on('metis', when='+metis') @@ -300,6 +300,7 @@ class Mfem(Package, CudaPackage, ROCmPackage): patch('mfem-4.2-umpire.patch', when='@4.2.0+umpire') patch('mfem-4.2-slepc.patch', when='@4.2.0+slepc') patch('mfem-4.2-petsc-3.15.0.patch', when='@4.2.0+petsc ^petsc@3.15.0:') + patch('mfem-4.3-hypre-2.23.0.patch', when='@4.3.0') # Patch to fix MFEM makefile syntax error. See # https://github.com/mfem/mfem/issues/1042 for the bug report and |