diff options
| author | Veselin Dobrev <v-dobrev@users.noreply.github.com> | 2021-06-03 11:32:31 -0700 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-06-03 11:32:31 -0700 |
| commit | e4a559a571b88b3f4f52278f5df030e0dad1d8bd (patch) | |
| tree | 24f93d26783b3de3c4f4d9129f7e3b54239e275b | |
| parent | 8aae76eee0806bc0ac0cb91bf9b2eb3fc360b98b (diff) | |
| download | spack-e4a559a571b88b3f4f52278f5df030e0dad1d8bd.tar.gz spack-e4a559a571b88b3f4f52278f5df030e0dad1d8bd.tar.bz2 spack-e4a559a571b88b3f4f52278f5df030e0dad1d8bd.tar.xz spack-e4a559a571b88b3f4f52278f5df030e0dad1d8bd.zip | |
CEED v4.0 release (#22735)
* petsc: add hip variant
* libceed: add 0.8, disable occa by default, and let autodetect AVX
Disabling OCCA because backend updates did not make this release and
there are some known bugs so most users won't have reason to use OCCA.
https://github.com/CEED/libCEED/pull/688
* WIP: ceed: 4.0 release
* MFEM package updates (#19748)
* MFEM package updates
* mfem: flake8
* [mfem] Various fixes and tweaks.
[arpack-ng] Add a patch to fix building with IBM XL Fortran.
[libceed] Fix building with IBM XL C/C++.
[pumi] Add C++11 flag for version 2.2.3.
* [mfem] Fix the shared CUDA build.
Reported by: @MPhysXDev
* [mfem] Fix a TODO item
* [mfem] Tweak the AmgX dependencies
* [suite-sparse] Fix the version of the mpfr dependency
* MFEM: add initial HIP support using the ROCmPackage.
* MFEM: add 'slepc' variant.
* MFEM: update the patch for v4.2 for SLEPc.
* mfem: apply 'mfem-4.2-slepc.patch' just to v4.2.
* ceed: apply 'spack style'
* [mfem] Add a patch for mfem v4.2 to work with petsc v3.15.0.
[laghos] Add laghos version 3.1 based on the latest commit in
the repository; this version works with mfem v4.2.
[ceed] For ceed v4.0 use laghos v3.1.
* [libceed] Explicitly set 'CC_VENDOR=icc' when using 'intel'
compiler.
* [mfem] Allow pumi >= 2.2.3 with mfem >= 4.2.0.
[ceed] Use pumi v2.2.5 with ceed v4.0.0.
* [ceed] Explicitly use occa v1.1.0 with ceed v4.0.0.
Use mfem@4.2.0+rocm with ceed@4.0.0+mfem+hip.
* [ceed] Add NekRS v21 as a dependency for ceed v4.0.0.
* [ceed] Fix NekRS version: 21 --> 21.0
* [ceed] Propagate +cuda variant to petsc for ceed v4.0.
* [mfem] Propagate '+rocm' variant to some other packages.
* [ceed] Use +rocm variant of nekrs instead of +hip.
* [ceed] Do not enable magma with ceed@4.0.0+hip.
* [libceed] Fix hip build with libceed@0.8.
* [laghos] For v3.1, use the release .tar.gz file instead of commit.
* Remove cuda & hip variants as they are inherited
* [ceed] Remove comments and FIXMEs about 'magma+hip'.
* [ceed] [libceed] Remove TODOs about occa + hip.
* libceed: use ROCmPackage and +rocm
* petsc: use ROCmPackage for HIP
* libceed, petsc: use CudaPackage
* ceed: forward cuda_arch and amdgpu_target
* [mfem] Use Spack's CudaPackage as a base class; as a result,
'cuda_arch' values should not include the 'sm_' prefix.
Also, propagate 'cuda_arch' and 'amdgpu_target' variants
to enabled dependencies.
* petsc: variant is +rocm, package name is hip
Co-authored-by: Jed Brown <jed@jedbrown.org>
Co-authored-by: Thilina Rathnayake <thilinarmtb@gmail.com>
15 files changed, 503 insertions, 111 deletions
diff --git a/var/spack/repos/builtin/packages/arpack-ng/package.py b/var/spack/repos/builtin/packages/arpack-ng/package.py index dab97ea656..ee3e3c218c 100644 --- a/var/spack/repos/builtin/packages/arpack-ng/package.py +++ b/var/spack/repos/builtin/packages/arpack-ng/package.py @@ -63,6 +63,11 @@ class ArpackNg(Package): # Fujitsu compiler does not support 'isnan' function. # isnan: function that determines whether it is NaN. patch('incompatible_isnan_fix.patch', when='%fj') + patch('incompatible_isnan_fix.patch', when='@3.7.0%xl') + patch('incompatible_isnan_fix.patch', when='@3.7.0%xl_r') + + patch('xlf.patch', when='@3.7.0%xl', level=0) + patch('xlf.patch', when='@3.7.0%xl_r', level=0) depends_on('blas') depends_on('lapack') diff --git a/var/spack/repos/builtin/packages/arpack-ng/xlf.patch b/var/spack/repos/builtin/packages/arpack-ng/xlf.patch new file mode 100644 index 0000000000..1746ee591d --- /dev/null +++ b/var/spack/repos/builtin/packages/arpack-ng/xlf.patch @@ -0,0 +1,50 @@ +--- debug.h.orig 2019-01-12 07:24:12.000000000 -0800 ++++ debug.h 2020-11-10 14:02:59.269942675 -0800 +@@ -5,12 +5,12 @@ + ! %---------------------------------% + ! | See debug.doc for documentation | + ! %---------------------------------% +- integer logfil, ndigit, mgetv0, & +- msaupd, msaup2, msaitr, mseigt, msapps, msgets, mseupd,& +- mnaupd, mnaup2, mnaitr, mneigh, mnapps, mngets, mneupd,& +- mcaupd, mcaup2, mcaitr, mceigh, mcapps, mcgets, mceupd +- common /debug/ & +- logfil, ndigit, mgetv0, & +- msaupd, msaup2, msaitr, mseigt, msapps, msgets, mseupd,& +- mnaupd, mnaup2, mnaitr, mneigh, mnapps, mngets, mneupd,& +- mcaupd, mcaup2, mcaitr, mceigh, mcapps, mcgets, mceupd ++ integer logfil, ndigit, mgetv0, ++ & msaupd, msaup2, msaitr, mseigt, msapps, msgets, mseupd, ++ & mnaupd, mnaup2, mnaitr, mneigh, mnapps, mngets, mneupd, ++ & mcaupd, mcaup2, mcaitr, mceigh, mcapps, mcgets, mceupd ++ common /debug/ ++ & logfil, ndigit, mgetv0, ++ & msaupd, msaup2, msaitr, mseigt, msapps, msgets, mseupd, ++ & mnaupd, mnaup2, mnaitr, mneigh, mnapps, mngets, mneupd, ++ & mcaupd, mcaup2, mcaitr, mceigh, mcapps, mcgets, mceupd +--- stat.h.orig 2019-01-12 07:24:12.000000000 -0800 ++++ stat.h 2020-11-10 14:14:01.276289946 -0800 +@@ -9,13 +9,13 @@ + save t0, t1, t2, t3, t4, t5 + ! + integer nopx, nbx, nrorth, nitref, nrstrt +- real tsaupd, tsaup2, tsaitr, tseigt, tsgets, tsapps, tsconv,& +- tnaupd, tnaup2, tnaitr, tneigh, tngets, tnapps, tnconv,& +- tcaupd, tcaup2, tcaitr, tceigh, tcgets, tcapps, tcconv,& +- tmvopx, tmvbx, tgetv0, titref, trvec +- common /timing/ & +- nopx, nbx, nrorth, nitref, nrstrt, & +- tsaupd, tsaup2, tsaitr, tseigt, tsgets, tsapps, tsconv,& +- tnaupd, tnaup2, tnaitr, tneigh, tngets, tnapps, tnconv,& +- tcaupd, tcaup2, tcaitr, tceigh, tcgets, tcapps, tcconv,& +- tmvopx, tmvbx, tgetv0, titref, trvec ++ real tsaupd, tsaup2, tsaitr, tseigt, tsgets, tsapps, tsconv, ++ & tnaupd, tnaup2, tnaitr, tneigh, tngets, tnapps, tnconv, ++ & tcaupd, tcaup2, tcaitr, tceigh, tcgets, tcapps, tcconv, ++ & tmvopx, tmvbx, tgetv0, titref, trvec ++ common /timing/ ++ & nopx, nbx, nrorth, nitref, nrstrt, ++ & tsaupd, tsaup2, tsaitr, tseigt, tsgets, tsapps, tsconv, ++ & tnaupd, tnaup2, tnaitr, tneigh, tngets, tnapps, tnconv, ++ & tcaupd, tcaup2, tcaitr, tceigh, tcgets, tcapps, tcconv, ++ & tmvopx, tmvbx, tgetv0, titref, trvec diff --git a/var/spack/repos/builtin/packages/ceed/package.py b/var/spack/repos/builtin/packages/ceed/package.py index 1090385a32..9a2c66f753 100644 --- a/var/spack/repos/builtin/packages/ceed/package.py +++ b/var/spack/repos/builtin/packages/ceed/package.py @@ -6,7 +6,7 @@ from spack import * -class Ceed(BundlePackage): +class Ceed(BundlePackage, CudaPackage, ROCmPackage): """Ceed is a collection of benchmarks, miniapps, software libraries and APIs for efficient high-order finite element and spectral element discretizations for exascale applications developed in the Department of @@ -18,12 +18,11 @@ class Ceed(BundlePackage): maintainers = ['jedbrown', 'v-dobrev', 'tzanio'] + version('4.0.0') version('3.0.0') version('2.0.0') version('1.0.0') - variant('cuda', default=False, - description='Enable CUDA support') variant('mfem', default=True, description='Build MFEM, Laghos and Remhos') variant('nek', default=True, description='Build Nek5000, GSLIB, Nekbone, and NekCEM') @@ -38,6 +37,17 @@ class Ceed(BundlePackage): # TODO: Add 'int64' variant? # LibCEED + # ceed 4.0 + depends_on('libceed@0.8~cuda', when='@4.0.0~cuda') + for arch in CudaPackage.cuda_arch_values: + depends_on('libceed@0.8+cuda+magma cuda_arch={0}'.format(arch), + when='@4.0.0+cuda cuda_arch={0}'.format(arch)) + depends_on('libceed@0.8~rocm', when='@4.0.0~rocm') + for target in ROCmPackage.amdgpu_targets: + depends_on('libceed@0.8+rocm amdgpu_target={0}'.format(target), + when='@4.0.0+rocm amdgpu_target={0}'.format(target)) + depends_on('libceed@0.8+occa', when='@4.0.0+occa') + depends_on('libceed@0.8~occa', when='@4.0.0~occa') # ceed-3.0 depends_on('libceed@0.6~cuda', when='@3.0.0~cuda') depends_on('libceed@0.6+cuda+magma', when='@3.0.0+cuda') @@ -55,6 +65,9 @@ class Ceed(BundlePackage): depends_on('libceed@0.2~occa', when='@1.0.0~occa') # OCCA + # ceed-4.0 + depends_on('occa@1.1.0~cuda', when='@4.0.0+occa~cuda') + depends_on('occa@1.1.0+cuda', when='@4.0.0+occa+cuda') # ceed-3.0 depends_on('occa@1.0.9~cuda', when='@3.0.0+occa~cuda') depends_on('occa@1.0.9+cuda', when='@3.0.0+occa+cuda') @@ -65,13 +78,23 @@ class Ceed(BundlePackage): depends_on('occa@1.0.0-alpha.5~cuda', when='@1.0.0+occa~cuda') depends_on('occa@1.0.0-alpha.5+cuda', when='@1.0.0+occa+cuda') + # NekRS + # ceed-4.0 + depends_on('nekrs@21.0', when='@4.0.0+nek') + for arch in CudaPackage.cuda_arch_values: + depends_on('nekrs@21.0+cuda cuda_arch={0}'.format(arch), + when='@4.0.0+nek+cuda cuda_arch={0}'.format(arch)) + for target in ROCmPackage.amdgpu_targets: + depends_on('nekrs@21.0+rocm amdgpu_target={0}'.format(target), + when='@4.0.0+nek+rocm amdgpu_target={0}'.format(target)) + # Nek5000, GSLIB, Nekbone, and NekCEM - # ceed-3.0 - depends_on('nek5000@19.0', when='@3.0.0+nek') - depends_on('nektools@19.0%gcc', when='@3.0.0+nek') - depends_on('gslib@1.0.6', when='@3.0.0+nek') - depends_on('nekbone@17.0', when='@3.0.0+nek') - depends_on('nekcem@c8db04b', when='@3.0.0+nek') + # ceed-3.0 and ceed-4.0 + depends_on('nek5000@19.0', when='@3.0.0:4.99+nek') + depends_on('nektools@19.0%gcc', when='@3.0.0:4.99+nek') + depends_on('gslib@1.0.6', when='@3.0.0:4.99+nek') + depends_on('nekbone@17.0', when='@3.0.0:4.99+nek') + depends_on('nekcem@c8db04b', when='@3.0.0:4.99+nek') # ceed-2.0 depends_on('nek5000@17.0', when='@2.0.0+nek') depends_on('nektools@17.0%gcc', when='@2.0.0+nek') @@ -86,6 +109,17 @@ class Ceed(BundlePackage): depends_on('nekcem@0b8bedd', when='@1.0.0+nek') # PETSc + # ceed 4.0 + depends_on('petsc@3.15.0:3.15.99', when='@4.0.0:4.99.99+petsc') + for arch in CudaPackage.cuda_arch_values: + depends_on('petsc+cuda cuda_arch={0}'.format(arch), + when='@4.0.0+petsc+cuda cuda_arch={0}'.format(arch)) + for target in ROCmPackage.amdgpu_targets: + depends_on('petsc@3.15.0:3.15.99+rocm amdgpu_target={0}'.format(target), + when='@4.0.0:4.99.99+petsc+rocm amdgpu_target={0}'.format(target)) + depends_on('petsc@3.15.0:3.15.99~hdf5~superlu-dist', + when='@4.0.0+petsc+quickbuild') + depends_on('petsc@3.15.0:3.15.99+mpi+double~int64', when='@4.0.0:4.99.99+petsc~mfem') # ceed-3.0 depends_on('petsc+cuda', when='@3.0.0+petsc+cuda') # For a +quickbuild we disable hdf5, and superlu-dist in PETSc. @@ -125,6 +159,10 @@ class Ceed(BundlePackage): depends_on('hpgmg@a0a5510df23b+fe', when='@1.0.0+petsc') # MAGMA + # ceed-4.0 + for arch in CudaPackage.cuda_arch_values: + depends_on('magma@2.5.4 cuda_arch={0}'.format(arch), + when='@4.0.0+cuda cuda_arch={0}'.format(arch)) # ceed-3.0 depends_on('magma@2.5.3', when='@3.0.0+cuda') # ceed-2.0 @@ -133,6 +171,8 @@ class Ceed(BundlePackage): depends_on('magma@2.3.0', when='@1.0.0+cuda') # PUMI + # ceed-4.0 + depends_on('pumi@2.2.5', when='@4.0.0+pumi') # ceed-3.0 depends_on('pumi@2.2.2', when='@3.0.0+pumi') # ceed-2.0 @@ -141,6 +181,22 @@ class Ceed(BundlePackage): depends_on('pumi@2.1.0', when='@1.0.0+pumi') # MFEM, Laghos, Remhos + # ceed-4.0 + depends_on('mfem@4.2.0+mpi+examples+miniapps', when='@4.0.0+mfem~petsc') + depends_on('mfem@4.2.0+mpi+petsc+examples+miniapps', + when='@4.0.0+mfem+petsc') + depends_on('mfem@4.2.0+pumi', when='@4.0.0+mfem+pumi') + depends_on('mfem@4.2.0+gslib', when='@4.0.0+mfem+nek') + depends_on('mfem@4.2.0+libceed', when='@4.0.0+mfem') + for arch in CudaPackage.cuda_arch_values: + depends_on('mfem@4.2.0+cuda cuda_arch={0}'.format(arch), + when='@4.0.0+mfem+cuda cuda_arch={0}'.format(arch)) + for target in ROCmPackage.amdgpu_targets: + depends_on('mfem@4.2.0+rocm amdgpu_target={0}'.format(target), + when='@4.0.0+mfem+rocm amdgpu_target={0}'.format(target)) + depends_on('mfem@4.2.0+occa', when='@4.0.0+mfem+occa') + depends_on('laghos@3.1', when='@4.0.0+mfem') + depends_on('remhos@1.0', when='@4.0.0+mfem') # ceed-3.0 depends_on('mfem@4.1.0+mpi+examples+miniapps', when='@3.0.0+mfem~petsc') depends_on('mfem@4.1.0+mpi+petsc+examples+miniapps', diff --git a/var/spack/repos/builtin/packages/laghos/package.py b/var/spack/repos/builtin/packages/laghos/package.py index 03dd7ca4ab..0f327ceb1c 100644 --- a/var/spack/repos/builtin/packages/laghos/package.py +++ b/var/spack/repos/builtin/packages/laghos/package.py @@ -21,6 +21,7 @@ class Laghos(MakefilePackage): maintainers = ['v-dobrev', 'tzanio', 'vladotomov'] version('develop', branch='master') + version('3.1', sha256='49b65edcbf9732c7f6c228958620e18980c43ad8381315a8ba9957ecb7534cd5') version('3.0', sha256='4db56286e15b42ecdc8d540c4888a7dec698b019df9c7ccb8319b7ea1f92d8b4') version('2.0', sha256='dd3632d5558889beec2cd3c49eb60f633f99e6d886ac868731610dd006c44c14') version('1.1', sha256='53b9bfe2af263c63eb4544ca1731dd26f40b73a0d2775a9883db51821bf23b7f') @@ -32,7 +33,8 @@ class Laghos(MakefilePackage): depends_on('mfem+mpi~metis', when='~metis') depends_on('mfem@develop', when='@develop') - depends_on('mfem@4.1.0:', when='@3.0') + depends_on('mfem@4.2.0', when='@3.1') + depends_on('mfem@4.1.0:4.1.99', when='@3.0') # Recommended mfem version for laghos v2.0 is: ^mfem@3.4.1-laghos-v2.0 depends_on('mfem@3.4.0:', when='@2.0') # Recommended mfem version for laghos v1.x is: ^mfem@3.3.1-laghos-v1.0 diff --git a/var/spack/repos/builtin/packages/libceed/libceed-v0.8-hip.patch b/var/spack/repos/builtin/packages/libceed/libceed-v0.8-hip.patch new file mode 100644 index 0000000000..c947e12efe --- /dev/null +++ b/var/spack/repos/builtin/packages/libceed/libceed-v0.8-hip.patch @@ -0,0 +1,14 @@ +diff --git a/Makefile b/Makefile +index 4f1737ee..3f1811f2 100644 +--- a/Makefile ++++ b/Makefile +@@ -403,6 +403,9 @@ ifneq ($(HIP_LIB_DIR),) + CPPFLAGS += $(subst =,,$(shell $(HIP_DIR)/bin/hipconfig -C)) + endif + $(libceeds) : CPPFLAGS += -I$(HIP_DIR)/include ++ ifneq ($(HIPBLAS_DIR),) ++ PKG_LIBS += -L$(HIPBLAS_DIR)/lib ++ endif + PKG_LIBS += -L$(abspath $(HIP_LIB_DIR)) -lamdhip64 -lhipblas + LIBCEED_CONTAINS_CXX = 1 + libceed.c += interface/ceed-hip.c diff --git a/var/spack/repos/builtin/packages/libceed/package.py b/var/spack/repos/builtin/packages/libceed/package.py index b73fa98c4d..d94d13059b 100644 --- a/var/spack/repos/builtin/packages/libceed/package.py +++ b/var/spack/repos/builtin/packages/libceed/package.py @@ -6,7 +6,7 @@ from spack import * -class Libceed(Package): +class Libceed(Package, CudaPackage, ROCmPackage): """The CEED API Library: Code for Efficient Extensible Discretizations.""" homepage = "https://github.com/CEED/libCEED" @@ -15,6 +15,7 @@ class Libceed(Package): maintainers = ['jedbrown', 'v-dobrev', 'tzanio'] version('develop', branch='main') + version('0.8', tag='v0.8') version('0.7', tag='v0.7') version('0.6', commit='c7f533e01e2f3f6720fbf37aac2af2ffed225f60') # tag v0.6 + small portability fixes version('0.5', tag='v0.5') @@ -22,9 +23,7 @@ class Libceed(Package): version('0.2', tag='v0.2') version('0.1', tag='v0.1') - variant('occa', default=True, description='Enable OCCA backends') - variant('cuda', default=False, description='Enable CUDA support') - variant('hip', default=False, description='Enable HIP support') + variant('occa', default=False, description='Enable OCCA backends') variant('debug', default=False, description='Enable debug build') variant('libxsmm', default=False, description='Enable LIBXSMM backend') variant('magma', default=False, description='Enable MAGMA backend') @@ -34,8 +33,9 @@ class Libceed(Package): conflicts('+hip', when='@:0.6') depends_on('cuda', when='+cuda') - depends_on('hip@3.8.0', when='@0.7:0.7.99+hip') - depends_on('hip@3.8.0:', when='@0.8:+hip') + depends_on('hip@3.8.0', when='@0.7:0.7.99+rocm') + depends_on('hip@3.8.0:', when='@0.8:+rocm') + depends_on('hipblas@3.8.0:', when='@0.8:+rocm') depends_on('occa@develop', when='@develop+occa') depends_on('occa@1.1.0', when='@0.7:+occa') @@ -48,6 +48,7 @@ class Libceed(Package): depends_on('magma', when='+magma') + patch('libceed-v0.8-hip.patch', when='@0.8+rocm') patch('pkgconfig-version-0.4.diff', when='@0.4') # occa: do not occaFree kernels @@ -77,6 +78,8 @@ class Libceed(Package): opt = '-O3 -g -ffp-contract=fast' if compiler.version >= ver(4.9): opt += ' -fopenmp-simd' + if self.spec.target.family in ['x86_64', 'aarch64']: + opt += ' -march=native' elif compiler.name == 'apple-clang': opt = '-O3 -g -march=native -ffp-contract=fast' if compiler.version >= ver(10): @@ -87,11 +90,19 @@ class Libceed(Package): opt += ' -fopenmp-simd' elif compiler.name in ['xl', 'xl_r']: opt = '-O -g -qsimd=auto' + elif compiler.name == 'intel': + opt = '-O3 -g' + makeopts += ['CC_VENDOR=icc'] else: opt = '-O -g' + # Note: spack will inject additional target-specific flags through + # the compiler wrapper. makeopts += ['OPT=%s' % opt] - if 'avx' in self.spec.target: + if spec.satisfies('@0.7') and compiler.name in ['xl', 'xl_r']: + makeopts += ['CXXFLAGS.XL=-qpic -std=c++11 -MMD'] + + if spec.satisfies('@:0.7') and 'avx' in self.spec.target: makeopts.append('AVX=1') if '+cuda' in spec: @@ -105,8 +116,10 @@ class Libceed(Package): # Disable CUDA auto-detection: makeopts += ['CUDA_DIR=/disable-cuda'] - if '+hip' in spec: + if '+rocm' in spec: makeopts += ['HIP_DIR=%s' % spec['hip'].prefix] + if spec.satisfies('@0.8'): + makeopts += ['HIPBLAS_DIR=%s' % spec['hipblas'].prefix] if '+libxsmm' in spec: makeopts += ['XSMM_DIR=%s' % spec['libxsmm'].prefix] diff --git a/var/spack/repos/builtin/packages/mfem/mfem-4.2-petsc-3.15.0.patch b/var/spack/repos/builtin/packages/mfem/mfem-4.2-petsc-3.15.0.patch new file mode 100644 index 0000000000..cb79a2130f --- /dev/null +++ b/var/spack/repos/builtin/packages/mfem/mfem-4.2-petsc-3.15.0.patch @@ -0,0 +1,32 @@ +diff --git a/linalg/petsc.cpp b/linalg/petsc.cpp +index fcfdd325f..90eedf042 100644 +--- a/linalg/petsc.cpp ++++ b/linalg/petsc.cpp +@@ -1872,8 +1872,13 @@ void PetscSolver::SetPrintLevel(int plev) + { + ierr = PetscViewerAndFormatCreate(viewer,PETSC_VIEWER_DEFAULT,&vf); + PCHKERRQ(viewer,ierr); ++#if PETSC_VERSION_LT(3,15,0) + ierr = KSPMonitorSet(ksp,(myMonitor)KSPMonitorTrueResidualNorm,vf, + (myPetscFunc)PetscViewerAndFormatDestroy); ++#else ++ ierr = KSPMonitorSet(ksp,(myMonitor)KSPMonitorTrueResidual,vf, ++ (myPetscFunc)PetscViewerAndFormatDestroy); ++#endif + PCHKERRQ(ksp,ierr); + } + } +@@ -2887,8 +2892,13 @@ void PetscBDDCSolver::BDDCSolverConstructor(const PetscBDDCSolverParams &opts) + + ierr = VecGetArrayRead(pvec_coords,&garray); CCHKERRQ(PETSC_COMM_SELF,ierr); + ierr = VecGetArray(lvec_coords,&larray); CCHKERRQ(PETSC_COMM_SELF,ierr); ++#if PETSC_VERSION_LT(3,15,0) + ierr = PetscSFBcastBegin(sf,MPIU_SCALAR,garray,larray); CCHKERRQ(comm,ierr); + ierr = PetscSFBcastEnd(sf,MPIU_SCALAR,garray,larray); CCHKERRQ(comm,ierr); ++#else ++ ierr = PetscSFBcastBegin(sf,MPIU_SCALAR,garray,larray,MPI_REPLACE); CCHKERRQ(comm,ierr); ++ ierr = PetscSFBcastEnd(sf,MPIU_SCALAR,garray,larray,MPI_REPLACE); CCHKERRQ(comm,ierr); ++#endif + ierr = VecRestoreArrayRead(pvec_coords,&garray); CCHKERRQ(PETSC_COMM_SELF,ierr); + ierr = VecRestoreArray(lvec_coords,&larray); CCHKERRQ(PETSC_COMM_SELF,ierr); + } diff --git a/var/spack/repos/builtin/packages/mfem/mfem-4.2-slepc.patch b/var/spack/repos/builtin/packages/mfem/mfem-4.2-slepc.patch new file mode 100644 index 0000000000..8ba4338c4e --- /dev/null +++ b/var/spack/repos/builtin/packages/mfem/mfem-4.2-slepc.patch @@ -0,0 +1,40 @@ +diff --git a/linalg/slepc.hpp b/linalg/slepc.hpp +index 7f8911ba2..5b1f1c686 100644 +--- a/linalg/slepc.hpp ++++ b/linalg/slepc.hpp +@@ -19,12 +19,15 @@ + + #include "petsc.hpp" + +-// Forward declarations +-typedef struct _p_EPS *EPS; ++// Forward declaration of SLEPc's internal struct _p_EPS: ++struct _p_EPS; + + namespace mfem + { + ++// Declare an alias of SLEPc's EPS type, mfem::slepc::EPS: ++namespace slepc { typedef struct ::_p_EPS *EPS; } ++ + void MFEMInitializeSlepc(); + void MFEMInitializeSlepc(int*,char***); + void MFEMInitializeSlepc(int*,char***,const char[],const char[]); +@@ -37,7 +40,7 @@ private: + mutable bool clcustom; + + /// SLEPc linear eigensolver object +- EPS eps; ++ slepc::EPS eps; + + /// Real and imaginary part of eigenvector + mutable PetscParVector *VR, *VC; +@@ -102,7 +105,7 @@ public: + void SetSpectralTransformation(SpectralTransformation transformation); + + /// Conversion function to SLEPc's EPS type. +- operator EPS() const { return eps; } ++ operator slepc::EPS() const { return eps; } + + /// Conversion function to PetscObject + operator PetscObject() const {return (PetscObject)eps; } diff --git a/var/spack/repos/builtin/packages/mfem/mfem-4.2-umpire.patch b/var/spack/repos/builtin/packages/mfem/mfem-4.2-umpire.patch new file mode 100644 index 0000000000..9ab0214d6c --- /dev/null +++ b/var/spack/repos/builtin/packages/mfem/mfem-4.2-umpire.patch @@ -0,0 +1,24 @@ +diff --git a/general/mem_manager.cpp b/general/mem_manager.cpp +index ff4a03465..748f6e673 100644 +--- a/general/mem_manager.cpp ++++ b/general/mem_manager.cpp +@@ -482,7 +482,8 @@ public: + HostMemorySpace(), + name(mm.GetUmpireAllocatorHostName()), + rm(umpire::ResourceManager::getInstance()), +- h_allocator(rm.isAllocator(name)? rm.getAllocator(name): ++ h_allocator((!std::strcmp(name, "HOST") || rm.isAllocator(name)) ? ++ rm.getAllocator(name) : + rm.makeAllocator<umpire::strategy::DynamicPool> + (name, rm.getAllocator("HOST"))), + strat(h_allocator.getAllocationStrategy()) { } +@@ -506,7 +507,8 @@ public: + DeviceMemorySpace(), + name(mm.GetUmpireAllocatorDeviceName()), + rm(umpire::ResourceManager::getInstance()), +- d_allocator(rm.isAllocator(name)? rm.getAllocator(name): ++ d_allocator((!std::strcmp(name, "DEVICE") || rm.isAllocator(name)) ? ++ rm.getAllocator(name) : + rm.makeAllocator<umpire::strategy::DynamicPool> + (name, rm.getAllocator("DEVICE"))) { } + void Alloc(Memory &base) { base.d_ptr = d_allocator.allocate(base.bytes); } diff --git a/var/spack/repos/builtin/packages/mfem/package.py b/var/spack/repos/builtin/packages/mfem/package.py index d9e88be3d0..3b97e77190 100644 --- a/var/spack/repos/builtin/packages/mfem/package.py +++ b/var/spack/repos/builtin/packages/mfem/package.py @@ -6,9 +6,10 @@ from spack import * import os import shutil +import sys -class Mfem(Package): +class Mfem(Package, CudaPackage, ROCmPackage): """Free, lightweight, scalable C++ library for finite element methods.""" tags = ['FEM', 'finite elements', 'high-order', 'AMR', 'HPC'] @@ -100,9 +101,8 @@ class Mfem(Package): description='Enable METIS support') variant('openmp', default=False, description='Enable OpenMP parallelism') - variant('cuda', default=False, description='Enable CUDA support') - variant('cuda_arch', default='sm_60', - description='CUDA architecture to compile for') + # Note: '+cuda' and 'cuda_arch' variants are added by the CudaPackage + # Note: '+rocm' and 'amdgpu_target' variants are added by the ROCmPackage variant('occa', default=False, description='Enable OCCA backend') variant('raja', default=False, description='Enable RAJA backend') variant('libceed', default=False, description='Enable libCEED backend') @@ -121,6 +121,8 @@ class Mfem(Package): description='Enable serial, sparse direct solvers') variant('petsc', default=False, description='Enable PETSc solvers, preconditioners, etc.') + variant('slepc', default=False, + description='Enable SLEPc integration') variant('sundials', default=False, description='Enable Sundials time integrators') variant('pumi', default=False, @@ -143,6 +145,7 @@ class Mfem(Package): description='Enable secure sockets using GnuTLS') variant('libunwind', default=False, description='Enable backtrace on error support using Libunwind') + # TODO: SIMD, Ginkgo, ADIOS2, HiOp, MKL CPardiso, Axom/Sidre variant('timer', default='auto', values=('auto', 'std', 'posix', 'mac', 'mpi'), description='Timing functions to use in mfem::StopWatch') @@ -153,18 +156,21 @@ class Mfem(Package): conflicts('+shared', when='@:3.3.2') conflicts('~static~shared') - conflicts('~threadsafe', when='+openmp') + conflicts('~threadsafe', when='@:3.99.99+openmp') conflicts('+cuda', when='@:3.99.99') + conflicts('+rocm', when='@:4.1.99') + conflicts('+cuda+rocm') conflicts('+netcdf', when='@:3.1') conflicts('+superlu-dist', when='@:3.1') # STRUMPACK support was added in mfem v3.3.2, however, here we allow only - # strumpack v3 support which is available starting with mfem v4.0: + # strumpack v3+ support for which is available starting with mfem v4.0: conflicts('+strumpack', when='@:3.99.99') conflicts('+gnutls', when='@:3.1') conflicts('+zlib', when='@:3.2') conflicts('+mpfr', when='@:3.2') conflicts('+petsc', when='@:3.2') + conflicts('+slepc', when='@:4.1.99') conflicts('+sundials', when='@:3.2') conflicts('+pumi', when='@:3.3.2') conflicts('+gslib', when='@:4.0.99') @@ -183,6 +189,7 @@ class Mfem(Package): conflicts('+superlu-dist', when='~mpi') conflicts('+strumpack', when='~mpi') conflicts('+petsc', when='~mpi') + conflicts('+slepc', when='~petsc') conflicts('+pumi', when='~mpi') conflicts('timer=mpi', when='~mpi') @@ -194,27 +201,32 @@ class Mfem(Package): depends_on('blas', when='+lapack') depends_on('lapack@3.0:', when='+lapack') - depends_on('cuda', when='+cuda') - depends_on('sundials@2.7.0', when='@:3.3.0+sundials~mpi') depends_on('sundials@2.7.0+mpi+hypre', when='@:3.3.0+sundials+mpi') depends_on('sundials@2.7.0:', when='@3.3.2:+sundials~mpi') depends_on('sundials@2.7.0:+mpi+hypre', when='@3.3.2:+sundials+mpi') depends_on('sundials@5.0.0:', when='@4.0.1-xsdk:+sundials~mpi') depends_on('sundials@5.0.0:+mpi+hypre', when='@4.0.1-xsdk:+sundials+mpi') - depends_on('sundials@5.4.0:+cuda', when='@4.2.0:+sundials+cuda') - depends_on('pumi@2.2.3', when='@4.2.0:+pumi') + for sm_ in CudaPackage.cuda_arch_values: + depends_on('sundials@5.4.0:+cuda cuda_arch={0}'.format(sm_), + when='@4.2.0:+sundials+cuda cuda_arch={0}'.format(sm_)) + depends_on('pumi@2.2.3:', when='@4.2.0:+pumi') depends_on('pumi', when='+pumi~shared') depends_on('pumi+shared', when='+pumi+shared') depends_on('gslib@1.0.5:+mpi', when='+gslib+mpi') depends_on('gslib@1.0.5:~mpi~mpiio', when='+gslib~mpi') depends_on('suite-sparse', when='+suite-sparse') depends_on('superlu-dist', when='+superlu-dist') - depends_on('strumpack@3.0.0:', when='+strumpack') + depends_on('strumpack@3.0.0:', when='+strumpack~shared') + depends_on('strumpack@3.0.0:+shared', when='+strumpack+shared') + for sm_ in CudaPackage.cuda_arch_values: + depends_on('strumpack+cuda cuda_arch={0}'.format(sm_), + when='+strumpack+cuda cuda_arch={0}'.format(sm_)) # The PETSc tests in MFEM will fail if PETSc is not configured with # SuiteSparse and MUMPS. On the other hand, if we require the variants # '+suite-sparse+mumps' of PETSc, the xsdk package concretization fails. depends_on('petsc@3.8:+mpi+double+hypre', when='+petsc') + depends_on('slepc@3.8.0:', when='+slepc') # Recommended when building outside of xsdk: # depends_on('petsc@3.8:+mpi+double+hypre+suite-sparse+mumps', # when='+petsc') @@ -235,33 +247,53 @@ class Mfem(Package): # when using hypre version >= 2.16.0. # This issue is resolved in v4.2. conflicts('+strumpack', when='mfem@4.0.0:4.1.99 ^hypre@2.16.0:') + conflicts('+strumpack ^strumpack+cuda', when='~cuda') depends_on('occa@1.0.8:', when='@:4.1.99+occa') depends_on('occa@1.1.0:', when='@4.2.0:+occa') depends_on('occa+cuda', when='+occa+cuda') + # TODO: propagate '+rocm' variant to occa when it is supported depends_on('raja@0.10.0:', when='@4.0.1:+raja') depends_on('raja@0.7.0:0.9.0', when='@4.0.0+raja') - depends_on('raja+cuda', when='+raja+cuda') + for sm_ in CudaPackage.cuda_arch_values: + depends_on('raja+cuda cuda_arch={0}'.format(sm_), + when='+raja+cuda cuda_arch={0}'.format(sm_)) + for gfx in ROCmPackage.amdgpu_targets: + depends_on('raja+rocm amdgpu_target={0}'.format(gfx), + when='+raja+rocm amdgpu_target={0}'.format(gfx)) depends_on('libceed@0.6:', when='@:4.1.99+libceed') depends_on('libceed@0.7:', when='@4.2.0:+libceed') - depends_on('libceed+cuda', when='+libceed+cuda') + for sm_ in CudaPackage.cuda_arch_values: + depends_on('libceed+cuda cuda_arch={0}'.format(sm_), + when='+libceed+cuda cuda_arch={0}'.format(sm_)) + for gfx in ROCmPackage.amdgpu_targets: + depends_on('libceed+rocm amdgpu_target={0}'.format(gfx), + when='+libceed+rocm amdgpu_target={0}'.format(gfx)) depends_on('umpire@2.0.0:', when='+umpire') - depends_on('umpire+cuda', when='+umpire+cuda') + for sm_ in CudaPackage.cuda_arch_values: + depends_on('umpire+cuda cuda_arch={0}'.format(sm_), + when='+umpire+cuda cuda_arch={0}'.format(sm_)) + for gfx in ROCmPackage.amdgpu_targets: + depends_on('umpire+rocm amdgpu_target={0}'.format(gfx), + when='+umpire+rocm amdgpu_target={0}'.format(gfx)) - depends_on('amgx', when='+amgx') - # MPI is enabled by default - depends_on('amgx~mpi', when='+amgx~mpi') + # AmgX: propagate the cuda_arch and mpi settings: for sm_ in CudaPackage.cuda_arch_values: - depends_on('amgx cuda_arch={0}'.format(sm_), - when='+amgx cuda_arch=sm_{0}'.format(sm_)) + depends_on('amgx+mpi cuda_arch={0}'.format(sm_), + when='+amgx+mpi cuda_arch={0}'.format(sm_)) + depends_on('amgx~mpi cuda_arch={0}'.format(sm_), + when='+amgx~mpi cuda_arch={0}'.format(sm_)) patch('mfem_ppc_build.patch', when='@3.2:3.3.0 arch=ppc64le') patch('mfem-3.4.patch', when='@3.4.0') patch('mfem-3.3-3.4-petsc-3.9.patch', when='@3.3.0:3.4.0 +petsc ^petsc@3.9.0:') + patch('mfem-4.2-umpire.patch', when='@4.2.0+umpire') + patch('mfem-4.2-slepc.patch', when='@4.2.0+slepc') + patch('mfem-4.2-petsc-3.15.0.patch', when='@4.2.0+petsc ^petsc@3.15.0:') # Patch to fix MFEM makefile syntax error. See # https://github.com/mfem/mfem/issues/1042 for the bug report and @@ -339,6 +371,21 @@ class Mfem(Package): return lib return LibraryList([]) + # Determine how to run MPI tests, e.g. when using '--test=root', when + # Spack is run inside a batch system job. + mfem_mpiexec = 'mpirun' + mfem_mpiexec_np = '-np' + if 'SLURM_JOBID' in os.environ: + mfem_mpiexec = 'srun' + mfem_mpiexec_np = '-n' + elif 'LSB_JOBID' in os.environ: + if 'LLNL_COMPUTE_NODES' in os.environ: + mfem_mpiexec = 'lrun' + mfem_mpiexec_np = '-n' + else: + mfem_mpiexec = 'jsrun' + mfem_mpiexec_np = '-p' + metis5_str = 'NO' if ('+metis' in spec) and spec['metis'].satisfies('@5:'): metis5_str = 'YES' @@ -365,6 +412,7 @@ class Mfem(Package): 'MFEM_USE_SUITESPARSE=%s' % yes_no('+suite-sparse'), 'MFEM_USE_SUNDIALS=%s' % yes_no('+sundials'), 'MFEM_USE_PETSC=%s' % yes_no('+petsc'), + 'MFEM_USE_SLEPC=%s' % yes_no('+slepc'), 'MFEM_USE_PUMI=%s' % yes_no('+pumi'), 'MFEM_USE_GSLIB=%s' % yes_no('+gslib'), 'MFEM_USE_NETCDF=%s' % yes_no('+netcdf'), @@ -373,11 +421,14 @@ class Mfem(Package): 'MFEM_USE_OPENMP=%s' % yes_no('+openmp'), 'MFEM_USE_CONDUIT=%s' % yes_no('+conduit'), 'MFEM_USE_CUDA=%s' % yes_no('+cuda'), + 'MFEM_USE_HIP=%s' % yes_no('+rocm'), 'MFEM_USE_OCCA=%s' % yes_no('+occa'), 'MFEM_USE_RAJA=%s' % yes_no('+raja'), 'MFEM_USE_AMGX=%s' % yes_no('+amgx'), 'MFEM_USE_CEED=%s' % yes_no('+libceed'), - 'MFEM_USE_UMPIRE=%s' % yes_no('+umpire')] + 'MFEM_USE_UMPIRE=%s' % yes_no('+umpire'), + 'MFEM_MPIEXEC=%s' % mfem_mpiexec, + 'MFEM_MPIEXEC_NP=%s' % mfem_mpiexec_np] cxxflags = spec.compiler_flags['cxxflags'] @@ -400,7 +451,7 @@ class Mfem(Package): cxxflags = [(xcompiler + flag) for flag in cxxflags] if '+cuda' in spec: cxxflags += [ - '-x=cu --expt-extended-lambda -arch=%s' % cuda_arch, + '-x=cu --expt-extended-lambda -arch=sm_%s' % cuda_arch, '-ccbin %s' % (spec['mpi'].mpicxx if '+mpi' in spec else env['CXX'])] if self.spec.satisfies('@4.0.0:'): @@ -414,8 +465,8 @@ class Mfem(Package): options += ['STATIC=NO'] if '+shared' in spec: options += [ - 'SHARED=YES', 'PICFLAG=%s' % self.compiler.cxx_pic_flag - ] + 'SHARED=YES', + 'PICFLAG=%s' % (xcompiler + self.compiler.cxx_pic_flag)] if '+mpi' in spec: options += ['MPICXX=%s' % spec['mpi'].mpicxx] @@ -459,24 +510,51 @@ class Mfem(Package): # fortran library and also the MPI fortran library: if '~shared' in strumpack: if os.path.basename(env['FC']) == 'gfortran': - sp_lib += ['-lgfortran'] - if '^mpich' in strumpack: + gfortran = Executable(env['FC']) + libext = 'dylib' if sys.platform == 'darwin' else 'so' + libfile = os.path.abspath(gfortran( + '-print-file-name=libgfortran.%s' % libext, + output=str).strip()) + gfortran_lib = LibraryList(libfile) + sp_lib += [ld_flags_from_library_list(gfortran_lib)] + if ('^mpich' in strumpack) or ('^mvapich2' in strumpack): sp_lib += ['-lmpifort'] elif '^openmpi' in strumpack: sp_lib += ['-lmpi_mpifh'] + elif '^spectrum-mpi' in strumpack: + sp_lib += ['-lmpi_ibm_mpifh'] if '+openmp' in strumpack: - sp_opt += [self.compiler.openmp_flag] - if '^scalapack' in strumpack: + # The '+openmp' in the spec means strumpack will TRY to find + # OpenMP; if not found, we should not add any flags -- how do + # we figure out if strumpack found OpenMP? + if not self.spec.satisfies('%apple-clang'): + sp_opt += [xcompiler + self.compiler.openmp_flag] + if '^parmetis' in strumpack: + parmetis = strumpack['parmetis'] + sp_opt += [parmetis.headers.cpp_flags] + sp_lib += [ld_flags_from_library_list(parmetis.libs)] + if '^netlib-scalapack' in strumpack: scalapack = strumpack['scalapack'] sp_opt += ['-I%s' % scalapack.prefix.include] sp_lib += [ld_flags_from_dirs([scalapack.prefix.lib], ['scalapack'])] + elif '^scalapack' in strumpack: + scalapack = strumpack['scalapack'] + sp_opt += [scalapack.headers.cpp_flags] + sp_lib += [ld_flags_from_library_list(scalapack.libs)] if '+butterflypack' in strumpack: bp = strumpack['butterflypack'] sp_opt += ['-I%s' % bp.prefix.include] sp_lib += [ld_flags_from_dirs([bp.prefix.lib], ['dbutterflypack', 'zbutterflypack'])] + if '+zfp' in strumpack: + zfp = strumpack['zfp'] + sp_opt += ['-I%s' % zfp.prefix.include] + sp_lib += [ld_flags_from_dirs([zfp.prefix.lib], ['zfp'])] + if '+cuda' in strumpack: + # assuming also ('+cuda' in spec) + sp_lib += ['-lcusolver', '-lcublas'] options += [ 'STRUMPACK_OPT=%s' % ' '.join(sp_opt), 'STRUMPACK_LIB=%s' % ' '.join(sp_lib)] @@ -504,6 +582,12 @@ class Mfem(Package): else: options += ['PETSC_DIR=%s' % petsc.prefix] + if '+slepc' in spec: + slepc = spec['slepc'] + options += [ + 'SLEPC_OPT=%s' % slepc.headers.cpp_flags, + 'SLEPC_LIB=%s' % ld_flags_from_library_list(slepc.libs)] + if '+pumi' in spec: pumi_libs = ['pumi', 'crv', 'ma', 'mds', 'apf', 'pcu', 'gmi', 'parma', 'lion', 'mth', 'apf_zoltan', 'spr'] @@ -563,12 +647,19 @@ class Mfem(Package): 'LIBUNWIND_LIB=%s' % ld_flags_from_library_list(libs)] if '+openmp' in spec: - options += ['OPENMP_OPT=%s' % self.compiler.openmp_flag] + options += [ + 'OPENMP_OPT=%s' % (xcompiler + self.compiler.openmp_flag)] if '+cuda' in spec: options += [ 'CUDA_CXX=%s' % join_path(spec['cuda'].prefix, 'bin', 'nvcc'), - 'CUDA_ARCH=%s' % cuda_arch] + 'CUDA_ARCH=sm_%s' % cuda_arch] + + if '+rocm' in spec: + amdgpu_target = ','.join(spec.variants['amdgpu_target'].value) + options += [ + 'HIP_CXX=%s' % spec['hip'].hipcc, + 'HIP_ARCH=%s' % amdgpu_target] if '+occa' in spec: options += ['OCCA_OPT=-I%s' % spec['occa'].prefix.include, @@ -730,9 +821,13 @@ class Mfem(Package): @when('@4.1.0') def patch(self): # Remove the byte order mark since it messes with some compilers - filter_file(u'\uFEFF', '', 'fem/gslib.hpp') - filter_file(u'\uFEFF', '', 'fem/gslib.cpp') - filter_file(u'\uFEFF', '', 'linalg/hiop.hpp') + files_with_bom = [ + 'fem/gslib.hpp', 'fem/gslib.cpp', 'linalg/hiop.hpp', + 'miniapps/gslib/field-diff.cpp', 'miniapps/gslib/findpts.cpp', + 'miniapps/gslib/pfindpts.cpp'] + bom = '\xef\xbb\xbf' if sys.version_info < (3,) else u'\ufeff' + for f in files_with_bom: + filter_file(bom, '', f) @property def suitesparse_components(self): @@ -745,13 +840,14 @@ class Mfem(Package): @property def sundials_components(self): """Return the SUNDIALS components needed by MFEM.""" + spec = self.spec sun_comps = 'arkode,cvodes,nvecserial,kinsol' - if '+mpi' in self.spec: - if self.spec.satisfies('@4.2:'): + if '+mpi' in spec: + if spec.satisfies('@4.2:'): sun_comps += ',nvecparallel,nvecmpiplusx' else: sun_comps += ',nvecparhyp,nvecparallel' - if '+cuda' in self.spec: + if '+cuda' in spec and '+cuda' in spec['sundials']: sun_comps += ',nveccuda' return sun_comps diff --git a/var/spack/repos/builtin/packages/mfem/test_builds.sh b/var/spack/repos/builtin/packages/mfem/test_builds.sh index 4f3c3f787b..fd6624eaf5 100755 --- a/var/spack/repos/builtin/packages/mfem/test_builds.sh +++ b/var/spack/repos/builtin/packages/mfem/test_builds.sh @@ -2,50 +2,66 @@ # Set a compiler to test with, e.g. '%gcc', '%clang', etc. compiler='' +cuda_arch="70" mfem='mfem'${compiler} mfem_dev='mfem@develop'${compiler} backends='+occa+raja+libceed' -# Using occa@develop to help the spack concretization -backends_specs='^occa@develop~cuda ^raja~openmp' +backends_specs='^occa~cuda ^raja~openmp' -# As of 03/20/20 +mpfr breaks one of the unit tests in both @4.1.0 and @develop, -# so it is disabled here for now. -# mpfr='+mpfr' -mpfr='' +# help the concrtizer find suitable hdf5 version (conduit constraint) +hdf5_spec='^hdf5@1.8.19:1.8.999' +# petsc spec +petsc_spec='^petsc+suite-sparse+mumps' +# strumpack spec without cuda +strumpack_spec='^strumpack~slate~openmp~cuda' +strumpack_cuda_spec='^strumpack~slate~openmp' builds=( # preferred version: ${mfem} ${mfem}'~mpi~metis~zlib' - # NOTE: Skip +strumpack since mfem needs hypre < 2.16.0 in that case - ${mfem}"$backends"'+superlu-dist+suite-sparse+petsc \ - +sundials+pumi+gslib'${mpfr}'+netcdf+zlib+gnutls+libunwind+conduit \ - ^petsc+suite-sparse+mumps'" $backends_specs" + ${mfem}"$backends"'+superlu-dist+strumpack+suite-sparse+petsc+slepc \ + +sundials+pumi+gslib+mpfr+netcdf+zlib+gnutls+libunwind+conduit \ + '"$backends_specs $petsc_spec $strumpack_spec $hdf5_spec" ${mfem}'~mpi \ - '"$backends"'+suite-sparse+sundials+gslib'${mpfr}'+netcdf \ - +zlib+gnutls+libunwind+conduit'" $backends_specs" + '"$backends"'+suite-sparse+sundials+gslib+mpfr+netcdf \ + +zlib+gnutls+libunwind+conduit'" $backends_specs $hdf5_spec" # develop version: ${mfem_dev}'+shared~static' ${mfem_dev}'+shared~static~mpi~metis~zlib' - # NOTE: Skip +strumpack since mfem needs hypre < 2.16.0 in that case + # NOTE: Shared build with +gslib works on mac but not on linux + # FIXME: As of 2020/11/03 the next config fails in PETSc ex5p: + # ${mfem_dev}'+shared~static \ + # '"$backends"'+superlu-dist+strumpack+suite-sparse+petsc \ + # +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \ + # '"$backends_specs $petsc_spec $strumpack_spec $hdf5_spec" + # Removing just petsc works: ${mfem_dev}'+shared~static \ - '"$backends"'+superlu-dist+suite-sparse+petsc \ - +sundials+pumi'${mpfr}'+netcdf+zlib+gnutls+libunwind+conduit \ - ^petsc+suite-sparse+mumps'" $backends_specs" + '"$backends"'+superlu-dist+strumpack+suite-sparse \ + +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \ + '"$backends_specs $strumpack_spec $hdf5_spec" + # Removing just strumpack works on linux, fails on mac: + # ${mfem_dev}'+shared~static \ + # '"$backends"'+superlu-dist+suite-sparse+petsc \ + # +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \ + # '"$backends_specs $petsc_spec $hdf5_spec" + # Petsc and strumpack: fails on linux and mac in PETSc ex5p: + # ${mfem_dev}'+shared~static +strumpack+petsc \ + # '$petsc_spec $strumpack_spec" + ${mfem_dev}'+shared~static~mpi \ - '"$backends"'+suite-sparse+sundials'${mpfr}'+netcdf \ - +zlib+gnutls+libunwind+conduit'" $backends_specs" + '"$backends"'+suite-sparse+sundials+mpfr+netcdf \ + +zlib+gnutls+libunwind+conduit'" $backends_specs $hdf5_spec" ) builds2=( # preferred version ${mfem}"$backends $backends_specs" ${mfem}'+superlu-dist' - # NOTE: On mac +strumpack works only with gcc, as of 03/20/20. - ${mfem}'+strumpack ^hypre@2.15.1' + ${mfem}'+strumpack'" $strumpack_spec" ${mfem}'+suite-sparse~mpi' ${mfem}'+suite-sparse' ${mfem}'+sundials~mpi' @@ -54,16 +70,17 @@ builds2=( ${mfem}'+gslib' ${mfem}'+netcdf~mpi' ${mfem}'+netcdf' - ${mfem}${mpfr} + ${mfem}'+mpfr' ${mfem}'+gnutls' ${mfem}'+conduit~mpi' ${mfem}'+conduit' - ${mfem}'+petsc ^petsc+suite-sparse+mumps' + ${mfem}'+umpire' + ${mfem}'+petsc'" $petsc_spec" + ${mfem}'+petsc+slepc'" $petsc_spec" # develop version ${mfem_dev}"$backends $backends_specs" ${mfem_dev}'+superlu-dist' - # NOTE: On mac +strumpack works only with gcc, as of 03/20/20. - ${mfem_dev}'+strumpack ^hypre@2.15.1' + ${mfem_dev}'+strumpack'" $strumpack_spec" ${mfem_dev}'+suite-sparse~mpi' ${mfem_dev}'+suite-sparse' ${mfem_dev}'+sundials~mpi' @@ -72,19 +89,50 @@ builds2=( ${mfem_dev}'+gslib' ${mfem_dev}'+netcdf~mpi' ${mfem_dev}'+netcdf' - ${mfem_dev}${mpfr} + ${mfem_dev}'+mpfr' ${mfem_dev}'+gnutls' ${mfem_dev}'+conduit~mpi' ${mfem_dev}'+conduit' - ${mfem_dev}'+petsc ^petsc+suite-sparse+mumps' + ${mfem_dev}'+umpire' + ${mfem_dev}'+petsc'" $petsc_spec" + ${mfem_dev}'+petsc+slepc'" $petsc_spec" ) +builds_cuda=( + ${mfem}'+cuda cuda_arch='"${cuda_arch}" + + ${mfem}'+cuda+raja+occa+libceed cuda_arch='"${cuda_arch}"' \ + ^raja+cuda~openmp' + + ${mfem}'+cuda+openmp+raja+occa+libceed cuda_arch='"${cuda_arch}"' \ + +superlu-dist+strumpack+suite-sparse+petsc+slepc \ + +sundials+pumi+gslib+mpfr+netcdf+zlib+gnutls+libunwind+conduit \ + ^raja+cuda+openmp'" $strumpack_cuda_spec $petsc_spec $hdf5_spec" + + # same builds as above with ${mfem_dev} + ${mfem_dev}'+cuda cuda_arch='"${cuda_arch}" + + ${mfem_dev}'+cuda+raja+occa+libceed cuda_arch='"${cuda_arch}"' \ + ^raja+cuda~openmp' + + # add '^sundials+hypre' to help the concretizer + ${mfem_dev}'+cuda+openmp+raja+occa+libceed cuda_arch='"${cuda_arch}"' \ + +superlu-dist+strumpack+suite-sparse+petsc+slepc \ + +sundials+pumi+gslib+mpfr+netcdf+zlib+gnutls+libunwind+conduit \ + ^raja+cuda+openmp'" $strumpack_cuda_spec $petsc_spec"' \ + ^sundials+hypre'" $hdf5_spec" +) + + trap 'printf "\nScript interrupted.\n"; exit 33' INT SEP='==========================================================================' sep='--------------------------------------------------------------------------' -for bld in "${builds[@]}" "${builds2[@]}"; do +run_builds=("${builds[@]}" "${builds2[@]}") +# run_builds=("${builds_cuda[@]}") + +for bld in "${run_builds[@]}"; do printf "\n%s\n" "${SEP}" printf " %s\n" "${bld}" printf "%s\n" "${SEP}" diff --git a/var/spack/repos/builtin/packages/nekrs/package.py b/var/spack/repos/builtin/packages/nekrs/package.py index 731c34e21f..3f1fd44e69 100644 --- a/var/spack/repos/builtin/packages/nekrs/package.py +++ b/var/spack/repos/builtin/packages/nekrs/package.py @@ -22,15 +22,9 @@ class Nekrs(Package, CudaPackage, ROCmPackage): version('21.0', tag='v21.0') - variant('cuda', - default=False, - description='Activates support for CUDA') variant('opencl', default=False, description='Activates support for OpenCL') - variant('hip', - default=False, - description='Activates support for HIP') # Conflicts: # nekrs includes following packages, but in order to build as part of @@ -44,9 +38,6 @@ class Nekrs(Package, CudaPackage, ROCmPackage): depends_on('git') depends_on('cmake') - depends_on('cuda', when='+cuda') - depends_on('hip', when='+rocm') - @run_before('install') def fortran_check(self): if not self.compiler.f77: diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 6617df1ba2..7c397999b8 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -5,7 +5,7 @@ import os -class Petsc(Package): +class Petsc(Package, CudaPackage, ROCmPackage): """PETSc is a suite of data structures and routines for the scalable (parallel) solution of scientific applications modeled by partial differential equations. @@ -76,7 +76,6 @@ class Petsc(Package): variant('shared', default=True, description='Enables the build of shared libraries') variant('mpi', default=True, description='Activates MPI support') - variant('cuda', default=False, description='Activates CUDA support') variant('double', default=True, description='Switches between single and double precision') variant('complex', default=False, description='Build with complex numbers') @@ -192,6 +191,9 @@ class Petsc(Package): depends_on('lapack') depends_on('mpi', when='+mpi') depends_on('cuda', when='+cuda') + depends_on('hip', when='+rocm') + depends_on('hipblas', when='+rocm') + depends_on('hipsparse', when='+rocm') # Build dependencies depends_on('python@2.6:2.8', type='build', when='@:3.10.99') @@ -375,40 +377,57 @@ class Petsc(Package): '--with-scalapack=0' ]) - if spec.satisfies('+openmp'): - options.append('--with-openmp=1') - # Activates library support if needed (i.e. direct dependency) if '^libjpeg-turbo' in spec: - jpeg_library = 'libjpeg-turbo' + jpeg_library = ('libjpeg-turbo', 'libjpeg') else: jpeg_library = 'libjpeg' - for library in ('cuda', 'metis', 'hypre', 'parmetis', 'mumps', - 'trilinos', 'fftw', 'valgrind', 'gmp', 'libpng', - 'giflib', 'mpfr', 'netcdf-c', 'parallel-netcdf', - 'moab', 'random123', 'exodusii', 'cgns', 'memkind', - 'p4est', 'saws', 'libyaml', 'hwloc', jpeg_library): + for library in ( + 'cuda', + 'hip', + 'metis', + 'hypre', + 'parmetis', + 'mumps', + 'trilinos', + 'fftw', + 'valgrind', + 'gmp', + 'libpng', + 'giflib', + 'mpfr', + ('netcdf-c', 'netcdf'), + ('parallel-netcdf', 'pnetcdf'), + 'moab', + 'openmp', + 'random123', + 'exodusii', + 'cgns', + 'memkind', + 'p4est', + 'saws', + ('libyaml', 'yaml'), + 'hwloc', + jpeg_library, + ): # Cannot check `library in spec` because of transitive deps # Cannot check variants because parmetis keys on +metis + if isinstance(library, tuple): + library, petsclibname = library + else: + petsclibname = library + library_requested = library in spec.dependencies_dict() options.append( '--with-{library}={value}'.format( - library=('libjpeg' if library == 'libjpeg-turbo' - else 'netcdf' if library == 'netcdf-c' - else 'pnetcdf' if library == 'parallel-netcdf' - else 'yaml' if library == 'libyaml' - else library), + library=petsclibname, value=('1' if library_requested else '0')) ) if library_requested: options.append( '--with-{library}-dir={path}'.format( - library=('libjpeg' if library == 'libjpeg-turbo' - else 'netcdf' if library == 'netcdf-c' - else 'pnetcdf' if library == 'parallel-netcdf' - else 'yaml' if library == 'libyaml' - else library), path=spec[library].prefix) + library=petsclibname, path=spec[library].prefix) ) # PETSc does not pick up SuperluDist from the dir as they look for diff --git a/var/spack/repos/builtin/packages/pumi/package.py b/var/spack/repos/builtin/packages/pumi/package.py index 08930cd70d..b32f7f4f3e 100644 --- a/var/spack/repos/builtin/packages/pumi/package.py +++ b/var/spack/repos/builtin/packages/pumi/package.py @@ -78,6 +78,8 @@ class Pumi(CMakePackage): ('ON' if '~simmodsuite_version_check' in spec else 'OFF'), '-DMESHES=%s' % join_path(self.stage.source_path, 'pumi_meshes') ] + if spec.satisfies('@2.2.3'): + args += ['-DCMAKE_CXX_STANDARD=11'] if self.spec.satisfies('simmodsuite=base'): args.append('-DENABLE_SIMMETRIX=ON') if self.spec.satisfies('simmodsuite=kernels') or \ diff --git a/var/spack/repos/builtin/packages/suite-sparse/package.py b/var/spack/repos/builtin/packages/suite-sparse/package.py index 8b3c049faf..0182658afc 100644 --- a/var/spack/repos/builtin/packages/suite-sparse/package.py +++ b/var/spack/repos/builtin/packages/suite-sparse/package.py @@ -38,7 +38,7 @@ class SuiteSparse(Package): variant('cuda', default=False, description='Build with CUDA') variant('openmp', default=False, description='Build with OpenMP') - depends_on('mpfr', type=('build', 'link'), when='@5.8.0:') + depends_on('mpfr@4.0.0:', type=('build', 'link'), when='@5.8.0:') depends_on('gmp', type=('build', 'link'), when='@5.8.0:') depends_on('blas') depends_on('lapack') |