More variants for MFEM (#4533)

* fixes #967

* Version bump to 0.9.1

- Bugfixes for spack find
- 0.9.1 can read specs from current develop.

* Don't assume spack is in the path when building docs.

* adding mfem variants; do not assume build defaults

* adjusting format to clarify options

* fixing some build issues

* fixing mfem+sundials

* removing mesquite variant...turns out it is obsolete

* adding examples to mfem; fixing sundials darwin installs

* adding some additional variants and setting some configure flags by fiat

* switching most check_variants logic to conflicts() directives

* adding comment about cmake config

* fixing flake8 issues; thought I had already done that

* moving func def into install method

* switch yes_no to no_yes; fix miss-spelling of miniapps

* switching make target all to lib

* changing yes_no to function as requested; fix problem with same in sundials

* fix missing import sys directive

* adding master and develop versions to amrex

3 files changed, 205 insertions, 131 deletions

diff --git a/var/spack/repos/builtin/packages/amrex/package.py b/var/spack/repos/builtin/packages/amrex/package.py
index a13200b3e0..772bbce3a5 100644
--- a/var/spack/repos/builtin/packages/amrex/package.py
+++ b/var/spack/repos/builtin/packages/amrex/package.py
@@ -33,8 +33,9 @@ class Amrex(CMakePackage):
     homepage = "https://ccse.lbl.gov/AMReX/index.html"
     url      = "https://github.com/AMReX-Codes/amrex.git"
 
-    version('develop', git='https://github.com/AMReX-Codes/amrex.git', tag='master')
     version('17.06', git='https://github.com/AMReX-Codes/amrex.git', commit='836d3c7')
+    version('master', git='https://github.com/AMReX-Codes/amrex.git', tag='master')
+    version('develop', git='https://github.com/AMReX-Codes/amrex.git', tag='develop')
 
     variant('dims',
         default='3',
diff --git a/var/spack/repos/builtin/packages/mfem/package.py b/var/spack/repos/builtin/packages/mfem/package.py
index 2848d77887..ffd3233a9f 100644
--- a/var/spack/repos/builtin/packages/mfem/package.py
+++ b/var/spack/repos/builtin/packages/mfem/package.py
@@ -23,6 +23,7 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 from spack import *
+import re
 
 
 class Mfem(Package):
@@ -62,132 +63,194 @@ class Mfem(Package):
             '841ea5cf58de6fae4de0f553b0e01ebaab9cd9c67fa821e8a715666ecf18fc57',
             url='http://goo.gl/xrScXn', extension='.tar.gz')
 
-    variant('metis', default=False, description='Activate support for metis')
-    variant('hypre', default=False, description='Activate support for hypre')
-    variant('suite-sparse', default=False,
-            description='Activate support for SuiteSparse')
-    variant('mpi', default=True, description='Activate support for MPI')
+    variant('mpi', default=True,
+        description='Enable MPI parallelism')
+    variant('hypre', default=True,
+        description='Required for MPI parallelism')
+    variant('openmp', default=False,
+        description='Enable OpenMP parallelism')
+    variant('threadsafe', default=False,
+        description=('Enable thread safe features.'
+            ' Required for OpenMP.'
+            ' May cause minor performance issues.'))
     variant('superlu-dist', default=False,
-            description='Activate support for SuperLU_Dist')
-    variant('lapack', default=False, description='Activate support for LAPACK')
-    variant('debug', default=False, description='Build debug version')
-    variant('netcdf', default=False, description='Activate NetCDF support')
+        description='Enable MPI parallel, sparse direct solvers')
+    variant('suite-sparse', default=False,
+        description='Enable serial, sparse direct solvers')
+    variant('petsc', default=False,
+        description='Enable PETSc solvers, preconditioners, etc..')
+    variant('sundials', default=False,
+        description='Enable Sundials time integrators')
+    variant('mpfr', default=False,
+        description='Enable precise, 1D quadrature rules')
+    variant('lapack', default=False,
+        description='Use external blas/lapack routines')
+    variant('debug', default=False,
+        description='Build debug instead of optimized version')
+    variant('netcdf', default=False,
+        description='Enable Cubit/Genesis reader')
+    variant('gzstream', default=True,
+        description='Support zip\'d streams for I/O')
+    variant('examples', default=False,
+        description='Build and install examples')
+    variant('miniapps', default=False,
+        description='Build and install miniapps')
+
+    conflicts('+mpi', when='~hypre')
+    conflicts('+suite-sparse', when='~lapack')
+    conflicts('+superlu-dist', when='@:3.1')
+    conflicts('+netcdf', when='@:3.1')
 
     depends_on('blas', when='+lapack')
+    depends_on('blas', when='+suite-sparse')
     depends_on('lapack', when='+lapack')
+    depends_on('lapack', when='+suite-sparse')
 
     depends_on('mpi', when='+mpi')
-    depends_on('metis', when='+mpi')
-    depends_on('hypre', when='+mpi')
-
-    depends_on('hypre', when='+hypre')
-
-    depends_on('metis@4:', when='+metis')
-
-    depends_on('suite-sparse', when='+suite-sparse')
-    depends_on('blas', when='+suite-sparse')
-    depends_on('lapack', when='+suite-sparse')
+    depends_on('metis')
+    depends_on('parmetis', when='+superlu-dist')
+    depends_on('metis@5:', when='+superlu-dist')
     depends_on('metis@5:', when='+suite-sparse ^suite-sparse@4.5:')
-    depends_on('cmake', when='^metis@5:', type='build')
+    depends_on('hypre~internal-superlu', when='+mpi')
+    depends_on('hypre@develop~internal-superlu', when='+petsc +hypre')
 
+    depends_on('sundials@2.7:+hypre', when='+sundials')
+    depends_on('suite-sparse', when='+suite-sparse')
     depends_on('superlu-dist', when='@3.2: +superlu-dist')
+    depends_on('petsc@develop', when='+petsc')
 
+    depends_on('mpfr', when='+mpfr')
+    depends_on('cmake', when='^metis@5:', type='build')
     depends_on('netcdf', when='@3.2: +netcdf')
     depends_on('zlib', when='@3.2: +netcdf')
     depends_on('hdf5', when='@3.2: +netcdf')
+    depends_on('libunwind', when='+debug')
+    depends_on('zlib', when='+gzstream')
 
     patch('mfem_ppc_build.patch', when='@3.2:3.3 arch=ppc64le')
 
     def check_variants(self, spec):
-        if '+mpi' in spec and ('+hypre' not in spec or '+metis' not in spec):
-            raise InstallError('mfem+mpi must be built with +hypre ' +
-                               'and +metis!')
-        if '+suite-sparse' in spec and ('+metis' not in spec or
-                                        '+lapack' not in spec):
-            raise InstallError('mfem+suite-sparse must be built with ' +
-                               '+metis and +lapack!')
         if 'metis@5:' in spec and '%clang' in spec and (
                 '^cmake %gcc' not in spec):
             raise InstallError('To work around CMake bug with clang, must ' +
                                'build mfem with mfem[+variants] %clang ' +
                                '^cmake %gcc to force CMake to build with gcc')
-        if '@:3.1' in spec and '+superlu-dist' in spec:
-            raise InstallError('MFEM does not support SuperLU_DIST for ' +
-                               'versions 3.1 and earlier')
-        if '@:3.1' in spec and '+netcdf' in spec:
-            raise InstallError('MFEM does not support NetCDF for versions' +
-                               '3.1 and earlier')
         return
 
+    #
+    # Note: Although MFEM does support CMake configuration, MFEM
+    # development team indicates that vanilla GNU Make is the
+    # preferred mode of configuration of MFEM and the mode most
+    # likely to be up to date in supporting *all* of MFEM's
+    # configuration options. So, don't use CMake
+    #
     def install(self, spec, prefix):
         self.check_variants(spec)
 
-        options = ['PREFIX=%s' % prefix]
-
-        if '+lapack' in spec:
-            lapack_lib = (spec['lapack'].libs + spec['blas'].libs).ld_flags  # NOQA: ignore=E501
-            options.extend([
-                'MFEM_USE_LAPACK=YES',
-                'LAPACK_OPT=-I%s' % spec['lapack'].prefix.include,
-                'LAPACK_LIB=%s' % lapack_lib])
+        def yes_no(varstr):
+            return 'YES' if varstr in self.spec else 'NO'
+
+        metis5_str = 'NO'
+        if '+superlu-dist' in spec or  \
+            spec.satisfies('+suite-sparse ^suite-sparse@4.5:') or \
+                spec['metis'].satisfies('@5:'):
+                metis5_str = 'YES'
+
+        threadsafe_str = 'NO'
+        if '+openmp' in spec or '+threadsafe' in spec:
+            threadsafe_str = 'YES'
+
+        options = [
+            'PREFIX=%s' % prefix,
+            'MFEM_USE_MEMALLOC=YES',
+            'MFEM_DEBUG=%s' % yes_no('+debug'),
+            'MFEM_USE_LIBUNWIND=%s' % yes_no('+debug'),
+            'MFEM_USE_GZSTREAM=%s' % yes_no('+gzstream'),
+            'MFEM_USE_METIS_5=%s' % metis5_str,
+            'MFEM_THREAD_SAFE=%s' % threadsafe_str,
+            'MFEM_USE_MPI=%s' % yes_no('+mpi'),
+            'MFEM_USE_LAPACK=%s' % yes_no('+lapack'),
+            'MFEM_USE_SUPERLU=%s' % yes_no('+superlu-dist'),
+            'MFEM_USE_SUITESPARSE=%s' % yes_no('+suite-sparse'),
+            'MFEM_USE_SUNDIALS=%s' % yes_no('+sundials'),
+            'MFEM_USE_PETSC=%s' % yes_no('+petsc'),
+            'MFEM_USE_NETCDF=%s' % yes_no('+netcdf'),
+            'MFEM_USE_MPFR=%s' % yes_no('+mpfr'),
+            'MFEM_USE_OPENMP=%s' % yes_no('+openmp')]
 
         if '+hypre' in spec:
-            options.extend([
+            options += [
                 'HYPRE_DIR=%s' % spec['hypre'].prefix,
                 'HYPRE_OPT=-I%s' % spec['hypre'].prefix.include,
-                'HYPRE_LIB=-L%s' % spec['hypre'].prefix.lib +
-                ' -lHYPRE'])
+                'HYPRE_LIB=-L%s' % spec['hypre'].prefix.lib + ' -lHYPRE']
 
-        if 'parmetis' in spec:
-            metis_lib = '-L%s -lparmetis -lmetis' % spec['parmetis'].prefix.lib
-            metis_str = 'MFEM_USE_METIS_5=YES'
-            options.extend([metis_str,
-                            'METIS_DIR=%s' % spec['parmetis'].prefix,
-                            'METIS_OPT=-I%s' % spec['parmetis'].prefix.include,
-                            'METIS_LIB=%s' % metis_lib])
-        elif 'metis' in spec:
-            metis_lib = '-L%s -lmetis' % spec['metis'].prefix.lib
-            if spec['metis'].satisfies('@5:'):
-                metis_str = 'MFEM_USE_METIS_5=YES'
-            else:
-                metis_str = 'MFEM_USE_METIS_5=NO'
-            options.extend([
-                metis_str,
-                'METIS_DIR=%s' % spec['metis'].prefix,
-                'METIS_OPT=-I%s' % spec['metis'].prefix.include,
-                'METIS_LIB=%s' % metis_lib])
-
-        if 'mpi' in spec:
-            options.extend(['MFEM_USE_MPI=YES'])
+        if '+lapack' in spec:
+            lapack_lib = (spec['lapack'].libs + spec['blas'].libs).ld_flags  # NOQA: ignore=E501
+            options += [
+                'LAPACK_OPT=-I%s' % spec['lapack'].prefix.include,
+                'LAPACK_LIB=%s' % lapack_lib]
 
         if '+superlu-dist' in spec:
+            metis_lib = '-L%s -lparmetis -lmetis' % spec['parmetis'].prefix.lib
+            options += [
+                'METIS_DIR=%s' % spec['parmetis'].prefix,
+                'METIS_OPT=-I%s' % spec['parmetis'].prefix.include,
+                'METIS_LIB=%s' % metis_lib]
             superlu_lib = '-L%s' % spec['superlu-dist'].prefix.lib
             superlu_lib += ' -lsuperlu_dist'
-            sl_inc = 'SUPERLU_OPT=-I%s' % spec['superlu-dist'].prefix.include
-            options.extend(['MFEM_USE_SUPERLU=YES',
-                            'SUPERLU_DIR=%s' % spec['superlu-dist'].prefix,
-                            sl_inc,
-                            'SUPERLU_LIB=%s' % superlu_lib])
+            options += [
+                'SUPERLU_DIR=%s' % spec['superlu-dist'].prefix,
+                'SUPERLU_OPT=-I%s' % spec['superlu-dist'].prefix.include,
+                'SUPERLU_LIB=%s' % superlu_lib]
+        else:
+            metis_lib = '-L%s -lmetis' % spec['metis'].prefix.lib
+            options += [
+                'METIS_DIR=%s' % spec['metis'].prefix,
+                'METIS_OPT=-I%s' % spec['metis'].prefix.include,
+                'METIS_LIB=%s' % metis_lib]
 
         if '+suite-sparse' in spec:
             ssp = spec['suite-sparse'].prefix
             ss_lib = '-L%s' % ssp.lib
-
             if '@3.2:' in spec:
                 ss_lib += ' -lklu -lbtf'
-
             ss_lib += (' -lumfpack -lcholmod -lcolamd' +
                        ' -lamd -lcamd -lccolamd -lsuitesparseconfig')
-
             no_rt = spec.satisfies('platform=darwin')
             if not no_rt:
                 ss_lib += ' -lrt'
             ss_lib += (' ' + metis_lib + ' ' + lapack_lib)
-
-            options.extend(['MFEM_USE_SUITESPARSE=YES',
-                            'SUITESPARSE_DIR=%s' % ssp,
-                            'SUITESPARSE_OPT=-I%s' % ssp.include,
-                            'SUITESPARSE_LIB=%s' % ss_lib])
+            options += [
+                'SUITESPARSE_DIR=%s' % ssp,
+                'SUITESPARSE_OPT=-I%s' % ssp.include,
+                'SUITESPARSE_LIB=%s' % ss_lib]
+
+        if '+sundials' in spec:
+            sundials_libs = (
+                '-lsundials_arkode -lsundials_cvode'
+                ' -lsundials_nvecserial -lsundials_kinsol')
+            if '+mpi' in spec:
+                sundials_libs += (
+                    ' -lsundials_nvecparhyp -lsundials_nvecparallel')
+            options += [
+                'SUNDIALS_DIR=%s' % spec['sundials'].prefix,
+                'SUNDIALS_OPT=-I%s' % spec['sundials'].prefix.include,
+                'SUNDIALS_LIB=-L%s %s' % (spec['sundials'].prefix.lib,
+                                          sundials_libs)]
+
+        if '+petsc' in spec:
+            f = open('%s/lib/pkgconfig/PETSc.pc' % spec['petsc'].prefix, 'r')
+            for line in f:
+                if re.search('^\s*Cflags: ', line):
+                    petsc_opts = re.sub('^\s*Cflags: (.*)', '\\1', line)
+                elif re.search('^\s*Libs.*: ', line):
+                    petsc_libs = re.sub('^\s*Libs.*: (.*)', '\\1', line)
+            f.close()
+            options += [
+                'PETSC_DIR=%s' % spec['petsc'].prefix,
+                'PETSC_OPT=%s' % petsc_opts,
+                'PETSC_LIB=-L%s -lpetsc %s' %
+                (spec['petsc'].prefix.lib, petsc_libs)]
 
         if '+netcdf' in spec:
             np = spec['netcdf'].prefix
@@ -196,28 +259,31 @@ class Mfem(Package):
             nlib = '-L%s -lnetcdf ' % np.lib
             nlib += '-L%s -lhdf5_hl -lhdf5 ' % h5p.lib
             nlib += '-L%s -lz' % zp.lib
-            options.extend(['MFEM_USE_NETCDF=YES',
-                            'NETCDF_DIR=%s' % np,
-                            'HDF5_DIR=%s' % h5p,
-                            'ZLIB_DIR=%s' % zp,
-                            'NETCDF_OPT=-I%s' % np.include,
-                            'NETCDF_LIB=%s' % nlib])
+            options += [
+                'NETCDF_DIR=%s' % np,
+                'HDF5_DIR=%s' % h5p,
+                'ZLIB_DIR=%s' % zp,
+                'NETCDF_OPT=-I%s' % np.include,
+                'NETCDF_LIB=%s' % nlib]
 
-        if '+debug' in spec:
-            options.extend(['MFEM_DEBUG=YES'])
+        if '+mpfr' in spec:
+            options += ['MPFR_LIB=-L%s -lmpfr' % spec['mpfr'].prefix.lib]
+
+        if '+openmp' in spec:
+            options += ['OPENMP_OPT = %s' % self.compiler.openmp_flag]
 
         make('config', *options)
-        make('all')
-
-        # Run a small test before installation
-        args = ['-m', join_path('data', 'star.mesh'), '--no-visualization']
-        if '+mpi' in spec:
-            Executable(join_path(spec['mpi'].prefix.bin,
-                                 'mpirun'))('-np',
-                                            '4',
-                                            join_path('examples', 'ex1p'),
-                                            *args)
-        else:
-            Executable(join_path('examples', 'ex1'))(*args)
+        make('lib')
+
+        if self.run_tests:
+            make('check')
 
         make('install')
+
+        if '+examples' in spec:
+            make('examples')
+            install_tree('examples', join_path(prefix, 'examples'))
+
+        if '+miniapps' in spec:
+            make('miniapps')
+            install_tree('miniapps', join_path(prefix, 'miniapps'))
diff --git a/var/spack/repos/builtin/packages/sundials/package.py b/var/spack/repos/builtin/packages/sundials/package.py
index 18c3d409f8..66f385f29f 100644
--- a/var/spack/repos/builtin/packages/sundials/package.py
+++ b/var/spack/repos/builtin/packages/sundials/package.py
@@ -24,6 +24,7 @@
 ##############################################################################
 from spack import *
 import os
+import sys
 
 
 class Sundials(Package):
@@ -33,21 +34,28 @@ class Sundials(Package):
     homepage = "http://computation.llnl.gov/casc/sundials/"
     url = "http://computation.llnl.gov/projects/sundials-suite-nonlinear-differential-algebraic-equation-solvers/download/sundials-2.6.2.tar.gz"
 
+    version('2.7.0', 'c304631b9bc82877d7b0e9f4d4fd94d3')
     version('2.6.2', '3deeb0ede9f514184c6bd83ecab77d95')
 
-    variant('mpi',     default=True,  description='Enable MPI support')
+    variant('mpi',     default=True,
+            description='Enable MPI parallelism')
     variant('lapack',  default=True,
-            description='Build with external BLAS/LAPACK libraries')
+            description='Use external BLAS/LAPACK libraries')
     variant('klu',     default=False,
-            description='Build with SuiteSparse KLU libraries')
+            description='Enable KLU sparse, direct solver')
     variant('superlu', default=False,
-            description='Build with SuperLU_MT libraries')
-    variant('openmp',  default=False, description='Enable OpenMP support')
+            description='Enable SuperLU_MT sparse, direct solver')
+    variant('openmp',  default=False,
+            description='Enable OpenMP parallelism')
     variant('pthread', default=True,
-            description='Enable POSIX threads support')
+            description='Enable Pthreads parallelism')
+    variant('hypre', default=False,
+            description='Enable Hypre parallel vector for MPI parallelism')
 
-    depends_on('cmake', type='build')
+    depends_on('cmake',              type='build')
     depends_on('mpi',                when='+mpi')
+    depends_on('mpi',                when='@2.7:+hypre')
+    depends_on('hypre',              when='@2.7:+hypre')
     depends_on('blas',               when='+lapack')
     depends_on('lapack',             when='+lapack')
     depends_on('suite-sparse',       when='+klu')
@@ -55,6 +63,10 @@ class Sundials(Package):
     depends_on('superlu-mt+pthread', when='+superlu+pthread')
 
     def install(self, spec, prefix):
+
+        def on_off(varstr):
+            return 'ON' if varstr in self.spec else 'OFF'
+
         cmake_args = std_cmake_args[:]
         cmake_args.extend([
             '-DBUILD_SHARED_LIBS=ON',
@@ -62,47 +74,53 @@ class Sundials(Package):
             '-DCMAKE_Fortran_FLAGS={0}'.format(self.compiler.pic_flag),
             '-DEXAMPLES_ENABLE=ON',
             '-DEXAMPLES_INSTALL=ON',
-            '-DFCMIX_ENABLE=ON'
+            '-DFCMIX_ENABLE=ON',
+            '-DMPI_ENABLE=%s' % on_off('+mpi'),
+            '-DLAPACK_ENABLE=%s' % on_off('+lapack'),
+            '-DKLU_ENABLE=%s' % on_off('+klu'),
+            '-DHYPRE_ENABLE=%s' % on_off('+hypre'),
+            '-DSUPERLUMT_ENABLE=%s' % on_off('+superlu'),
+            '-DOPENMP_ENABLE=%s' % on_off('+openmp'),
+            '-DPTHREAD_ENABLE=%s' % on_off('+pthread')
         ])
 
         # MPI support
         if '+mpi' in spec:
             cmake_args.extend([
-                '-DMPI_ENABLE=ON',
                 '-DMPI_MPICC={0}'.format(spec['mpi'].mpicc),
                 '-DMPI_MPIF77={0}'.format(spec['mpi'].mpif77)
             ])
-        else:
-            cmake_args.append('-DMPI_ENABLE=OFF')
+
+        # Building with Hypre
+        if '+hypre' in spec and spec.satisfies('@2.7:'):
+            cmake_args.extend([
+                '-DHYPRE_INCLUDE_DIR={0}'.format(
+                    spec['hypre'].prefix.include),
+                '-DHYPRE_LIBRARY_DIR={0}'.format(
+                    spec['hypre'].prefix.lib)
+            ])
 
         # Building with LAPACK and BLAS
         if '+lapack' in spec:
             cmake_args.extend([
-                '-DLAPACK_ENABLE=ON',
                 '-DLAPACK_LIBRARIES={0}'.format(
                     (spec['lapack'].libs +
                      spec['blas'].libs).joined(';')
                 )
             ])
-        else:
-            cmake_args.append('-DLAPACK_ENABLE=OFF')
 
         # Building with KLU
         if '+klu' in spec:
             cmake_args.extend([
-                '-DKLU_ENABLE=ON',
                 '-DKLU_INCLUDE_DIR={0}'.format(
                     spec['suite-sparse'].prefix.include),
                 '-DKLU_LIBRARY_DIR={0}'.format(
                     spec['suite-sparse'].prefix.lib)
             ])
-        else:
-            cmake_args.append('-DKLU_ENABLE=OFF')
 
         # Building with SuperLU_MT
         if '+superlu' in spec:
             cmake_args.extend([
-                '-DSUPERLUMT_ENABLE=ON',
                 '-DSUPERLUMT_INCLUDE_DIR={0}'.format(
                     spec['superlu-mt'].prefix.include),
                 '-DSUPERLUMT_LIBRARY_DIR={0}'.format(
@@ -116,20 +134,6 @@ class Sundials(Package):
                 msg = 'You must choose either +openmp or +pthread when '
                 msg += 'building with SuperLU_MT'
                 raise RuntimeError(msg)
-        else:
-            cmake_args.append('-DSUPERLUMT_ENABLE=OFF')
-
-        # OpenMP support
-        if '+openmp' in spec:
-            cmake_args.append('-DOPENMP_ENABLE=ON')
-        else:
-            cmake_args.append('-DOPENMP_ENABLE=OFF')
-
-        # POSIX threads support
-        if '+pthread' in spec:
-            cmake_args.append('-DPTHREAD_ENABLE=ON')
-        else:
-            cmake_args.append('-DPTHREAD_ENABLE=OFF')
 
         with working_dir('build', create=True):
             cmake('..', *cmake_args)
@@ -137,6 +141,9 @@ class Sundials(Package):
             make()
             make('install')
 
+            if (sys.platform == 'darwin'):
+                fix_darwin_install_name(prefix.lib)
+
         install('LICENSE', prefix)
 
         self.filter_compilers()