nalu-wind: add cuda variant (#22184)

1 files changed, 72 insertions, 80 deletions

diff --git a/var/spack/repos/builtin/packages/nalu-wind/package.py b/var/spack/repos/builtin/packages/nalu-wind/package.py
index 2d0347e603..93e155412d 100644
--- a/var/spack/repos/builtin/packages/nalu-wind/package.py
+++ b/var/spack/repos/builtin/packages/nalu-wind/package.py
@@ -7,13 +7,20 @@ from spack import *
 import sys
 
 
-class NaluWind(CMakePackage):
+def _parse_float(val):
+    try:
+        return float(val) > 0.0
+    except ValueError:
+        return False
+
+
+class NaluWind(CMakePackage, CudaPackage):
     """Nalu-Wind: Wind energy focused variant of Nalu."""
 
-    homepage = "https://github.com/exawind/nalu-wind"
+    homepage = "https://nalu-wind.readthedocs.io"
     git      = "https://github.com/exawind/nalu-wind.git"
 
-    maintainers = ['jrood-nrel']
+    maintainers = ['jrood-nrel', 'sayerhs']
 
     tags = ['ecp', 'ecp-apps']
 
@@ -24,8 +31,12 @@ class NaluWind(CMakePackage):
             description='Build dependencies as shared libraries')
     variant('pic', default=True,
             description='Position independent code')
-    variant('test_tol', default='default',
-            description='Tolerance for regression tests')
+    variant('abs_tol', default=1.0e-15,
+            values=_parse_float,
+            description='Absolute tolerance for regression tests')
+    variant('rel_tol', default=1.0e-12,
+            values=_parse_float,
+            description='Relative tolerance for regression tests')
     # Third party libraries
     variant('openfast', default=False,
             description='Compile with OpenFAST support')
@@ -37,6 +48,10 @@ class NaluWind(CMakePackage):
             description='Compile with Catalyst support')
     variant('fftw', default=False,
             description='Compile with FFTW support')
+    variant('openmp', default=False,
+            description='Compile with OpenMP support')
+    variant('boost', default=False,
+            description='Enable Boost integration')
 
     variant('wind-utils', default=False,
             description='Build wind-utils')
@@ -48,102 +63,79 @@ class NaluWind(CMakePackage):
     # Cannot build Trilinos as a shared library with STK on Darwin
     # which is why we have a 'shared' variant for Nalu-Wind
     # https://github.com/trilinos/Trilinos/issues/2994
-    depends_on('trilinos+exodus+tpetra+muelu+belos+ifpack2+amesos2+zoltan+stk+boost~superlu-dist+superlu+hdf5+zlib+pnetcdf+shards~hypre@master,develop', when='+shared')
-    depends_on('trilinos~shared+exodus+tpetra+muelu+belos+ifpack2+amesos2+zoltan+stk+boost~superlu-dist+superlu+hdf5+zlib+pnetcdf+shards~hypre@master,develop', when='~shared')
+    depends_on('trilinos@master,develop ~cuda~wrapper+shared+exodus+tpetra+muelu+belos+ifpack2+amesos2+zoltan+stk+boost~superlu-dist+superlu+hdf5+zlib+pnetcdf+shards~hypre', when='~cuda+shared')
+    depends_on('trilinos@master,develop ~cuda~wrapper~shared+exodus+tpetra+muelu+belos+ifpack2+amesos2+zoltan+stk+boost~superlu-dist+superlu+hdf5+zlib+pnetcdf+shards~hypre', when='~cuda~shared')
     # Optional dependencies
-    depends_on('openfast+cxx', when='+openfast+shared')
-    depends_on('openfast+cxx~shared', when='+openfast~shared')
-    depends_on('tioga', when='+tioga+shared')
-    depends_on('tioga~shared', when='+tioga~shared')
-    depends_on('hypre+mpi+int64~superlu-dist', when='+hypre+shared')
-    depends_on('hypre+mpi+int64~superlu-dist~shared', when='+hypre~shared')
+    depends_on('openfast@develop +cxx+shared', when='+openfast+shared')
+    depends_on('openfast@develop +cxx~shared', when='+openfast~shared')
+    depends_on('tioga@master +shared', when='+tioga+shared~cuda')
+    depends_on('tioga@master ~shared', when='+tioga~shared~cuda')
+    depends_on('tioga@develop ~shared', when='+tioga~shared+cuda')
+    depends_on('hypre@2.18.2: +mpi~cuda+int64~superlu-dist+shared', when='+hypre~cuda+shared')
+    depends_on('hypre@2.18.2: +mpi~cuda+int64~superlu-dist~shared', when='+hypre~cuda~shared')
+    depends_on('kokkos-nvcc-wrapper', type='build', when='+cuda')
+    for _arch in CudaPackage.cuda_arch_values:
+        depends_on('trilinos@master,develop ~shared+exodus+tpetra+muelu+belos+ifpack2+amesos2+zoltan+stk+boost~superlu-dist+superlu+hdf5+zlib+pnetcdf+shards~hypre+cuda+cuda_rdc+wrapper cuda_arch={0}'.format(_arch),
+                   when='+cuda cuda_arch={0}'.format(_arch))
+        depends_on('hypre@develop +mpi+cuda~int64~superlu-dist cuda_arch={0}'.format(_arch),
+                   when='+hypre+cuda cuda_arch={0}'.format(_arch))
     depends_on('trilinos-catalyst-ioss-adapter', when='+catalyst')
     # FFTW doesn't have a 'shared' variant at this moment
     depends_on('fftw+mpi', when='+fftw')
 
+    depends_on('boost cxxstd=14', when='+boost')
+
+    conflicts('+cuda', when='+shared')
+
     def cmake_args(self):
         spec = self.spec
-        options = []
-
-        options.extend([
-            '-DTrilinos_DIR:PATH=%s' % spec['trilinos'].prefix,
-            '-DYAML_DIR:PATH=%s' % spec['yaml-cpp'].prefix,
-            '-DCMAKE_C_COMPILER=%s' % spec['mpi'].mpicc,
-            '-DCMAKE_CXX_COMPILER=%s' % spec['mpi'].mpicxx,
-            '-DCMAKE_Fortran_COMPILER=%s' % spec['mpi'].mpifc,
-            '-DMPI_C_COMPILER=%s' % spec['mpi'].mpicc,
-            '-DMPI_CXX_COMPILER=%s' % spec['mpi'].mpicxx,
-            '-DMPI_Fortran_COMPILER=%s' % spec['mpi'].mpifc,
-            '-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=%s' % (
-                'ON' if '+pic' in spec else 'OFF'),
-        ])
 
+        args = [
+            self.define_from_variant('BUILD_SHARED_LIBS', 'shared'),
+            self.define_from_variant('CMAKE_POSITION_INDEPENDENT_CODE', 'pic'),
+            self.define('CMAKE_CXX_COMPILER', spec['mpi'].mpicxx),
+            self.define('CMAKE_C_COMPILER', spec['mpi'].mpicc),
+            self.define('CMAKE_Fortran_COMPILER', spec['mpi'].mpifc),
+            self.define('Trilinos_DIR', spec['trilinos'].prefix),
+            self.define('YAML_DIR', spec['yaml-cpp'].prefix),
+            self.define_from_variant('ENABLE_CUDA', 'cuda'),
+            self.define_from_variant('ENABLE_WIND_UTILS', 'wind-utils'),
+            self.define_from_variant('ENABLE_BOOST', 'boost'),
+            self.define_from_variant('ENABLE_OPENMP', 'openmp'),
+        ]
+
+        args.append(self.define_from_variant('ENABLE_OPENFAST', 'openfast'))
         if '+openfast' in spec:
-            options.extend([
-                '-DENABLE_OPENFAST:BOOL=ON',
-                '-DOpenFAST_DIR:PATH=%s' % spec['openfast'].prefix
-            ])
-        else:
-            options.append('-DENABLE_OPENFAST:BOOL=OFF')
+            args.append(self.define('OpenFAST_DIR', spec['openfast'].prefix))
 
+        args.append(self.define_from_variant('ENABLE_TIOGA', 'tioga'))
         if '+tioga' in spec:
-            options.extend([
-                '-DENABLE_TIOGA:BOOL=ON',
-                '-DTIOGA_DIR:PATH=%s' % spec['tioga'].prefix
-            ])
-        else:
-            options.append('-DENABLE_TIOGA:BOOL=OFF')
+            args.append(self.define('TIOGA_DIR', spec['tioga'].prefix))
 
+        args.append(self.define_from_variant('ENABLE_HYPRE', 'hypre'))
         if '+hypre' in spec:
-            options.extend([
-                '-DENABLE_HYPRE:BOOL=ON',
-                '-DHYPRE_DIR:PATH=%s' % spec['hypre'].prefix
-            ])
-        else:
-            options.append('-DENABLE_HYPRE:BOOL=OFF')
+            args.append(self.define('HYPRE_DIR', spec['hypre'].prefix))
 
+        args.append(self.define_from_variant('ENABLE_PARAVIEW_CATALYST', 'catalyst'))
         if '+catalyst' in spec:
-            options.extend([
-                '-DENABLE_PARAVIEW_CATALYST:BOOL=ON',
-                '-DPARAVIEW_CATALYST_INSTALL_PATH:PATH=%s' %
-                spec['trilinos-catalyst-ioss-adapter'].prefix
-            ])
-        else:
-            options.append('-DENABLE_PARAVIEW_CATALYST:BOOL=OFF')
+            args.append(self.define('PARAVIEW_CATALYST_INSTALL_PATH',
+                        spec['trilinos-catalyst-ioss-adapter'].prefix))
 
+        args.append(self.define_from_variant('ENABLE_FFTW', 'fftw'))
         if '+fftw' in spec:
-            options.extend([
-                '-DENABLE_FFTW:BOOL=ON',
-                '-DFFTW_DIR:PATH=%s' % spec['fftw'].prefix
-            ])
-        else:
-            options.append('-DENABLE_FFTW:BOOL=OFF')
+            args.append(self.define('FFTW_DIR', spec['fftw'].prefix))
 
-        if '+wind-utils' in spec:
-            options.append('-DENABLE_WIND_UTILS=ON')
-        else:
-            options.append('-DENABLE_WIND_UTILS=OFF')
+        args.append(self.define('ENABLE_TESTS', self.run_tests))
+        if self.run_tests:
+            args.extend([
+                self.define('TEST_TOLERANCE', spec.variants['abs_tol'].value),
+                self.define('TEST_REL_TOL', spec.variants['rel_tol'].value),
+            ])
 
         if 'darwin' in spec.architecture:
-            options.append('-DCMAKE_MACOSX_RPATH:BOOL=ON')
+            args.append(self.define('CMAKE_MACOSX_RPATH', 'ON'))
 
-        if self.run_tests:
-            options.append('-DENABLE_TESTS:BOOL=ON')
-        else:
-            options.append('-DENABLE_TESTS:BOOL=OFF')
-
-        if self.spec.variants['test_tol'].value != 'default':
-            try:
-                test_tol = float(self.spec.variants['test_tol'].value)
-                if test_tol <= 0.0:
-                    raise ValueError
-                options.append('-DTEST_TOLERANCE:STRING={tol}'.format(
-                    tol=test_tol))
-            except ValueError:
-                print("Specified test_tol must be a positive float. "
-                      "Using the default.")
-
-        return options
+        return args
 
     @run_before('cmake')
     def add_submodules(self):