Adds a package for LvArray (#19221)

* RAJA and CHAI changes.

* Added LvArray package.

3 files changed, 347 insertions, 8 deletions

diff --git a/var/spack/repos/builtin/packages/chai/package.py b/var/spack/repos/builtin/packages/chai/package.py
index 66296f5ddf..c7bdebaac7 100644
--- a/var/spack/repos/builtin/packages/chai/package.py
+++ b/var/spack/repos/builtin/packages/chai/package.py
@@ -25,6 +25,8 @@ class Chai(CMakePackage, CudaPackage):
 
     variant('shared', default=True, description='Build Shared Libs')
     variant('raja', default=False, description='Build plugin for RAJA')
+    variant('benchmarks', default=True, description='Build benchmarks.')
+    variant('examples', default=True, description='Build examples.')
 
     depends_on('cmake@3.8:', type='build')
     depends_on('umpire')
@@ -42,7 +44,7 @@ class Chai(CMakePackage, CudaPackage):
         if '+cuda' in spec:
             options.extend([
                 '-DENABLE_CUDA=ON',
-                '-DCUDA_TOOLKIT_ROOT_DIR=%s' % (spec['cuda'].prefix)])
+                '-DCUDA_TOOLKIT_ROOT_DIR=' + spec['cuda'].prefix])
 
             if not spec.satisfies('cuda_arch=none'):
                 cuda_arch = spec.variants['cuda_arch'].value
@@ -52,10 +54,20 @@ class Chai(CMakePackage, CudaPackage):
         else:
             options.append('-DENABLE_CUDA=OFF')
 
+        if '+raja' in spec:
+            options.extend(['-DENABLE_RAJA_PLUGIN=ON',
+                            '-DRAJA_DIR=' + spec['raja'].prefix])
+
         options.append('-Dumpire_DIR:PATH='
                        + spec['umpire'].prefix.share.umpire.cmake)
 
         options.append('-DENABLE_TESTS={0}'.format(
             'ON' if self.run_tests else 'OFF'))
 
+        options.append('-DENABLE_BENCHMARKS={0}'.format(
+            'ON' if '+benchmarks' in spec else 'OFF'))
+
+        options.append('-DENABLE_EXAMPLES={0}'.format(
+            'ON' if '+examples' in spec else 'OFF'))
+
         return options
diff --git a/var/spack/repos/builtin/packages/lvarray/package.py b/var/spack/repos/builtin/packages/lvarray/package.py
new file mode 100644
index 0000000000..366b562cc6
--- /dev/null
+++ b/var/spack/repos/builtin/packages/lvarray/package.py
@@ -0,0 +1,326 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+import warnings
+
+import socket
+import os
+
+from os import environ as env
+from os.path import join as pjoin
+
+
+def cmake_cache_entry(name, value, comment=""):
+    """Generate a string for a cmake cache variable"""
+
+    return 'set(%s "%s" CACHE PATH "%s")\n\n' % (name, value, comment)
+
+
+def cmake_cache_string(name, string, comment=""):
+    """Generate a string for a cmake cache variable"""
+
+    return 'set(%s "%s" CACHE STRING "%s")\n\n' % (name, string, comment)
+
+
+def cmake_cache_option(name, boolean_value, comment=""):
+    """Generate a string for a cmake configuration option"""
+
+    value = "ON" if boolean_value else "OFF"
+    return 'set(%s %s CACHE BOOL "%s")\n\n' % (name, value, comment)
+
+
+class Lvarray(CMakePackage, CudaPackage):
+    """LvArray portable HPC containers."""
+
+    homepage = "https://github.com/GEOSX/lvarray"
+    git      = "https://github.com/GEOSX/LvArray.git"
+
+    version('develop', branch='develop', submodules='True')
+    version('main', branch='main', submodules='True')
+    version('0.1.0', tag='v0.1.0', submodules='True')
+
+    variant('shared', default=True, description='Build Shared Libs')
+    variant('umpire', default=False, description='Build Umpire support')
+    variant('chai', default=False, description='Build Chai support')
+    variant('caliper', default=False, description='Build Caliper support')
+    variant('tests', default=True, description='Build tests')
+    variant('benchmarks', default=False, description='Build benchmarks')
+    variant('examples', default=False, description='Build examples')
+    variant('docs', default=False, description='Build docs')
+    variant('addr2line', default=True,
+            description='Build support for addr2line.')
+
+    depends_on('cmake@3.8:', type='build')
+    depends_on('cmake@3.9:', when='+cuda', type='build')
+
+    depends_on('raja')
+    depends_on('raja+cuda', when='+cuda')
+
+    depends_on('umpire', when='+umpire')
+    depends_on('umpire+cuda', when='+umpire+cuda')
+
+    depends_on('chai+raja', when='+chai')
+    depends_on('chai+raja+cuda', when='+chai+cuda')
+
+    depends_on('caliper', when='+caliper')
+
+    depends_on('doxygen@1.8.13:', when='+docs', type='build')
+    depends_on('py-sphinx@1.6.3:', when='+docs', type='build')
+
+    phases = ['hostconfig', 'cmake', 'build', 'install']
+
+    @run_after('build')
+    @on_package_attributes(run_tests=True)
+    def check(self):
+        """Searches the CMake-generated Makefile for the target ``test``
+        and runs it if found.
+        """
+        with working_dir(self.build_directory):
+            ctest('-V', '--force-new-ctest-process', '-j 1')
+
+    @run_after('build')
+    def build_docs(self):
+        if '+docs' in self.spec:
+            with working_dir(self.build_directory):
+                make('docs')
+
+    def _get_sys_type(self, spec):
+        sys_type = str(spec.architecture)
+        # if on llnl systems, we can use the SYS_TYPE
+        if "SYS_TYPE" in env:
+            sys_type = env["SYS_TYPE"]
+        return sys_type
+
+    def _get_host_config_path(self, spec):
+        var = ''
+        if '+cuda' in spec:
+            var = '-'.join([var, 'cuda'])
+
+        hostname = socket.gethostname().rstrip('1234567890')
+        host_config_path = "%s-%s-%s%s.cmake" % (hostname,
+                                                 self._get_sys_type(spec),
+                                                 spec.compiler, var)
+
+        dest_dir = self.stage.source_path
+        host_config_path = os.path.abspath(pjoin(dest_dir, host_config_path))
+        return host_config_path
+
+    def hostconfig(self, spec, prefix, py_site_pkgs_dir=None):
+        """
+        This method creates a 'host-config' file that specifies
+        all of the options used to configure and build Umpire.
+
+        For more details about 'host-config' files see:
+            http://software.llnl.gov/conduit/building.html
+
+        Note:
+          The `py_site_pkgs_dir` arg exists to allow a package that
+          subclasses this package provide a specific site packages
+          dir when calling this function. `py_site_pkgs_dir` should
+          be an absolute path or `None`.
+
+          This is necessary because the spack `site_packages_dir`
+          var will not exist in the base class. For more details
+          on this issue see: https://github.com/spack/spack/issues/6261
+        """
+
+        #######################
+        # Compiler Info
+        #######################
+        c_compiler = env["SPACK_CC"]
+        cpp_compiler = env["SPACK_CXX"]
+
+        #######################################################################
+        # By directly fetching the names of the actual compilers we appear
+        # to doing something evil here, but this is necessary to create a
+        # 'host config' file that works outside of the spack install env.
+        #######################################################################
+
+        sys_type = self._get_sys_type(spec)
+
+        ##############################################
+        # Find and record what CMake is used
+        ##############################################
+
+        cmake_exe = spec['cmake'].command.path
+        cmake_exe = os.path.realpath(cmake_exe)
+
+        host_config_path = self._get_host_config_path(spec)
+        cfg = open(host_config_path, "w")
+        cfg.write("#{0}\n".format("#" * 80))
+        cfg.write("# Generated host-config - Edit at own risk!\n")
+        cfg.write("#{0}\n".format("#" * 80))
+
+        cfg.write("#{0}\n".format("-" * 80))
+        cfg.write("# SYS_TYPE: {0}\n".format(sys_type))
+        cfg.write("# Compiler Spec: {0}\n".format(spec.compiler))
+        cfg.write("# CMake executable path: %s\n" % cmake_exe)
+        cfg.write("#{0}\n\n".format("-" * 80))
+
+        #######################
+        # Compiler Settings
+        #######################
+
+        cfg.write("#{0}\n".format("-" * 80))
+        cfg.write("# Compilers\n")
+        cfg.write("#{0}\n\n".format("-" * 80))
+        cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler))
+        cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler))
+
+        # use global spack compiler flags
+        cflags = ' '.join(spec.compiler_flags['cflags'])
+        if cflags:
+            cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags))
+
+        cxxflags = ' '.join(spec.compiler_flags['cxxflags'])
+        if cxxflags:
+            cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags))
+
+        release_flags = "-O3 -DNDEBUG"
+        cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_RELEASE",
+                                     release_flags))
+        reldebinf_flags = "-O3 -g -DNDEBUG"
+        cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_RELWITHDEBINFO",
+                                     reldebinf_flags))
+        debug_flags = "-O0 -g"
+        cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags))
+
+        if "+cuda" in spec:
+            cfg.write("#{0}\n".format("-" * 80))
+            cfg.write("# Cuda\n")
+            cfg.write("#{0}\n\n".format("-" * 80))
+
+            cfg.write(cmake_cache_option("ENABLE_CUDA", True))
+            cfg.write(cmake_cache_entry("CMAKE_CUDA_STANDARD", 14))
+
+            cudatoolkitdir = spec['cuda'].prefix
+            cfg.write(cmake_cache_entry("CUDA_TOOLKIT_ROOT_DIR",
+                                        cudatoolkitdir))
+            cudacompiler = "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc"
+            cfg.write(cmake_cache_entry("CMAKE_CUDA_COMPILER", cudacompiler))
+
+            cmake_cuda_flags = ('-restrict --expt-extended-lambda -Werror '
+                                'cross-execution-space-call,reorder,'
+                                'deprecated-declarations')
+
+            archSpecifiers = ("-mtune", "-mcpu", "-march", "-qtune", "-qarch")
+            for archSpecifier in archSpecifiers:
+                for compilerArg in spec.compiler_flags['cxxflags']:
+                    if compilerArg.startswith(archSpecifier):
+                        cmake_cuda_flags += ' -Xcompiler ' + compilerArg
+
+            if not spec.satisfies('cuda_arch=none'):
+                cuda_arch = spec.variants['cuda_arch'].value
+                cmake_cuda_flags += ' -arch sm_{0}'.format(cuda_arch[0])
+
+            cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS", cmake_cuda_flags))
+
+            cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS_RELEASE",
+                                         "-O3 -Xcompiler -O3 -DNDEBUG"))
+            cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS_RELWITHDEBINFO",
+                                         "-O3 -g -lineinfo -Xcompiler -O3"))
+            cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS_DEBUG",
+                                         "-O0 -Xcompiler -O0 -g -G"))
+
+        else:
+            cfg.write(cmake_cache_option("ENABLE_CUDA", False))
+
+        cfg.write("#{0}\n".format("-" * 80))
+        cfg.write("# RAJA\n")
+        cfg.write("#{0}\n\n".format("-" * 80))
+
+        raja_dir = spec['raja'].prefix
+        cfg.write(cmake_cache_entry("RAJA_DIR", raja_dir))
+
+        cfg.write("#{0}\n".format("-" * 80))
+        cfg.write("# Umpire\n")
+        cfg.write("#{0}\n\n".format("-" * 80))
+
+        if "+umpire" in spec:
+            cfg.write(cmake_cache_option("ENABLE_UMPIRE", True))
+            umpire_dir = spec['umpire'].prefix
+            cfg.write(cmake_cache_entry("UMPIRE_DIR", umpire_dir))
+        else:
+            cfg.write(cmake_cache_option("ENABLE_UMPIRE", False))
+
+        cfg.write("#{0}\n".format("-" * 80))
+        cfg.write("# CHAI\n")
+        cfg.write("#{0}\n\n".format("-" * 80))
+
+        if "+chai" in spec:
+            cfg.write(cmake_cache_option("ENABLE_CHAI", True))
+            chai_dir = spec['chai'].prefix
+            cfg.write(cmake_cache_entry("CHAI_DIR", chai_dir))
+        else:
+            cfg.write(cmake_cache_option("ENABLE_CHAI", False))
+
+        cfg.write("#{0}\n".format("-" * 80))
+        cfg.write("# Caliper\n")
+        cfg.write("#{0}\n\n".format("-" * 80))
+
+        if "+caliper" in spec:
+            cfg.write("#{0}\n".format("-" * 80))
+            cfg.write("# Caliper\n")
+            cfg.write("#{0}\n\n".format("-" * 80))
+
+            cfg.write(cmake_cache_option("ENABLE_CALIPER", True))
+            caliper_dir = spec['caliper'].prefix
+            cfg.write(cmake_cache_entry("CALIPER_DIR", caliper_dir))
+        else:
+            cfg.write(cmake_cache_option("ENABLE_CALIPER", False))
+
+        cfg.write("#{0}\n".format("-" * 80))
+        cfg.write("# Documentation\n")
+        cfg.write("#{0}\n\n".format("-" * 80))
+        if "+docs" in spec:
+            sphinx_dir = spec['py-sphinx'].prefix
+            cfg.write(cmake_cache_string('SPHINX_EXECUTABLE',
+                                         os.path.join(sphinx_dir,
+                                                      'bin',
+                                                      'sphinx-build')))
+
+            doxygen_dir = spec['doxygen'].prefix
+            cfg.write(cmake_cache_string('DOXYGEN_EXECUTABLE',
+                                         os.path.join(doxygen_dir,
+                                                      'bin',
+                                                      'doxygen')))
+
+        cfg.write("#{0}\n".format("-" * 80))
+        cfg.write("# addr2line\n")
+        cfg.write("#{0}\n\n".format("-" * 80))
+        cfg.write(cmake_cache_option('ENABLE_ADDR2LINE', '+addr2line' in spec))
+
+        cfg.write("#{0}\n".format("-" * 80))
+        cfg.write("# Other\n")
+        cfg.write("#{0}\n\n".format("-" * 80))
+
+    def cmake_args(self):
+        spec = self.spec
+        host_config_path = self._get_host_config_path(spec)
+
+        options = []
+        options.extend(['-C', host_config_path])
+
+        # Shared libs
+        options.append(self.define_from_variant('BUILD_SHARED_LIBS', 'shared'))
+
+        if '~tests~examples~benchmarks' in spec:
+            options.append('-DENABLE_TESTS=OFF')
+        else:
+            options.append('-DENABLE_TESTS=ON')
+
+        if '~test' in spec:
+            options.append('-DDISABLE_UNIT_TESTS=ON')
+        elif "+tests" in spec and ('%intel' in spec or '%xl' in spec):
+            warnings.warn('The LvArray unit tests take an excessive amount of'
+                          ' time to build with the Intel or IBM compilers.')
+
+        options.append(self.define_from_variant('ENABLE_EXAMPLES', 'examples'))
+        options.append(self.define_from_variant('ENABLE_BENCHMARKS',
+                                                'benchmarks'))
+        options.append(self.define_from_variant('ENABLE_DOCS', 'docs'))
+
+        return options
diff --git a/var/spack/repos/builtin/packages/raja/package.py b/var/spack/repos/builtin/packages/raja/package.py
index f9ff3b4e00..7da9c6c4fd 100644
--- a/var/spack/repos/builtin/packages/raja/package.py
+++ b/var/spack/repos/builtin/packages/raja/package.py
@@ -32,6 +32,8 @@ class Raja(CMakePackage, CudaPackage):
 
     variant('openmp', default=True, description='Build OpenMP backend')
     variant('shared', default=True, description='Build Shared Libs')
+    variant('examples', default=True, description='Build examples.')
+    variant('exercises', default=True, description='Build exercises.')
 
     depends_on('cmake@3.8:', type='build')
     depends_on('cmake@3.9:', when='+cuda', type='build')
@@ -41,7 +43,7 @@ class Raja(CMakePackage, CudaPackage):
 
         options = []
         options.append('-DENABLE_OPENMP={0}'.format(
-            'ON' if '+openmp' in spec else 'Off'))
+            'ON' if '+openmp' in spec else 'OFF'))
 
         if '+cuda' in spec:
             options.extend([
@@ -57,8 +59,11 @@ class Raja(CMakePackage, CudaPackage):
         options.append('-DBUILD_SHARED_LIBS={0}'.format(
             'ON' if '+shared' in spec else 'OFF'))
 
-        options.append('-DENABLE_CHAI={0}'.format(
-            'ON' if '+chai' in spec else 'OFF'))
+        options.append('-DENABLE_EXAMPLES={0}'.format(
+            'ON' if '+examples' in spec else 'OFF'))
+
+        options.append('-DENABLE_EXERCISES={0}'.format(
+            'ON' if '+exercises' in spec else 'OFF'))
 
         # Work around spack adding -march=ppc64le to SPACK_TARGET_ARGS which
         # is used by the spack compiler wrapper.  This can go away when BLT
@@ -68,8 +73,4 @@ class Raja(CMakePackage, CudaPackage):
         else:
             options.append('-DENABLE_TESTS=ON')
 
-        if '+chai' in spec:
-            options.extend([
-                '-DENABLE_CHAI=ON'])
-
         return options