Add CARE package, fixes for ROCmPackage and subclasses (#20070)

* use develop version of blt with fixes for rocm

* package updates for care+rocm

* fixes for plain cpu build

* add camp dependency on raja

5 files changed, 178 insertions, 16 deletions

diff --git a/var/spack/repos/builtin/packages/camp/package.py b/var/spack/repos/builtin/packages/camp/package.py
index 882bfec528..d2d213664a 100644
--- a/var/spack/repos/builtin/packages/camp/package.py
+++ b/var/spack/repos/builtin/packages/camp/package.py
@@ -19,13 +19,17 @@ class Camp(CMakePackage, CudaPackage, ROCmPackage):
     version('master', branch='master', submodules='True')
     version('0.1.0', sha256='fd4f0f2a60b82a12a1d9f943f8893dc6fe770db493f8fae5ef6f7d0c439bebcc')
 
-    depends_on('cmake@3.8:', type='build')
-    depends_on('cmake@3.9:', type='build', when="+cuda")
+    # TODO: figure out gtest dependency and then set this default True.
+    variant('tests', default=False, description='Build tests')
+
+    depends_on('blt', type='build')
+    depends_on('blt@0.3.7:', type='build', when='+rocm')
 
     def cmake_args(self):
         spec = self.spec
 
         options = []
+        options.append('-DBLT_SOURCE_DIR={0}'.format(spec['blt'].prefix))
 
         if '+cuda' in spec:
             options.extend([
@@ -55,6 +59,6 @@ class Camp(CMakePackage, CudaPackage, ROCmPackage):
             options.append('-DENABLE_HIP=OFF')
 
         options.append('-DENABLE_TESTS={0}'.format(
-            "On" if self.run_tests else "Off"))
+            'ON' if '+tests' in spec else 'OFF'))
 
         return options
diff --git a/var/spack/repos/builtin/packages/care/package.py b/var/spack/repos/builtin/packages/care/package.py
new file mode 100644
index 0000000000..7c3b6a6391
--- /dev/null
+++ b/var/spack/repos/builtin/packages/care/package.py
@@ -0,0 +1,136 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Care(CMakePackage, CudaPackage, ROCmPackage):
+    """
+    Algorithms for chai managed arrays.
+    """
+
+    homepage = "https://github.com/LLNL/CARE"
+    git      = "https://github.com/LLNL/CARE.git"
+
+    version('develop', branch='develop', submodules='True')
+    version('master', branch='main', submodules='True')
+    version('0.3.0', tag='v0.3.0', submodules='True')
+    version('0.2.0', tag='v0.2.0', submodules='True')
+
+    variant('openmp', default=False, description='Build Shared Libs')
+    variant('implicit_conversions', default=True, description='Enable implicit'
+            'conversions to/from raw pointers')
+    variant('benchmarks', default=True, description='Build benchmarks.')
+    variant('examples', default=True, description='Build examples.')
+    variant('docs', default=False, description='Build documentation')
+    # TODO: figure out gtest dependency and then set this default True
+    # and remove the +tests conflict below.
+    variant('tests', default=False, description='Build tests')
+
+    depends_on('blt', type='build')
+    depends_on('blt@0.3.7:', type='build', when='+rocm')
+
+    depends_on('camp')
+    depends_on('umpire@develop')
+    depends_on('raja@develop')
+    depends_on('chai@develop+enable_pick~benchmarks')
+
+    # WARNING: this package currently only supports an internal cub
+    # package. This will cause a race condition if compiled with another
+    # package that uses cub. TODO: have all packages point to the same external
+    # cub package.
+    depends_on('camp+cuda', when='+cuda')
+    depends_on('umpire+cuda', when='+cuda')
+    depends_on('raja+cuda~openmp', when='+cuda')
+    depends_on('chai+cuda', when='+cuda')
+
+    # variants +rocm and amdgpu_targets are not automatically passed to
+    # dependencies, so do it manually.
+    depends_on('camp+rocm', when='+rocm')
+    depends_on('umpire+rocm', when='+rocm')
+    depends_on('raja+rocm~openmp', when='+rocm')
+    depends_on('chai+rocm', when='+rocm')
+    for val in ROCmPackage.amdgpu_targets:
+        depends_on('camp amdgpu_target=%s' % val, when='amdgpu_target=%s' % val)
+        depends_on('umpire amdgpu_target=%s' % val, when='amdgpu_target=%s' % val)
+        depends_on('raja amdgpu_target=%s' % val, when='amdgpu_target=%s' % val)
+        depends_on('chai amdgpu_target=%s' % val, when='amdgpu_target=%s' % val)
+
+    conflicts('+openmp', when='+rocm')
+    conflicts('+openmp', when='+cuda')
+
+    def cmake_args(self):
+        spec = self.spec
+
+        options = []
+        options.append('-DBLT_SOURCE_DIR={0}'.format(spec['blt'].prefix))
+
+        if '+cuda' in spec:
+            options.extend([
+                '-DENABLE_CUDA=ON',
+                '-DCUDA_TOOLKIT_ROOT_DIR=' + spec['cuda'].prefix])
+
+            if not spec.satisfies('cuda_arch=none'):
+                cuda_arch = spec.variants['cuda_arch'].value
+                options.append('-DCUDA_ARCH=sm_{0}'.format(cuda_arch[0]))
+                flag = '-arch sm_{0}'.format(cuda_arch[0])
+                options.append('-DCMAKE_CUDA_FLAGS:STRING={0}'.format(flag))
+        else:
+            options.append('-DENABLE_CUDA=OFF')
+
+        if '+rocm' in spec:
+            options.extend([
+                '-DENABLE_HIP=ON',
+                '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix)])
+
+            archs = self.spec.variants['amdgpu_target'].value
+            if archs != 'none':
+                arch_str = ",".join(archs)
+                options.append(
+                    '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch_str)
+                )
+        else:
+            options.append('-DENABLE_HIP=OFF')
+
+        options.append('-DCARE_ENABLE_IMPLICIT_CONVERSIONS={0}'.format(
+            'ON' if '+implicit_conversions' in spec else 'OFF'))
+
+        options.append('-DCAMP_DIR:PATH='
+                       + spec['camp'].prefix.share.camp.cmake)
+        options.append('-DUMPIRE_DIR:PATH='
+                       + spec['umpire'].prefix.share.umpire.cmake)
+        options.append('-DRAJA_DIR:PATH='
+                       + spec['raja'].prefix.share.raja.cmake)
+        options.append('-DCHAI_DIR:PATH='
+                       + spec['chai'].prefix.share.chai.cmake)
+
+        options.append('-DCARE_ENABLE_TESTS={0}'.format(
+            'ON' if '+tests' in spec else 'OFF'))
+        # For tests to work, we also need BLT_ENABLE_TESTS to be on.
+        # This will take care of the gtest dependency. CARE developers should
+        # consider consolidating these flags in the future.
+        options.append('-DBLT_ENABLE_TESTS={0}'.format(
+            'ON' if '+tests' in spec else 'OFF'))
+
+        # There are both CARE_ENABLE_* and ENABLE_* variables in here because
+        # one controls the BLT infrastructure and the other controls the CARE
+        # infrastructure. The goal is to just be able to use the CARE_ENABLE_*
+        # variables, but CARE isn't set up correctly for that yet.
+        options.append('-DENABLE_BENCHMARKS={0}'.format(
+            'ON' if '+benchmarks' in spec else 'OFF'))
+        options.append('-DCARE_ENABLE_BENCHMARKS={0}'.format(
+            'ON' if '+benchmarks' in spec else 'OFF'))
+
+        options.append('-DENABLE_EXAMPLES={0}'.format(
+            'ON' if '+examples' in spec else 'OFF'))
+        options.append('-DCARE_ENABLE_EXAMPLES={0}'.format(
+            'ON' if '+examples' in spec else 'OFF'))
+
+        options.append('-DENABLE_DOCS={0}'.format(
+            'ON' if '+docs' in spec else 'OFF'))
+        options.append('-DCARE_ENABLE_DOCS={0}'.format(
+            'ON' if '+docs' in spec else 'OFF'))
+
+        return options
diff --git a/var/spack/repos/builtin/packages/chai/package.py b/var/spack/repos/builtin/packages/chai/package.py
index 6fd33dea8b..2c30bb9bb6 100644
--- a/var/spack/repos/builtin/packages/chai/package.py
+++ b/var/spack/repos/builtin/packages/chai/package.py
@@ -23,16 +23,22 @@ class Chai(CMakePackage, CudaPackage, ROCmPackage):
     version('1.1.0', tag='v1.1.0', submodules='True')
     version('1.0', tag='v1.0', submodules='True')
 
+    variant('enable_pick', default=False, description='Enable pick method')
     variant('shared', default=True, description='Build Shared Libs')
     variant('raja', default=False, description='Build plugin for RAJA')
     variant('benchmarks', default=True, description='Build benchmarks.')
     variant('examples', default=True, description='Build examples.')
+    # TODO: figure out gtest dependency and then set this default True
+    # and remove the +tests conflict below.
+    variant('tests', default=False, description='Build tests')
 
     depends_on('cmake@3.8:', type='build')
+    depends_on('cmake@3.9:', type='build', when="+cuda")
+    depends_on('blt', type='build')
+    depends_on('blt@0.3.7:', type='build', when='+rocm')
     depends_on('umpire')
     depends_on('raja', when="+raja")
 
-    depends_on('cmake@3.9:', type='build', when="+cuda")
     depends_on('umpire+cuda', when="+cuda")
     depends_on('raja+cuda', when="+raja+cuda")
 
@@ -44,10 +50,13 @@ class Chai(CMakePackage, CudaPackage, ROCmPackage):
         depends_on('umpire amdgpu_target=%s' % val, when='amdgpu_target=%s' % val)
         depends_on('raja amdgpu_target=%s' % val, when='+raja amdgpu_target=%s' % val)
 
+    conflicts('+benchmarks', when='~tests')
+
     def cmake_args(self):
         spec = self.spec
 
         options = []
+        options.append('-DBLT_SOURCE_DIR={0}'.format(spec['blt'].prefix))
 
         if '+cuda' in spec:
             options.extend([
@@ -80,18 +89,14 @@ class Chai(CMakePackage, CudaPackage, ROCmPackage):
             options.extend(['-DENABLE_RAJA_PLUGIN=ON',
                             '-DRAJA_DIR=' + spec['raja'].prefix])
 
+        options.append('-DENABLE_PICK={0}'.format(
+            'ON' if '+enable_pick' in spec else 'OFF'))
+
         options.append('-Dumpire_DIR:PATH='
                        + spec['umpire'].prefix.share.umpire.cmake)
 
         options.append('-DENABLE_TESTS={0}'.format(
-            'ON' if self.run_tests else 'OFF'))
-
-        # give clear error for conflict between self.run_tests and
-        # benchmarks variant.
-        if not self.run_tests and '+benchmarks' in spec:
-            raise InstallError(
-                'ENABLE_BENCHMARKS requires ENABLE_TESTS to be ON'
-            )
+            'ON' if '+tests' in spec  else 'OFF'))
 
         options.append('-DENABLE_BENCHMARKS={0}'.format(
             'ON' if '+benchmarks' in spec else 'OFF'))
diff --git a/var/spack/repos/builtin/packages/raja/package.py b/var/spack/repos/builtin/packages/raja/package.py
index c2cdf5a607..95af9c750b 100644
--- a/var/spack/repos/builtin/packages/raja/package.py
+++ b/var/spack/repos/builtin/packages/raja/package.py
@@ -32,16 +32,31 @@ class Raja(CMakePackage, CudaPackage, ROCmPackage):
     variant('shared', default=True, description='Build Shared Libs')
     variant('examples', default=True, description='Build examples.')
     variant('exercises', default=True, description='Build exercises.')
+    # TODO: figure out gtest dependency and then set this default True
+    # and remove the +tests conflict below.
+    variant('tests', default=False, description='Build tests')
 
-    conflicts('+openmp', when='+rocm')
+    depends_on('blt', type='build')
+    depends_on('blt@0.3.7:', type='build', when='+rocm')
+
+    depends_on('camp')
+    depends_on('camp+cuda', when='+cuda')
+
+    # variants +rocm and amdgpu_targets are not automatically passed to
+    # dependencies, so do it manually.
+    depends_on('camp+rocm', when='+rocm')
+    for val in ROCmPackage.amdgpu_targets:
+        depends_on('camp amdgpu_target=%s' % val, when='amdgpu_target=%s' % val)
 
-    depends_on('cmake@3.8:', type='build')
-    depends_on('cmake@3.9:', when='+cuda', type='build')
+    conflicts('+openmp', when='+rocm')
 
     def cmake_args(self):
         spec = self.spec
 
         options = []
+
+        options.append('-DBLT_SOURCE_DIR={0}'.format(spec['blt'].prefix))
+
         options.append('-DENABLE_OPENMP={0}'.format(
             'ON' if '+openmp' in spec else 'OFF'))
 
@@ -84,7 +99,8 @@ class Raja(CMakePackage, CudaPackage, ROCmPackage):
         if self.spec.satisfies('%clang target=ppc64le:') or not self.run_tests:
             options.append('-DENABLE_TESTS=OFF')
         else:
-            options.append('-DENABLE_TESTS=ON')
+            options.append('-DENABLE_TESTS={0}'.format(
+                'ON' if '+tests' in spec else 'OFF'))
 
         return options
 
diff --git a/var/spack/repos/builtin/packages/umpire/package.py b/var/spack/repos/builtin/packages/umpire/package.py
index 8b035e59df..ffb4cdbae1 100644
--- a/var/spack/repos/builtin/packages/umpire/package.py
+++ b/var/spack/repos/builtin/packages/umpire/package.py
@@ -61,6 +61,7 @@ class Umpire(CMakePackage, CudaPackage, ROCmPackage):
     depends_on('cmake@3.9:', when='+cuda', type='build')
 
     depends_on('blt', type='build')
+    depends_on('blt@0.3.7:', type='build', when='+rocm')
 
     # variants +rocm and amdgpu_targets are not automatically passed to
     # dependencies, so do it manually.