Add rocBLAS (#18069)

* Add rocBLAS

* Add rocBLAS support to SIRIUS

3 files changed, 120 insertions, 3 deletions

diff --git a/var/spack/repos/builtin/packages/rocblas/0001-Fix-compilation-error-with-StringRef-to-basic-string.patch b/var/spack/repos/builtin/packages/rocblas/0001-Fix-compilation-error-with-StringRef-to-basic-string.patch
new file mode 100644
index 0000000000..54845c5a4a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rocblas/0001-Fix-compilation-error-with-StringRef-to-basic-string.patch
@@ -0,0 +1,25 @@
+From f4e7ceba4593540d6d6bf35a958d187cc84c5172 Mon Sep 17 00:00:00 2001
+From: Harmen Stoppels <harmenstoppels@gmail.com>
+Date: Mon, 10 Aug 2020 23:39:22 +0200
+Subject: [PATCH] Fix compilation error with StringRef to basic string
+
+---
+ Tensile/Tensile/Source/lib/include/Tensile/llvm/YAML.hpp | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/Tensile/Tensile/Source/lib/include/Tensile/llvm/YAML.hpp b/Tensile/Tensile/Source/lib/include/Tensile/llvm/YAML.hpp
+index 16e64ff9..0d00f470 100644
+--- a/Tensile/Tensile/Source/lib/include/Tensile/llvm/YAML.hpp
++++ b/Tensile/Tensile/Source/lib/include/Tensile/llvm/YAML.hpp
+@@ -261,7 +261,7 @@ namespace llvm
+ 
+             static void inputOne(IO& io, StringRef key, Hide<T>& value)
+             {
+-                Impl::inputOne(io, key, *value);
++                Impl::inputOne(io, key.str(), *value);
+             }
+ 
+             static void output(IO& io, Hide<T>& value)
+-- 
+2.25.1
+
diff --git a/var/spack/repos/builtin/packages/rocblas/package.py b/var/spack/repos/builtin/packages/rocblas/package.py
new file mode 100644
index 0000000000..572f96d7e6
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rocblas/package.py
@@ -0,0 +1,70 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack import *
+
+
+class Rocblas(CMakePackage):
+    """Radeon Open Compute BLAS library"""
+
+    homepage = "https://github.com/ROCmSoftwarePlatform/rocBLAS/"
+    url      = "https://github.com/ROCmSoftwarePlatform/rocBLAS/archive/rocm-3.5.0.tar.gz"
+
+    maintainers = ['haampie']
+
+    version('3.5.0', sha256='8560fabef7f13e8d67da997de2295399f6ec595edfd77e452978c140d5f936f0')
+
+    amdgpu_targets = ('all', 'gfx803', 'gfx900', 'gfx906', 'gfx908')
+
+    variant('amdgpu_target', default='all', multi=True, values=amdgpu_targets)
+
+    depends_on('cmake@3:', type='build')
+    depends_on('rocm-cmake@3.5.0', type='build', when='@3.5.0')
+    depends_on('rocm-device-libs@3.5.0', type='build', when='@3.5.0')
+
+    depends_on('hip@3.5.0', when='@3.5.0')
+    depends_on('comgr@3.5.0', type='build', when='@3.5.0')
+
+    depends_on('python', type='build')
+    depends_on('py-virtualenv', type='build')
+    depends_on('perl-file-which', type='build')
+    depends_on('py-pyyaml', type='build')
+    depends_on('py-wheel', type='build')
+
+    # Tensile uses LLVM
+    depends_on('llvm-amdgpu')
+
+    resource(name='Tensile',
+             git='https://github.com/ROCmSoftwarePlatform/Tensile.git',
+             commit='f842a1a4427624eff6cbddb2405c36dec9a210cd',
+             when='@3.5.0')
+
+    patch('0001-Fix-compilation-error-with-StringRef-to-basic-string.patch')
+
+    def setup_build_environment(self, env):
+        env.set('CXX', self.spec['hip'].hipcc)
+
+    def cmake_args(self):
+        archs = ",".join(self.spec.variants['amdgpu_target'].value)
+
+        tensile = join_path(self.stage.source_path, 'Tensile')
+
+        args = [
+            '-Damd_comgr_DIR={0}'.format(self.spec['comgr'].prefix),
+            '-DBUILD_CLIENTS_TESTS=OFF',
+            '-DBUILD_CLIENTS_BENCHMARKS=OFF',
+            '-DBUILD_CLIENTS_SAMPLES=OFF',
+            '-DRUN_HEADER_TESTING=OFF',
+            '-DBUILD_WITH_TENSILE=ON',
+            '-DBUILD_WITH_TENSILE_HOST=OFF',
+            '-DTensile_TEST_LOCAL_PATH={0}'.format(tensile),
+            '-DTensile_COMPILER=hipcc',
+            '-DTensile_ARCHITECTURE={0}'.format(archs),
+            '-DTensile_LOGIC=asm_full',
+            '-DTensile_CODE_OBJECT_VERSION=V3'
+        ]
+
+        return args
diff --git a/var/spack/repos/builtin/packages/sirius/package.py b/var/spack/repos/builtin/packages/sirius/package.py
index 9fe1481399..08ea866b56 100644
--- a/var/spack/repos/builtin/packages/sirius/package.py
+++ b/var/spack/repos/builtin/packages/sirius/package.py
@@ -38,6 +38,12 @@ class Sirius(CMakePackage, CudaPackage):
     version('6.3.2', sha256='1723e5ad338dad9a816369a6957101b2cae7214425406b12e8712c82447a7ee5')
     version('6.1.5', sha256='379f0a2e5208fd6d91c2bd4939c3a5c40002975fb97652946fa1bfe4a3ef97cb')
 
+    amdgpu_targets = (
+        'gfx701', 'gfx801', 'gfx802', 'gfx803',
+        'gfx900', 'gfx906', 'gfx908', 'gfx1010',
+        'gfx1011', 'gfx1012'
+    )
+
     variant('shared', default=False, description="Build shared libraries")
     variant('openmp', default=True, description="Build with OpenMP support")
     variant('fortran', default=False, description="Build Fortran bindings")
@@ -48,6 +54,8 @@ class Sirius(CMakePackage, CudaPackage):
     variant('scalapack', default=False, description="Enable scalapack support")
     variant('magma', default=False, description="Enable MAGMA support")
     variant('nlcglib', default=False, description="enable robust wave function optimization")
+    variant('rocm', default=False, description='Use ROCm GPU support')
+    variant('amdgpu_target', default=('gfx803', 'gfx900', 'gfx906'), multi=True, values=amdgpu_targets)
     variant('build_type', default='Release',
             description='CMake build type',
             values=('Debug', 'Release', 'RelWithDebInfo'))
@@ -82,13 +90,19 @@ class Sirius(CMakePackage, CudaPackage):
     depends_on('nlcglib', when='+nlcglib')
     depends_on('libvdwxc+mpi', when='+vdwxc')
     depends_on('scalapack', when='+scalapack')
-    depends_on('cuda', when='+cuda')
+
+    # rocm
+    depends_on('hip', when='+rocm')
+    depends_on('hsakmt-roct', when='+rocm', type='link')
+    depends_on('hsa-rocr-dev', when='+rocm', type='link')
+    depends_on('rocblas', when='+rocm')
+
     extends('python', when='+python')
 
     conflicts('+shared', when='@6.3.0:6.4.999')
 
     # TODO:
-    # add support for CRAY_LIBSCI, ROCm, testing
+    # add support for CRAY_LIBSCI, testing
 
     patch("strip-spglib-include-subfolder.patch", when='@6.1.5')
     patch("link-libraries-fortran.patch", when='@6.1.5')
@@ -143,7 +157,8 @@ class Sirius(CMakePackage, CudaPackage):
             _def('+scalapack'),
             _def('+fortran', 'CREATE_FORTRAN_BINDINGS'),
             _def('+python', 'CREATE_PYTHON_MODULE'),
-            _def('+cuda')
+            _def('+cuda'),
+            _def('+rocm')
         ]
 
         if '@:6.2.999' in self.spec:
@@ -185,4 +200,11 @@ class Sirius(CMakePackage, CudaPackage):
                     '-DCMAKE_CUDA_FLAGS=-arch=sm_{0}'.format(cuda_arch[0])
                 ]
 
+        if '+rocm' in spec:
+            archs = ",".join(self.spec.variants['amdgpu_target'].value)
+            args.extend([
+                '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix),
+                '-DHIP_HCC_FLAGS=--amdgpu-target={0}'.format(archs)
+            ])
+
         return args