From 3b9155239b83cd22074f8db057e9ad28f37f008e Mon Sep 17 00:00:00 2001 From: Danny Taller <66029857+dtaller@users.noreply.github.com> Date: Wed, 18 Nov 2020 11:52:21 -0800 Subject: hip support for umpire, chai, raja, camp (#19715) * create HipPackage base class and do some refactoring * comments and added conflict to raja for openmp with hip --- var/spack/repos/builtin/packages/camp/package.py | 25 +---------- var/spack/repos/builtin/packages/chai/package.py | 27 +++++++++++- var/spack/repos/builtin/packages/hip/package.py | 51 +++++++++++++++------- var/spack/repos/builtin/packages/raja/package.py | 13 +++++- var/spack/repos/builtin/packages/umpire/package.py | 20 ++++++++- 5 files changed, 93 insertions(+), 43 deletions(-) (limited to 'var') diff --git a/var/spack/repos/builtin/packages/camp/package.py b/var/spack/repos/builtin/packages/camp/package.py index 67285e9742..0d0f26280c 100644 --- a/var/spack/repos/builtin/packages/camp/package.py +++ b/var/spack/repos/builtin/packages/camp/package.py @@ -6,7 +6,7 @@ from spack import * -class Camp(CMakePackage, CudaPackage): +class Camp(CMakePackage, CudaPackage, HipPackage): """ Compiler agnostic metaprogramming library providing concepts, type operations and tuples for C++ and cuda @@ -22,27 +22,6 @@ class Camp(CMakePackage, CudaPackage): depends_on('cmake@3.8:', type='build') depends_on('cmake@3.9:', type='build', when="+cuda") - variant('hip', default=False, description='Enable HIP support') - - # possible amd gpu targets for hip builds - # TODO: we should add a hip build system description equivalent to - # lib/spack/spack/build_systems/cuda.py, where possible hip amd gpu - # architectures are defined in a similar way as for cuda gpu - # architectures. In the meantime, require users to define - # amd gpu type for hip builds with a variant here. - amdgpu_targets = ( - 'gfx701', 'gfx801', 'gfx802', 'gfx803', - 'gfx900', 'gfx906', 'gfx908', 'gfx1010', - 'gfx1011', 'gfx1012', 'none' - ) - variant('amdgpu_target', default='none', values=amdgpu_targets) - - depends_on('llvm-amdgpu', when='+hip') - depends_on('hip', when='+hip') - - # need amd gpu type for hip builds - conflicts('amdgpu_target=none', when='+hip') - def cmake_args(self): spec = self.spec @@ -66,7 +45,7 @@ class Camp(CMakePackage, CudaPackage): options.extend([ '-DENABLE_HIP=ON', '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix), - '-DHIP_HCC_FLAGS=--amdgpu-target={0}'.format(arch)]) + '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch)]) else: options.append('-DENABLE_HIP=OFF') diff --git a/var/spack/repos/builtin/packages/chai/package.py b/var/spack/repos/builtin/packages/chai/package.py index c7bdebaac7..d4fa53071d 100644 --- a/var/spack/repos/builtin/packages/chai/package.py +++ b/var/spack/repos/builtin/packages/chai/package.py @@ -6,7 +6,7 @@ from spack import * -class Chai(CMakePackage, CudaPackage): +class Chai(CMakePackage, CudaPackage, HipPackage): """ Copy-hiding array interface for data migration between memory spaces """ @@ -36,6 +36,15 @@ class Chai(CMakePackage, CudaPackage): depends_on('umpire+cuda', when="+cuda") depends_on('raja+cuda', when="+raja+cuda") + # variants +hip and amdgpu_targets are not automatically passed to + # dependencies, so do it manually. + amdgpu_targets = HipPackage.amd_gputargets_list() + depends_on('umpire+hip', when='+hip') + depends_on('raja+hip', when="+raja+hip") + for val in amdgpu_targets: + depends_on('umpire amdgpu_target=%s' % val, when='amdgpu_target=%s' % val) + depends_on('raja amdgpu_target=%s' % val, when='+raja amdgpu_target=%s' % val) + def cmake_args(self): spec = self.spec @@ -54,6 +63,15 @@ class Chai(CMakePackage, CudaPackage): else: options.append('-DENABLE_CUDA=OFF') + if '+hip' in spec: + arch = self.spec.variants['amdgpu_target'].value + options.extend([ + '-DENABLE_HIP=ON', + '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix), + '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch)]) + else: + options.append('-DENABLE_HIP=OFF') + if '+raja' in spec: options.extend(['-DENABLE_RAJA_PLUGIN=ON', '-DRAJA_DIR=' + spec['raja'].prefix]) @@ -64,6 +82,13 @@ class Chai(CMakePackage, CudaPackage): options.append('-DENABLE_TESTS={0}'.format( 'ON' if self.run_tests else 'OFF')) + # give clear error for conflict between self.run_tests and + # benchmarks variant. + if not self.run_tests and '+benchmarks' in spec: + raise InstallError( + 'ENABLE_BENCHMARKS requires ENABLE_TESTS to be ON' + ) + options.append('-DENABLE_BENCHMARKS={0}'.format( 'ON' if '+benchmarks' in spec else 'OFF')) diff --git a/var/spack/repos/builtin/packages/hip/package.py b/var/spack/repos/builtin/packages/hip/package.py index 72730c36a7..261b34e936 100644 --- a/var/spack/repos/builtin/packages/hip/package.py +++ b/var/spack/repos/builtin/packages/hip/package.py @@ -50,17 +50,25 @@ class Hip(CMakePackage): patch('0002-Fix-detection-of-HIP_CLANG_ROOT.patch', when='@3.5.0:') def setup_run_environment(self, env): - env.set('ROCM_PATH', '') + # NOTE: DO NOT PUT LOGIC LIKE self.spec[name] in this function!!!!! + # It DOES NOT WORK FOR EXTERNAL PACKAGES!!!! See get_rocm_prefix_info + rocm_prefixes = self.get_rocm_prefix_info() + + env.set('ROCM_PATH', rocm_prefixes['rocm-path']) env.set('HIP_COMPILER', 'clang') env.set('HIP_PLATFORM', 'hcc') - env.set('HIP_CLANG_PATH', self.spec['llvm-amdgpu'].prefix.bin) - env.set('HSA_PATH', self.spec['hsa-rocr-dev'].prefix) - env.set('ROCMINFO_PATH', self.spec['rocminfo'].prefix) - env.set('HIP_PATH', self.prefix) - env.set('DEVICE_LIB_PATH', - self.spec['rocm-device-libs'].prefix.amdgcn.bitcode) + env.set('HIP_CLANG_PATH', rocm_prefixes['llvm-amdgpu'].bin) + env.set('HSA_PATH', rocm_prefixes['hsa-rocr-dev']) + env.set('ROCMINFO_PATH', rocm_prefixes['rocminfo']) + env.set('DEVICE_LIB_PATH', rocm_prefixes['rocm-device-libs'].lib) + env.set('HIP_PATH', rocm_prefixes['rocm-path']) env.set('HIPCC_COMPILE_FLAGS_APPEND', - '--rocm-path={0}'.format(self.prefix)) + '--rocm-path={0}'.format(rocm_prefixes['rocm-path'])) + + if 'amdgpu_target' in self.spec.variants: + arch = self.spec.variants['amdgpu_target'].value + if arch != 'none': + env.set('HCC_AMDGPU_TARGET', arch) def setup_dependent_run_environment(self, env, dependent_spec): self.setup_run_environment(env) @@ -85,31 +93,42 @@ class Hip(CMakePackage): raise RuntimeError(msg) return { + 'rocm-path': fallback_prefix, 'llvm-amdgpu': fallback_prefix.llvm, 'hsa-rocr-dev': fallback_prefix.hsa, 'rocminfo': fallback_prefix.bin, 'rocm-device-libs': fallback_prefix, } else: - return dict((name, self.spec[name].prefix) - for name in ('llvm-amdgpu', 'hsa-rocr-dev', 'rocminfo', - 'rocm-device-libs')) + mydict = dict((name, self.spec[name].prefix) + for name in ('llvm-amdgpu', 'hsa-rocr-dev', + 'rocminfo', 'rocm-device-libs')) + mydict['rocm-path'] = os.path.dirname(self.spec.prefix) + return mydict def setup_dependent_build_environment(self, env, dependent_spec): # Indirection for dependency paths because hip may be an external in - # Spack. See block comment on get_rocm_prefix_info + # Spack. See block comment on get_rocm_prefix_info . + + # NOTE: DO NOT PUT LOGIC LIKE self.spec[name] in this function!!!!! + # It DOES NOT WORK FOR EXTERNAL PACKAGES!!!! See get_rocm_prefix_info rocm_prefixes = self.get_rocm_prefix_info() - env.set('ROCM_PATH', '') + env.set('ROCM_PATH', rocm_prefixes['rocm-path']) env.set('HIP_COMPILER', 'clang') env.set('HIP_PLATFORM', 'hcc') env.set('HIP_CLANG_PATH', rocm_prefixes['llvm-amdgpu'].bin) env.set('HSA_PATH', rocm_prefixes['hsa-rocr-dev']) env.set('ROCMINFO_PATH', rocm_prefixes['rocminfo']) - env.set('DEVICE_LIB_PATH', - self.spec['rocm-device-libs'].prefix.amdgcn.bitcode) + env.set('DEVICE_LIB_PATH', rocm_prefixes['rocm-device-libs'].lib) + env.set('HIP_PATH', rocm_prefixes['rocm-path']) env.set('HIPCC_COMPILE_FLAGS_APPEND', - '--rocm-path={0}'.format(self.prefix)) + '--rocm-path={0}'.format(rocm_prefixes['rocm-path'])) + + if 'amdgpu_target' in dependent_spec.variants: + arch = dependent_spec.variants['amdgpu_target'].value + if arch != 'none': + env.set('HCC_AMDGPU_TARGET', arch) def setup_dependent_package(self, module, dependent_spec): self.spec.hipcc = join_path(self.prefix.bin, 'hipcc') diff --git a/var/spack/repos/builtin/packages/raja/package.py b/var/spack/repos/builtin/packages/raja/package.py index 8d1db659cc..7c139640f7 100644 --- a/var/spack/repos/builtin/packages/raja/package.py +++ b/var/spack/repos/builtin/packages/raja/package.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) -class Raja(CMakePackage, CudaPackage): +class Raja(CMakePackage, CudaPackage, HipPackage): """RAJA Parallel Framework.""" homepage = "http://software.llnl.gov/RAJA/" @@ -33,6 +33,8 @@ class Raja(CMakePackage, CudaPackage): variant('examples', default=True, description='Build examples.') variant('exercises', default=True, description='Build exercises.') + conflicts('+openmp', when='+hip') + depends_on('cmake@3.8:', type='build') depends_on('cmake@3.9:', when='+cuda', type='build') @@ -54,6 +56,15 @@ class Raja(CMakePackage, CudaPackage): else: options.append('-DENABLE_CUDA=OFF') + if '+hip' in spec: + arch = self.spec.variants['amdgpu_target'].value + options.extend([ + '-DENABLE_HIP=ON', + '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix), + '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch)]) + else: + options.append('-DENABLE_HIP=OFF') + options.append('-DBUILD_SHARED_LIBS={0}'.format( 'ON' if '+shared' in spec else 'OFF')) diff --git a/var/spack/repos/builtin/packages/umpire/package.py b/var/spack/repos/builtin/packages/umpire/package.py index 6b39aad5f6..14eed00747 100644 --- a/var/spack/repos/builtin/packages/umpire/package.py +++ b/var/spack/repos/builtin/packages/umpire/package.py @@ -7,7 +7,7 @@ import llnl.util.lang as lang import llnl.util.tty as tty -class Umpire(CMakePackage, CudaPackage): +class Umpire(CMakePackage, CudaPackage, HipPackage): """An application-focused API for memory management on NUMA & GPU architectures""" @@ -61,6 +61,14 @@ class Umpire(CMakePackage, CudaPackage): depends_on('cmake@3.9:', when='+cuda', type='build') depends_on('blt', type='build') + + # variants +hip and amdgpu_targets are not automatically passed to + # dependencies, so do it manually. + depends_on('camp+hip', when='+hip') + amdgpu_targets = HipPackage.amd_gputargets_list() + for val in amdgpu_targets: + depends_on('camp amdgpu_target=%s' % val, when='amdgpu_target=%s' % val) + depends_on('camp') conflicts('+numa', when='@:0.3.2') @@ -70,7 +78,6 @@ class Umpire(CMakePackage, CudaPackage): spec = self.spec options = [] - options.append("-DBLT_SOURCE_DIR={0}".format(spec['blt'].prefix)) options.append("-Dcamp_DIR={0}".format(spec['camp'].prefix)) @@ -90,6 +97,15 @@ class Umpire(CMakePackage, CudaPackage): else: options.append('-DENABLE_CUDA=Off') + if '+hip' in spec: + arch = self.spec.variants['amdgpu_target'].value + options.extend([ + '-DENABLE_HIP=ON', + '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix), + '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch)]) + else: + options.append('-DENABLE_HIP=OFF') + options.append('-DENABLE_C={0}'.format( 'On' if '+c' in spec else 'Off')) -- cgit v1.2.3-70-g09d2