From 078ee48c4bf8ca6526ced35a3ae982f41eae29e3 Mon Sep 17 00:00:00 2001 From: Asher Mancinelli Date: Tue, 29 Mar 2022 11:01:30 -0600 Subject: ExaGO: add v1.4.0 and v1.4.1 (#29712) * ExaGO: Handling of cuda architectures and amdgpu targets changed to effectively handle multiple targets. See #28441. * Add ROCm support to ExaGO and update ROCm support in HiOp * ExaGO+rocm requires HiOp+rocm * Newer versions of CMake may set HIP_CLANG_INCLUDE_PATH incorrectly: add comments to the ExaGO/HiOp packages explaining how to address this problem if it occurs. --- var/spack/repos/builtin/packages/exago/package.py | 60 +++++++++++++++++------ var/spack/repos/builtin/packages/hiop/package.py | 54 +++++++++++++++----- 2 files changed, 85 insertions(+), 29 deletions(-) (limited to 'var') diff --git a/var/spack/repos/builtin/packages/exago/package.py b/var/spack/repos/builtin/packages/exago/package.py index 9046748573..037c26bf07 100644 --- a/var/spack/repos/builtin/packages/exago/package.py +++ b/var/spack/repos/builtin/packages/exago/package.py @@ -6,7 +6,7 @@ from spack import * -class Exago(CMakePackage, CudaPackage): +class Exago(CMakePackage, CudaPackage, ROCmPackage): """ExaGO is a package for solving large-scale power grid optimization problems on parallel and distributed architectures, particularly targeted for exascale machines.""" @@ -15,7 +15,9 @@ class Exago(CMakePackage, CudaPackage): git = 'https://gitlab.pnnl.gov/exasgd/frameworks/exago.git' maintainers = ['ashermancinelli', 'CameronRutherford'] - version('1.3.0', commit='58b039d746a6eac8e84b0afc01354cd58caec485', submodules=True, preferred=True) + version('1.4.1', commit='ea607c685444b5f345bfdc9a59c345f0f30adde2', submodules=True, preferred=True) + version('1.4.0', commit='4f4c3fdb40b52ace2d6ba000e7f24b340ec8e886', submodules=True) + version('1.3.0', commit='58b039d746a6eac8e84b0afc01354cd58caec485', submodules=True) version('1.2.0', commit='255a214e', submodules=True) version('1.1.2', commit='db3bb16e', submodules=True) version('1.1.1', commit='0e0a3f27', submodules=True) @@ -30,6 +32,7 @@ class Exago(CMakePackage, CudaPackage): variant('mpi', default=True, description='Enable/Disable MPI') variant('raja', default=False, description='Enable/Disable RAJA') variant('python', default=True, description='Enable/Disable Python bindings') + conflicts('+python', when='@:1.3.0', msg='Python bindings require ExaGO 1.4') # Solver options variant('hiop', default=False, description='Enable/Disable HiOp') @@ -44,6 +47,7 @@ class Exago(CMakePackage, CudaPackage): depends_on('raja', when='+raja') depends_on('raja+cuda', when='+raja+cuda') + depends_on('raja+rocm', when='+raja+rocm') depends_on('raja@0.14.0:', when='@1.1.0: +raja') depends_on('umpire', when='+raja') depends_on('umpire@6.0.0:', when='@1.1.0: +raja') @@ -65,37 +69,61 @@ class Exago(CMakePackage, CudaPackage): depends_on('hiop@0.5.3:', when='@1.3.0:+hiop') depends_on('hiop+cuda', when='+hiop+cuda') + depends_on('hiop+rocm', when='+hiop+rocm') depends_on('hiop~mpi', when='+hiop~mpi') depends_on('hiop+mpi', when='+hiop+mpi') depends_on('petsc@3.13:3.14', when='@:1.2.99') - depends_on('petsc@3.16.0', when='@1.3.0:') + depends_on('petsc@3.16.0:', when='@1.3.0:') depends_on('petsc~mpi', when='~mpi') depends_on('ipopt', when='+ipopt') + depends_on('py-mpi4py', when='@1.3.0:+mpi+python') + flag_handler = build_system_flags def cmake_args(self): args = [] spec = self.spec - args.append("-DEXAGO_RUN_TESTS=ON") - - args.append(self.define_from_variant('EXAGO_ENABLE_MPI', 'mpi')) - args.append(self.define_from_variant('EXAGO_ENABLE_RAJA', 'raja')) - args.append(self.define_from_variant('EXAGO_ENABLE_HIOP', 'hiop')) - args.append(self.define_from_variant('EXAGO_ENABLE_IPOPT', 'ipopt')) - args.append(self.define_from_variant('EXAGO_ENABLE_GPU', 'cuda')) - args.append(self.define_from_variant('EXAGO_ENABLE_CUDA', 'cuda')) - args.append(self.define_from_variant('EXAGO_ENABLE_PYTHON', 'python')) - args.append("-DPETSC_DIR='{0}'".format(spec['petsc'].prefix)) + # NOTE: If building with spack develop on a cluster, you may want to + # change the ctest launch command to use your job scheduler like so: + # + # args.append( + # self.define('EXAGO_CTEST_LAUNCH_COMMAND', 'srun -t 10:00')) + + args.extend([ + self.define('EXAGO_ENABLE_GPU', '+cuda' in spec or '+rocm' in spec), + self.define_from_variant('EXAGO_ENABLE_CUDA', 'cuda'), + self.define_from_variant('EXAGO_ENABLE_HIP', 'rocm'), + self.define('PETSC_DIR', spec['petsc'].prefix), + self.define('EXAGO_RUN_TESTS', True), + self.define_from_variant('EXAGO_ENABLE_MPI', 'mpi'), + self.define_from_variant('EXAGO_ENABLE_RAJA', 'raja'), + self.define_from_variant('EXAGO_ENABLE_HIOP', 'hiop'), + self.define_from_variant('EXAGO_ENABLE_IPOPT', 'ipopt'), + self.define_from_variant('EXAGO_ENABLE_PYTHON', 'python'), + ]) if '+cuda' in spec: cuda_arch_list = spec.variants['cuda_arch'].value - cuda_arch = cuda_arch_list[0] - if cuda_arch != 'none': + if cuda_arch_list[0] != 'none': args.append( - "-DCMAKE_CUDA_ARCHITECTURES={0}".format(cuda_arch)) + self.define('CMAKE_CUDA_ARCHITECTURES', cuda_arch_list)) + + # NOTE: if +rocm, some HIP CMake variables may not be set correctly. + # Namely, HIP_CLANG_INCLUDE_PATH. If the configure phase fails due to + # this variable being undefined, adding the following line typically + # resolves this issue: + # + # args.append( + # self.define('HIP_CLANG_INCLUDE_PATH', + # '/opt/rocm-X.Y.Z/llvm/lib/clang/14.0.0/include/')) + if '+rocm' in spec: + rocm_arch_list = spec.variants['amdgpu_target'].value + if rocm_arch_list[0] != 'none': + args.append(self.define('GPU_TARGETS', rocm_arch_list)) + args.append(self.define('AMDGPU_TARGETS', rocm_arch_list)) return args diff --git a/var/spack/repos/builtin/packages/hiop/package.py b/var/spack/repos/builtin/packages/hiop/package.py index 98416778fc..cbe258aabe 100644 --- a/var/spack/repos/builtin/packages/hiop/package.py +++ b/var/spack/repos/builtin/packages/hiop/package.py @@ -76,7 +76,8 @@ class Hiop(CMakePackage, CudaPackage, ROCmPackage): depends_on('magma@{0}:'.format(magma_v), when='@{0}:+cuda'.format(hiop_v)) depends_on('magma@{0}:'.format(magma_v), when='@{0}:+rocm'.format(hiop_v)) - depends_on('raja+openmp', when='+raja') + depends_on('raja', when='+raja') + depends_on('raja+openmp', when='+raja~cuda~rocm') depends_on('raja@0.14.0:', when='@0.5.0:+raja') depends_on('raja+cuda', when='+raja+cuda') depends_on('raja+rocm', when='+raja+rocm') @@ -105,15 +106,19 @@ class Hiop(CMakePackage, CudaPackage, ROCmPackage): args = [] spec = self.spec - if spec.satisfies('+rocm') or spec.satisfies('+cuda'): - args.append('-DHIOP_USE_GPU=ON') - args.append('-DHIOP_USE_MAGMA=ON') + use_gpu = '+cuda' in spec or '+rocm' in spec + + if use_gpu: + args.extend([ + self.define('HIOP_USE_GPU', True), + self.define('HIOP_USE_MAGMA', True), + self.define('HIOP_MAGMA_DIR', spec['magma'].prefix), + ]) args.extend([ self.define('HIOP_BUILD_STATIC', True), self.define('LAPACK_FOUND', True), self.define('LAPACK_LIBRARIES', spec['lapack'].libs + spec['blas'].libs), - self.define('HIOP_USE_HIP', False), self.define_from_variant('HIOP_BUILD_SHARED', 'shared'), self.define_from_variant('HIOP_USE_MPI', 'mpi'), self.define_from_variant('HIOP_DEEPCHECKS', 'deepchecking'), @@ -127,6 +132,12 @@ class Hiop(CMakePackage, CudaPackage, ROCmPackage): self.define_from_variant('HIOP_TEST_WITH_BSUB', 'jsrun'), ]) + # NOTE: If building with spack develop on a cluster, you may want to + # change the ctest launch command to use your job scheduler like so: + # + # args.append( + # self.define('HIOP_CTEST_LAUNCH_COMMAND', 'srun -t 10:00')) + if '+mpi' in spec: args.extend([ self.define('MPI_HOME', spec['mpi'].prefix), @@ -134,17 +145,34 @@ class Hiop(CMakePackage, CudaPackage, ROCmPackage): self.define('MPI_CXX_COMPILER', spec['mpi'].mpicxx), self.define('MPI_Fortran_COMPILER', spec['mpi'].mpifc), ]) + # NOTE: On Cray platforms, libfabric is occasionally not picked up + # by Spack, causing HiOp's CMake code to fail to find MPI Fortran + # libraries. If this is the case, adding the following lines may + # resolve the issue. Searching /CMakeFiles/CMakeError.log + # for MPI Fortran errors is the fastest way to check for this error. + # + # args.append( + # self.define('MPI_Fortran_LINK_FLAGS', + # '-L/path/to/libfabric/lib64/ -lfabric')) if '+cuda' in spec: cuda_arch_list = spec.variants['cuda_arch'].value - cuda_arch = cuda_arch_list[0] - if cuda_arch != 'none': - args.extend([ - self.define('HIOP_NVCC_ARCH', 'sm_{0}'.format(cuda_arch)), - self.define('CMAKE_CUDA_ARCHITECTURES', cuda_arch), - ]) - if '+magma' in spec: - args.append(self.define('HIOP_MAGMA_DIR', spec['magma'].prefix)) + if cuda_arch_list[0] != 'none': + args.append(self.define('CMAKE_CUDA_ARCHITECTURES', cuda_arch_list)) + + # NOTE: if +rocm, some HIP CMake variables may not be set correctly. + # Namely, HIP_CLANG_INCLUDE_PATH. If the configure phase fails due to + # this variable being undefined, adding the following line typically + # resolves this issue: + # + # args.append( + # self.define('HIP_CLANG_INCLUDE_PATH', + # '/opt/rocm-X.Y.Z/llvm/lib/clang/14.0.0/include/')) + if '+rocm' in spec: + rocm_arch_list = spec.variants['amdgpu_target'].value + if rocm_arch_list[0] != 'none': + args.append(self.define('GPU_TARGETS', rocm_arch_list)) + args.append(self.define('AMDGPU_TARGETS', rocm_arch_list)) if '+kron' in spec: args.append(self.define('HIOP_UMFPACK_DIR', spec['suite-sparse'].prefix)) -- cgit v1.2.3-70-g09d2