From 46e7fbe120076439f4f0013b95e6fae8291effc5 Mon Sep 17 00:00:00 2001 From: Brian Van Essen Date: Fri, 31 Jul 2020 13:53:51 -0700 Subject: LBANN: add versions, update CUDA support and dependencies (#17813) * Update LBANN, Hydrogen, Aluminum to inherit CudaPackage * Update CMake constraints: LBANN, Hydrogen, and Aluminum now require cmake@3.16.0: (better support for pthreads with nvcc) * Aluminum: add variants for host-enabled MPI and RMA features in a MPI-GPU RDMA-enabled library * NCCL: add versions 2.7.5-1, 2.7.6-1, and 2.7.8-1 * Hydrogen: add version 1.4.0 * LBANN: add versions 0.99 and 0.100 * Aluminum: add versions 0.4.0 and 0.5.0 --- .../repos/builtin/packages/aluminum/package.py | 27 +++++++++++---- .../repos/builtin/packages/dihydrogen/package.py | 11 ++++++- .../repos/builtin/packages/hydrogen/package.py | 7 ++-- var/spack/repos/builtin/packages/lbann/package.py | 38 ++++++++++++---------- var/spack/repos/builtin/packages/nccl/package.py | 3 ++ 5 files changed, 58 insertions(+), 28 deletions(-) diff --git a/var/spack/repos/builtin/packages/aluminum/package.py b/var/spack/repos/builtin/packages/aluminum/package.py index c5d6a45ed8..fbcf0f5be8 100644 --- a/var/spack/repos/builtin/packages/aluminum/package.py +++ b/var/spack/repos/builtin/packages/aluminum/package.py @@ -7,7 +7,7 @@ import os from spack import * -class Aluminum(CMakePackage): +class Aluminum(CMakePackage, CudaPackage): """Aluminum provides a generic interface to high-performance communication libraries, with a focus on allreduce algorithms. Blocking and non-blocking algorithms and GPU-aware @@ -20,6 +20,8 @@ class Aluminum(CMakePackage): git = "https://github.com/LLNL/Aluminum.git" version('master', branch='master') + version('0.5.0', sha256='dc365a5849eaba925355a8efb27005c5f22bcd1dca94aaed8d0d29c265c064c1') + version('0.4.0', sha256='4d6fab5481cc7c994b32fb23a37e9ee44041a9f91acf78f981a97cb8ef57bb7d') version('0.3.3', sha256='26e7f263f53c6c6ee0fe216e981a558dfdd7ec997d0dd2a24285a609a6c68f3b') version('0.3.2', sha256='09b6d1bcc02ac54ba269b1123eee7be20f0104b93596956c014b794ba96b037f') version('0.2.1-1', sha256='066b750e9d1134871709a3e2414b96b166e0e24773efc7d512df2f1d96ee8eef') @@ -27,12 +29,13 @@ class Aluminum(CMakePackage): version('0.2', sha256='fc8f06c6d8faab17a2aedd408d3fe924043bf857da1094d5553f35c4d2af893b') version('0.1', sha256='3880b736866e439dd94e6a61eeeb5bb2abccebbac82b82d52033bc6c94950bdb') - variant('gpu', default=False, description='Builds with support for GPUs via CUDA and cuDNN') variant('nccl', default=False, description='Builds with support for NCCL communication lib') - variant('mpi_cuda', default=False, description='Builds with support for MPI-CUDA enabled library') + variant('ht', default=False, description='Builds with support for host-enabled MPI' + ' communication of accelerator data') + variant('mpi_gpu_rdma', default=False, description='Builds with support for using RMA' + ' features in a MPI-GPU RDMA enabled library') - depends_on('cmake@3.9.0:', type='build') - depends_on('cuda', when='+gpu') + depends_on('cmake@3.16.0:', type='build') depends_on('mpi') depends_on('nccl', when='+nccl') depends_on('hwloc') @@ -43,10 +46,20 @@ class Aluminum(CMakePackage): def cmake_args(self): spec = self.spec args = [ - '-DALUMINUM_ENABLE_CUDA:BOOL=%s' % ('+gpu' in spec), - '-DALUMINUM_ENABLE_MPI_CUDA:BOOL=%s' % ('+mpi_cuda' in spec), + '-DALUMINUM_ENABLE_CUDA:BOOL=%s' % ('+cuda' in spec), '-DALUMINUM_ENABLE_NCCL:BOOL=%s' % ('+nccl' in spec)] + if '@0.5:': + args.extend([ + '-DALUMINUM_ENABLE_HOST_TRANSFER:BOOL=%s' % ('+ht' in spec), + '-DALUMINUM_ENABLE_MPI_CUDA:BOOL=%s' % + ('+mpi_gpu_rdma' in spec), + '-DALUMINUM_ENABLE_MPI_CUDA_RMA:BOOL=%s' % + ('+mpi_gpu_rdma' in spec)]) + else: + args.extend([ + '-DALUMINUM_ENABLE_MPI_CUDA:BOOL=%s' % ('+ht' in spec)]) + # Add support for OS X to find OpenMP (LLVM installed via brew) if self.spec.satisfies('%clang platform=darwin'): clang = self.compiler.cc diff --git a/var/spack/repos/builtin/packages/dihydrogen/package.py b/var/spack/repos/builtin/packages/dihydrogen/package.py index 6de0cfb516..7f444d8aff 100644 --- a/var/spack/repos/builtin/packages/dihydrogen/package.py +++ b/var/spack/repos/builtin/packages/dihydrogen/package.py @@ -14,6 +14,7 @@ class Dihydrogen(CMakePackage, CudaPackage): needs of the distributed machine learning effort, LBANN.""" homepage = "https://github.com/LLNL/DiHydrogen.git" + url = "https://github.com/LLNL/DiHydrogen.git" git = "https://github.com/LLNL/DiHydrogen.git" maintainers = ['bvanessen'] @@ -36,22 +37,27 @@ class Dihydrogen(CMakePackage, CudaPackage): description='Enable ROCm/HIP language features.') variant('shared', default=True, description='Enables the build of shared libraries') + variant('docs', default=False, + description='Builds with support for building documentation') # Override the default set of CUDA architectures with the relevant # subset from lib/spack/spack/build_systems/cuda.py cuda_arch_values = [ '60', '61', '62', '70', '72', '75', + '80' ] variant('cuda_arch', description='CUDA architecture', values=spack.variant.auto_or_any_combination_of(*cuda_arch_values)) + depends_on('cmake@3.16.0:', type='build') + depends_on('mpi') depends_on('catch2', type='test') depends_on('aluminum', when='+al ~cuda') - depends_on('aluminum +gpu +nccl +mpi_cuda', when='+al +cuda') + depends_on('aluminum +cuda +nccl +ht +mpi_gpu_rdma', when='+al +cuda') depends_on('cuda', when=('+cuda' or '+legacy')) depends_on('cudnn', when=('+cuda' or '+legacy')) @@ -66,6 +72,9 @@ class Dihydrogen(CMakePackage, CudaPackage): depends_on('ninja', type='build') depends_on('cmake@3.14.0:', type='build') + depends_on('py-breathe', type='build', when='+docs') + depends_on('doxygen', type='build', when='+docs') + illegal_cuda_arch_values = [ '10', '11', '12', '13', '20', '21', diff --git a/var/spack/repos/builtin/packages/hydrogen/package.py b/var/spack/repos/builtin/packages/hydrogen/package.py index 9d12235d72..41d5541648 100644 --- a/var/spack/repos/builtin/packages/hydrogen/package.py +++ b/var/spack/repos/builtin/packages/hydrogen/package.py @@ -7,7 +7,7 @@ import os from spack import * -class Hydrogen(CMakePackage): +class Hydrogen(CMakePackage, CudaPackage): """Hydrogen: Distributed-memory dense and sparse-direct linear algebra and optimization library. Based on the Elemental library.""" @@ -18,6 +18,7 @@ class Hydrogen(CMakePackage): maintainers = ['bvanessen'] version('develop', branch='hydrogen') + version('1.4.0', sha256='c13374ff4a6c4d1076e47ba8c8d91a7082588b9958d1ed89cffb12f1d2e1452e') version('1.3.4', sha256='7979f6656f698f0bbad6798b39d4b569835b3013ff548d98089fce7c283c6741') version('1.3.3', sha256='a51a1cfd40ac74d10923dfce35c2c04a3082477683f6b35e7b558ea9f4bb6d51') version('1.3.2', sha256='50bc5e87955f8130003d04dfd9dcad63107e92b82704f8107baf95b0ccf98ed6') @@ -62,6 +63,8 @@ class Hydrogen(CMakePackage): variant('half', default=True, description='Builds with support for FP16 precision data types') + depends_on('cmake@3.16.0:', type='build') + # Note that #1712 forces us to enumerate the different blas variants depends_on('openblas', when='blas=openblas ~openmp_blas ~int64_blas') depends_on('openblas +ilp64', when='blas=openblas ~openmp_blas +int64_blas') @@ -83,7 +86,7 @@ class Hydrogen(CMakePackage): depends_on('netlib-lapack +external-blas', when='blas=essl') depends_on('aluminum', when='+al ~cuda') - depends_on('aluminum +gpu +nccl', when='+al +cuda') + depends_on('aluminum +cuda +nccl +ht +mpi_gpu_rdma', when='+al +cuda') # Note that this forces us to use OpenBLAS until #1712 is fixed depends_on('lapack', when='blas=openblas ~openmp_blas') diff --git a/var/spack/repos/builtin/packages/lbann/package.py b/var/spack/repos/builtin/packages/lbann/package.py index bc0da109ec..8eacb2fb71 100644 --- a/var/spack/repos/builtin/packages/lbann/package.py +++ b/var/spack/repos/builtin/packages/lbann/package.py @@ -8,7 +8,7 @@ import sys from spack import * -class Lbann(CMakePackage): +class Lbann(CMakePackage, CudaPackage): """LBANN: Livermore Big Artificial Neural Network Toolkit. A distributed memory, HPC-optimized, model and data parallel training toolkit for deep neural networks.""" @@ -20,7 +20,8 @@ class Lbann(CMakePackage): maintainers = ['bvanessen'] version('develop', branch='develop') - version('0.99', branch='develop') + version('0.100', sha256='d1bab4fb6f1b80ae83a7286cc536a32830890f6e5b0c3107a17c2600d0796912') + version('0.99', sha256='3358d44f1bc894321ce07d733afdf6cb7de39c33e3852d73c9f31f530175b7cd') version('0.98.1', sha256='9a2da8f41cd8bf17d1845edf9de6d60f781204ebd37bffba96d8872036c10c66') version('0.98', sha256='8d64b9ac0f1d60db553efa4e657f5ea87e790afe65336117267e9c7ae6f68239') version('0.97.1', sha256='2f2756126ac8bb993202cf532d72c4d4044e877f4d52de9fdf70d0babd500ce4') @@ -32,7 +33,6 @@ class Lbann(CMakePackage): version('0.92', sha256='9187c5bcbc562c2828fe619d53884ab80afb1bcd627a817edb935b80affe7b84') version('0.91', sha256='b69f470829f434f266119a33695592f74802cff4b76b37022db00ab32de322f5') - variant('gpu', default=False, description='Builds with support for GPUs via CUDA and cuDNN') variant('nccl', default=False, description='Builds with support for NCCL communication lib') variant('opencv', default=True, description='Builds with support for image processing routines with OpenCV') variant('seq_init', default=False, description='Force serial initialization of weight matrices.') @@ -52,6 +52,8 @@ class Lbann(CMakePackage): conflicts('@:0.90,0.99:', when='~conduit') + depends_on('cmake@3.16.0:', type='build') + # It seems that there is a need for one statement per version bounds depends_on('hydrogen +openmp_blas +shared +int64', when='@:0.90,0.95: ~al') depends_on('hydrogen +openmp_blas +shared +int64 +al', when='@:0.90,0.95: +al') @@ -77,12 +79,11 @@ class Lbann(CMakePackage): when='build_type=Debug @0.91:0.94') depends_on('aluminum', when='@:0.90,0.95: +al ~gpu') - depends_on('aluminum +gpu +mpi_cuda', when='@:0.90,0.95: +al +gpu ~nccl') - depends_on('aluminum +gpu +nccl +mpi_cuda', when='@:0.90,0.95: +al +gpu +nccl') + depends_on('aluminum +cuda +ht', when='@:0.90,0.95: +al +cuda ~nccl') + depends_on('aluminum +cuda +nccl +ht', when='@:0.90,0.95: +al +cuda +nccl') - depends_on('cuda', when='+gpu') - depends_on('cudnn', when='+gpu') - depends_on('cub', when='@0.94:0.98.2 +gpu') + depends_on('cudnn', when='+cuda') + depends_on('cub', when='@0.94:0.98.2 +cuda') depends_on('mpi') depends_on('hwloc') @@ -98,7 +99,7 @@ class Lbann(CMakePackage): '~videostab ~videoio ~vtk', when='+opencv') depends_on('cnpy') - depends_on('nccl', when='@0.94:0.98.2 +gpu +nccl') + depends_on('nccl', when='@0.94:0.98.2 +cuda +nccl') depends_on('conduit@0.4.0: +hdf5', when='@0.94:0.99 +conduit') depends_on('conduit@0.4.0: +hdf5', when='@:0.90,0.99:') @@ -118,6 +119,7 @@ class Lbann(CMakePackage): depends_on('py-protobuf+cpp@3.6.1:', type=('build', 'run'), when='@:0.90,0.99:') depends_on('py-breathe', type='build', when='+docs') + depends_on('doxygen', type='build', when='+docs') depends_on('py-m2r', type='build', when='+docs') depends_on('cereal') @@ -135,7 +137,6 @@ class Lbann(CMakePackage): cppflags.append('-DLBANN_SET_EL_RNG -ldl') return [ - '-DCMAKE_INSTALL_MESSAGE=LAZY', '-DCMAKE_CXX_FLAGS=%s' % ' '.join(cppflags), '-DLBANN_VERSION=spack', '-DCNPY_DIR={0}'.format(spec['cnpy'].prefix), @@ -148,12 +149,12 @@ class Lbann(CMakePackage): spec = self.spec args = self.common_config_args args.extend([ - '-DLBANN_WITH_TOPO_AWARE:BOOL=%s' % ('+gpu +nccl' in spec), + '-DLBANN_WITH_TOPO_AWARE:BOOL=%s' % ('+cuda +nccl' in spec), '-DLBANN_WITH_ALUMINUM:BOOL=%s' % ('+al' in spec), '-DLBANN_WITH_CONDUIT:BOOL=%s' % ('+conduit' in spec), - '-DLBANN_WITH_CUDA:BOOL=%s' % ('+gpu' in spec), - '-DLBANN_WITH_CUDNN:BOOL=%s' % ('+gpu' in spec), - '-DLBANN_WITH_SOFTMAX_CUDA:BOOL=%s' % ('+gpu' in spec), + '-DLBANN_WITH_CUDA:BOOL=%s' % ('+cuda' in spec), + '-DLBANN_WITH_CUDNN:BOOL=%s' % ('+cuda' in spec), + '-DLBANN_WITH_SOFTMAX_CUDA:BOOL=%s' % ('+cuda' in spec), '-DLBANN_SEQUENTIAL_INITIALIZATION:BOOL=%s' % ('+seq_init' in spec), '-DLBANN_WITH_TBINF=OFF', @@ -174,7 +175,8 @@ class Lbann(CMakePackage): spec['elemental'].prefix)]) if spec.satisfies('@0.94:0.98.2'): - args.extend(['-DLBANN_WITH_NCCL:BOOL=%s' % ('+gpu +nccl' in spec)]) + args.extend(['-DLBANN_WITH_NCCL:BOOL=%s' % + ('+cuda +nccl' in spec)]) if '+vtune' in spec: args.extend(['-DVTUNE_DIR={0}'.format(spec['vtune'].prefix)]) @@ -203,7 +205,7 @@ class Lbann(CMakePackage): args.extend(['-DOpenCV_DIR:STRING={0}'.format( spec['opencv'].prefix)]) - if '+gpu' in spec: + if '+cuda' in spec: args.extend([ '-DCUDA_TOOLKIT_ROOT_DIR={0}'.format( spec['cuda'].prefix)]) @@ -225,8 +227,8 @@ class Lbann(CMakePackage): spec = self.spec args = self.common_config_args args.extend([ - '-DWITH_CUDA:BOOL=%s' % ('+gpu' in spec), - '-DWITH_CUDNN:BOOL=%s' % ('+gpu' in spec), + '-DWITH_CUDA:BOOL=%s' % ('+cuda' in spec), + '-DWITH_CUDNN:BOOL=%s' % ('+cuda' in spec), '-DELEMENTAL_USE_CUBLAS:BOOL=%s' % ( '+cublas' in spec['elemental']), '-DWITH_TBINF=OFF', diff --git a/var/spack/repos/builtin/packages/nccl/package.py b/var/spack/repos/builtin/packages/nccl/package.py index d1f4de962d..f805f55b1c 100644 --- a/var/spack/repos/builtin/packages/nccl/package.py +++ b/var/spack/repos/builtin/packages/nccl/package.py @@ -14,6 +14,9 @@ class Nccl(MakefilePackage, CudaPackage): maintainers = ['adamjstewart'] + version('2.7.8-1', sha256='fa2bec307270f30fcf6280a85f24ea8801e0ce3b3027937c7325260a890b07e0') + version('2.7.6-1', sha256='60dd9b1743c2db6c05f60959edf98a4477f218115ef910d7ec2662f2fb5cf626') + version('2.7.5-1', sha256='26a8dec6fa0a776eb71205d618d58e26d372621719788a23b33db6fdce4426bf') version('2.7.3-1', sha256='dc7b8794373306e323363314c3327796e416f745e8003490fc1407a22dd7acd6') version('2.6.4-1', sha256='ed8c9dfd40e013003923ae006787b1a30d3cb363b47d2e4307eaa2624ebba2ba') version('2.5.7-1', sha256='781a6bb2278566be4abbdf22b2fa19afc7306cff4b312c82bd782979b368014e') -- cgit v1.2.3-60-g2f50