From 92040a475f4b43f9cb0ab63f813ac10605aeee35 Mon Sep 17 00:00:00 2001 From: Brian Van Essen Date: Tue, 14 May 2019 14:20:00 -0700 Subject: LBANN: add support for new python front end (#11395) * LBANN: add python dependency and required python library dependencies * LBANN: add version 0.99 as alias for develop branch * LBANN: build Conduit library by default * LBANN: CUB is included with Hydrogen for later versions * New package: py-graphviz (dependency for LBANN) * New package: py-texttable (dependency for LBANN) * New package: py-onnx (dependency for LBANN) * New package: py-typing-extensions (dependency for py-onnx) * Conduit: add version 0.4.0 * NCCL: Add versions 2.4.6-1 and 2.4.2-1 * Protobuf: Add versions 3.7.1 and 3.7.0 * py-protobuf: add version 3.7.1 * py-typing: add version 3.6.4 * cuDNN: add versions. The source differs depending on the architecture, so the versions include a suffix to encode this. For example this adds 7.5.1-10.1-x86_64 as well as 7.5.1-10.1-ppc64le * cuDNN: Newer versions require newer versions of CUDA * Hydrogen: add configure options for CUB and Aluminum --- .../repos/builtin/packages/conduit/package.py | 1 + var/spack/repos/builtin/packages/cudnn/package.py | 33 ++++++++++++-- .../repos/builtin/packages/hydrogen/package.py | 2 + var/spack/repos/builtin/packages/lbann/package.py | 50 ++++++++++++++++------ var/spack/repos/builtin/packages/nccl/package.py | 2 + .../repos/builtin/packages/protobuf/package.py | 2 + .../repos/builtin/packages/py-graphviz/package.py | 33 ++++++++++++++ .../repos/builtin/packages/py-onnx/package.py | 30 +++++++++++++ .../repos/builtin/packages/py-protobuf/package.py | 1 + .../repos/builtin/packages/py-texttable/package.py | 18 ++++++++ .../packages/py-typing-extensions/package.py | 21 +++++++++ .../repos/builtin/packages/py-typing/package.py | 1 + 12 files changed, 177 insertions(+), 17 deletions(-) create mode 100644 var/spack/repos/builtin/packages/py-graphviz/package.py create mode 100644 var/spack/repos/builtin/packages/py-onnx/package.py create mode 100644 var/spack/repos/builtin/packages/py-texttable/package.py create mode 100644 var/spack/repos/builtin/packages/py-typing-extensions/package.py diff --git a/var/spack/repos/builtin/packages/conduit/package.py b/var/spack/repos/builtin/packages/conduit/package.py index be63e9ecf7..3b4fb3e827 100644 --- a/var/spack/repos/builtin/packages/conduit/package.py +++ b/var/spack/repos/builtin/packages/conduit/package.py @@ -38,6 +38,7 @@ class Conduit(Package): git = "https://github.com/LLNL/conduit.git" version('master', branch='master', submodules=True, preferred=True) + version('0.4.0', sha256='c228e6f0ce5a9c0ffb98e0b3d886f2758ace1a4b40d00f3f118542c0747c1f52') version('0.3.1', 'b98d1476199a46bde197220cd9cde042') version('0.3.0', '6396f1d1ca16594d7c66d4535d4f898e') # note: checksums on github automatic release source tars changed ~9/17 diff --git a/var/spack/repos/builtin/packages/cudnn/package.py b/var/spack/repos/builtin/packages/cudnn/package.py index e0277388f9..35959637cf 100644 --- a/var/spack/repos/builtin/packages/cudnn/package.py +++ b/var/spack/repos/builtin/packages/cudnn/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os from spack import * @@ -12,16 +13,42 @@ class Cudnn(Package): homepage = "https://developer.nvidia.com/cudnn" - version('7.3', '72666d3532850752612706601258a0b2', + version('7.5.1-10.1-x86_64', '24017f4a56159d48fd5a31c8a930167b', + url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.1/cudnn-10.1-linux-x64-v7.5.1.10.tgz') + version('7.5.1-10.1-ppc64le', 'ec0993be21e0998fc12116e4c46bce02', + url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.1/cudnn-10.1-linux-ppc64le-v7.5.1.10.tgz') + version('7.5.1-10.0-x86_64', '703ed4be4d242ff4bc0ca48aaf2029bc', + url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.1/cudnn-10.0-linux-x64-v7.5.1.10.tgz') + version('7.5.1-10.0-ppc64le', '8348cbab01970b75836d945d8a75915e', + url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.1/cudnn-10.0-linux-ppc64le-v7.5.1.10.tgz') + version('7.5.0-10.1-x86_64', '4edc7fcfeada9f2363e97f9875154e8f', + url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.0/cudnn-10.1-linux-x64-v7.5.0.56.tgz') + version('7.5.0-10.1-ppc64le', 'b927bc91d93182cb2606e51c9eeee908', + url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.0/cudnn-10.1-linux-ppc64le-v7.5.0.56.tgz') + version('7.5.0-10.0-x86_64', 'b05ca261ffee1e08b496bf8a57790ee8', + url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.0/cudnn-10.0-linux-x64-v7.5.0.56.tgz') + version('7.5.0-10.0-ppc64le', 'e9d2fcdb5340df2408fa08026a2a3e1c', + url='https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.0/cudnn-10.0-linux-ppc64le-v7.5.0.56.tgz') + version('7.3.0', '72666d3532850752612706601258a0b2', url='http://developer.download.nvidia.com/compute/redist/cudnn/v7.3.0/cudnn-9.0-linux-x64-v7.3.0.29.tgz') - version('7.2', '17e010153a4a95bf9c2df2a3c3ceea63', + version('7.2.1', '17e010153a4a95bf9c2df2a3c3ceea63', url='http://developer.download.nvidia.com/compute/redist/cudnn/v7.2.1/cudnn-9.0-linux-x64-v7.2.1.38.tgz') version('6.0', 'a08ca487f88774e39eb6b0ef6507451d', url='http://developer.download.nvidia.com/compute/redist/cudnn/v6.0/cudnn-8.0-linux-x64-v6.0.tgz') version('5.1', '406f4ac7f7ee8aa9e41304c143461a69', url='http://developer.download.nvidia.com/compute/redist/cudnn/v5.1/cudnn-8.0-linux-x64-v5.1.tgz') - depends_on('cuda@8:') + depends_on('cuda@8:', when='@5.1:@7') + depends_on('cuda@9:', when='@7.2:@7.4') + depends_on('cuda@10:', when='@7.5.0-10.0-ppc64le,7.5.0-10.0-x86_64,7.5.1-10.0-ppc64le,7.5.1-10.0-x86_64') + depends_on('cuda@10.1:', when='@7.5.0-10.1-ppc64le,7.5.0-10.1-x86_64,7.5.1-10.1-ppc64le,7.5.1-10.1-x86_64') def install(self, spec, prefix): install_tree('.', prefix) + + if 'target=ppc64le platform=linux' in spec: + symlink(os.path.join(prefix, 'targets', 'ppc64le-linux', 'lib'), + prefix.lib) + symlink( + os.path.join(prefix, 'targets', 'ppc64le-linux', 'include'), + prefix.include) diff --git a/var/spack/repos/builtin/packages/hydrogen/package.py b/var/spack/repos/builtin/packages/hydrogen/package.py index 2fc476ffc5..5c806f312b 100644 --- a/var/spack/repos/builtin/packages/hydrogen/package.py +++ b/var/spack/repos/builtin/packages/hydrogen/package.py @@ -117,6 +117,8 @@ class Hydrogen(CMakePackage): '-DHydrogen_USE_64BIT_BLAS_INTS:BOOL=%s' % ('+int64_blas' in spec), '-DHydrogen_ENABLE_MPC:BOOL=%s' % ('+mpfr' in spec), '-DHydrogen_GENERAL_LAPACK_FALLBACK=ON', + '-DHydrogen_ENABLE_ALUMINUM=%s' % ('+al' in spec), + '-DHydrogen_ENABLE_CUB=%s' % ('+cuda' in spec), '-DHydrogen_ENABLE_CUDA=%s' % ('+cuda' in spec), '-DHydrogen_ENABLE_TESTING=%s' % ('+test' in spec), ] diff --git a/var/spack/repos/builtin/packages/lbann/package.py b/var/spack/repos/builtin/packages/lbann/package.py index 48784b3637..a131a85477 100644 --- a/var/spack/repos/builtin/packages/lbann/package.py +++ b/var/spack/repos/builtin/packages/lbann/package.py @@ -18,6 +18,7 @@ class Lbann(CMakePackage): git = "https://github.com/LLNL/lbann.git" version('develop', branch='develop') + version('0.99', branch='develop') version('0.98.1', sha256='9a2da8f41cd8bf17d1845edf9de6d60f781204ebd37bffba96d8872036c10c66') version('0.98', sha256='8d64b9ac0f1d60db553efa4e657f5ea87e790afe65336117267e9c7ae6f68239') version('0.97.1', sha256='2f2756126ac8bb993202cf532d72c4d4044e877f4d52de9fdf70d0babd500ce4') @@ -40,10 +41,14 @@ class Lbann(CMakePackage): description='The build type to build', values=('Debug', 'Release')) variant('al', default=True, description='Builds with support for Aluminum Library') - variant('conduit', default=False, description='Builds with support for Conduit Library') + variant('conduit', default=True, + description='Builds with support for Conduit Library ' + + '(note that for v0.99 conduit is required)') variant('vtune', default=False, description='Builds with support for Intel VTune') variant('docs', default=False, description='Builds with support for building documentation') + conflicts('@:0.90,0.99:', when='~conduit') + # It seems that there is a need for one statement per version bounds depends_on('hydrogen +openmp_blas +shared +int64', when='@:0.90,0.95: ~al') depends_on('hydrogen +openmp_blas +shared +int64 +al', when='@:0.90,0.95: +al') @@ -74,25 +79,39 @@ class Lbann(CMakePackage): depends_on('cuda', when='+gpu') depends_on('cudnn', when='+gpu') - depends_on('cub', when='+gpu') + depends_on('cub', when='@0.94:0.98.2 +gpu') depends_on('mpi') depends_on('hwloc') # LBANN wraps OpenCV calls in OpenMP parallel loops, build without OpenMP # Additionally disable video related options, they incorrectly link in a # bad OpenMP library when building with clang or Intel compilers + # Note that for Power systems we want the environment to add +powerpc +vsx depends_on('opencv@3.2.0: +core +highgui +imgproc +jpeg +png +tiff +zlib ' - '+fast-math +powerpc +vsx ~calib3d ~cuda ~dnn ~eigen' + '+fast-math ~calib3d ~cuda ~dnn ~eigen' '~features2d ~flann ~gtk ~ipp ~ipp_iw ~jasper ~java ~lapack ~ml' '~openmp ~opencl ~opencl_svm ~openclamdblas ~openclamdfft' '~pthreads_pf ~python ~qt ~stitching ~superres ~ts ~video' '~videostab ~videoio ~vtk', when='+opencv') - depends_on('protobuf@3.0.2: build_type=Release') + depends_on('protobuf@3.6.1: build_type=Release') depends_on('cnpy') - depends_on('nccl', when='+gpu +nccl') - - depends_on('conduit@master +hdf5', when='+conduit') + depends_on('nccl', when='@0.94:0.98.2 +gpu +nccl') + + depends_on('conduit@master +hdf5', when='@0.94:0.99 +conduit') + depends_on('conduit@master +hdf5', when='@:0.90,0.99:') + + depends_on('python@3: +shared', type=('build', 'run'), when='@:0.90,0.99:') + extends("python") + depends_on('py-setuptools', type='build') + depends_on('py-argparse', type='run', when='@:0.90,0.99:') + depends_on('py-configparser', type='run', when='@:0.90,0.99:') + depends_on('py-graphviz@0.10.1:', type='run', when='@:0.90,0.99:') + depends_on('py-matplotlib@3.0.0:', type='run', when='@:0.90,0.99:') + depends_on('py-numpy@1.16.0:', type=('build', 'run'), when='@:0.90,0.99:') + depends_on('py-onnx@1.3.0:', type='run', when='@:0.90,0.99:') + depends_on('py-pandas@0.24.1:', type='run', when='@:0.90,0.99:') + depends_on('py-texttable@1.4.0:', type='run', when='@:0.90,0.99:') depends_on('py-breathe', type='build', when='+docs') depends_on('py-m2r', type='build', when='+docs') @@ -128,7 +147,6 @@ class Lbann(CMakePackage): '-DLBANN_WITH_CONDUIT:BOOL=%s' % ('+conduit' in spec), '-DLBANN_WITH_CUDA:BOOL=%s' % ('+gpu' in spec), '-DLBANN_WITH_CUDNN:BOOL=%s' % ('+gpu' in spec), - '-DLBANN_WITH_NCCL:BOOL=%s' % ('+gpu +nccl' in spec), '-DLBANN_WITH_SOFTMAX_CUDA:BOOL=%s' % ('+gpu' in spec), '-DLBANN_SEQUENTIAL_INITIALIZATION:BOOL=%s' % ('+seq_init' in spec), @@ -146,6 +164,9 @@ class Lbann(CMakePackage): '-DElemental_DIR={0}/CMake/elemental'.format( spec['elemental'].prefix)]) + if self.spec.satisfies('@0.94:0.98.2'): + args.extend(['-DLBANN_WITH_NCCL:BOOL=%s' % ('+gpu +nccl' in spec)]) + if '+vtune' in spec: args.extend(['-DVTUNE_DIR={0}'.format(spec['vtune'].prefix)]) @@ -179,12 +200,13 @@ class Lbann(CMakePackage): args.extend([ '-DcuDNN_DIR={0}'.format( spec['cudnn'].prefix)]) - args.extend(['-DCUB_DIR={0}'.format( - spec['cub'].prefix)]) - if '+nccl' in spec: - args.extend([ - '-DNCCL_DIR={0}'.format( - spec['nccl'].prefix)]) + if self.spec.satisfies('@0.94:0.98.2'): + args.extend(['-DCUB_DIR={0}'.format( + spec['cub'].prefix)]) + if '+nccl' in spec: + args.extend([ + '-DNCCL_DIR={0}'.format( + spec['nccl'].prefix)]) return args diff --git a/var/spack/repos/builtin/packages/nccl/package.py b/var/spack/repos/builtin/packages/nccl/package.py index 789b2984d7..b4a1f7fc37 100644 --- a/var/spack/repos/builtin/packages/nccl/package.py +++ b/var/spack/repos/builtin/packages/nccl/package.py @@ -12,6 +12,8 @@ class Nccl(MakefilePackage): homepage = "https://github.com/NVIDIA/nccl" url = "https://github.com/NVIDIA/nccl/archive/v1.3.4-1.tar.gz" + version('2.4.6-1', sha256='ea4421061a7b9c454f2e088f68bfdbbcefab80ce81cafc70ee6c7742b1439591') + version('2.4.2-1', sha256='e3dd04b22eb541394bd818e5f78ac23a09cc549690d5d55d6fccc1a36155385a') version('2.3.7-1', sha256='e6eff80d9d2db13c61f8452e1400ca2f098d2dfe42857cb23413ce081c5b9e9b') version('2.3.5-5', sha256='bac9950b4d3980c25baa8e3e4541d2dfb4d21edf32ad3b89022d04920357142f') version('2.2', '5b9ce7fbdce0fde68e0f66318e6ff422') diff --git a/var/spack/repos/builtin/packages/protobuf/package.py b/var/spack/repos/builtin/packages/protobuf/package.py index 9ff9076ec3..a76494c65d 100644 --- a/var/spack/repos/builtin/packages/protobuf/package.py +++ b/var/spack/repos/builtin/packages/protobuf/package.py @@ -15,6 +15,8 @@ class Protobuf(CMakePackage): url = "https://github.com/protocolbuffers/protobuf/archive/v3.2.0.tar.gz" root_cmakelists_dir = "cmake" + version('3.7.1', sha256='f1748989842b46fa208b2a6e4e2785133cfcc3e4d43c17fecb023733f0f5443f') + version('3.7.0', sha256='a19dcfe9d156ae45d209b15e0faed5c7b5f109b6117bfc1974b6a7b98a850320') version('3.6.1', sha256='3d4e589d81b2006ca603c1ab712c9715a76227293032d05b26fca603f90b3f5b') version('3.5.2', 'ff6742018c172c66ecc627029ad54280') version('3.5.1.1', '5005003ae6b94773c4bbca87a644b131') diff --git a/var/spack/repos/builtin/packages/py-graphviz/package.py b/var/spack/repos/builtin/packages/py-graphviz/package.py new file mode 100644 index 0000000000..6cbe0ec74a --- /dev/null +++ b/var/spack/repos/builtin/packages/py-graphviz/package.py @@ -0,0 +1,33 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +from spack import * + + +class PyGraphviz(PythonPackage): + """Simple Python interface for Graphviz""" + + homepage = "https://github.com/xflr6/graphviz" + url = "https://pypi.io/packages/source/g/graphviz/graphviz-0.10.1.zip" + + version('0.10.1', sha256='d311be4fddfe832a56986ac5e1d6e8715d7fcb0208560da79d1bb0f72abef41f') + + variant('dev', default=False, description='development mode') + variant('docs', default=False, description='build documentation') + + depends_on('python@2.7:2.8,3.4:', type=('build', 'run')) + depends_on('py-setuptools', type='build') + depends_on('py-tox@3.0:', type=('build', 'run'), when='+dev') + depends_on('py-flake8', type=('build', 'run'), when='+dev') + depends_on('py-pep8-naming', type=('build', 'run'), when='+dev') + depends_on('py-wheel', type=('build', 'run'), when='+dev') + depends_on('py-twine', type=('build', 'run'), when='+dev') + depends_on('py-mock@2:', type='test') + depends_on('py-pytest@3.4:', type='test') + depends_on('py-pytest-mock@1.8:', type='test') + depends_on('py-pytest-cov', type='test') + depends_on('py-sphinx@1.7:', type=('build', 'run'), when='+docs') + depends_on('py-sphinx-rtd-theme', type=('build', 'run'), when='+docs') diff --git a/var/spack/repos/builtin/packages/py-onnx/package.py b/var/spack/repos/builtin/packages/py-onnx/package.py new file mode 100644 index 0000000000..861edc2203 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-onnx/package.py @@ -0,0 +1,30 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +from spack import * + + +class PyOnnx(PythonPackage): + """Open Neural Network Exchange (ONNX) is an open ecosystem that + empowers AI developers to choose the right tools as their + project evolves. ONNX provides an open source format for AI + models, both deep learning and traditional ML. It defines an + extensible computation graph model, as well as definitions of + built-in operators and standard data types. Currently we focus + on the capabilities needed for inferencing (scoring).""" + + homepage = "https://github.com/onnx/onnx" + url = "https://pypi.io/packages/source/O/Onnx/onnx-1.5.0.tar.gz" + + version('1.5.0', sha256='1a584a4ef62a6db178c257fffb06a9d8e61b41c0a80bfd8bcd8a253d72c4b0b4') + + depends_on('py-setuptools', type='build') + depends_on('protobuf') + depends_on('py-protobuf+cpp', type=('build', 'run')) + depends_on('py-numpy', type=('build', 'run')) + depends_on('py-six', type=('build', 'run')) + depends_on('py-typing@3.6.4:', type=('build', 'run')) + depends_on('py-typing-extensions@3.6.4:', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-protobuf/package.py b/var/spack/repos/builtin/packages/py-protobuf/package.py index e09fcf4a16..8730b265f7 100644 --- a/var/spack/repos/builtin/packages/py-protobuf/package.py +++ b/var/spack/repos/builtin/packages/py-protobuf/package.py @@ -20,6 +20,7 @@ class PyProtobuf(PythonPackage): variant('cpp', default=False, description='Enable the cpp implementation') + version('3.7.1', sha256='21e395d7959551e759d604940a115c51c6347d90a475c9baf471a1a86b5604a9') version('3.6.1', sha256='1489b376b0f364bcc6f89519718c057eb191d7ad6f1b395ffd93d1aa45587811') version('3.5.2.post1', '3b60685732bd0cbdc802dfcb6071efbcf5d927ce3127c13c33ea1a8efae3aa76') version('3.5.2', '09879a295fd7234e523b62066223b128c5a8a88f682e3aff62fb115e4a0d8be0') diff --git a/var/spack/repos/builtin/packages/py-texttable/package.py b/var/spack/repos/builtin/packages/py-texttable/package.py new file mode 100644 index 0000000000..2dee0f93ed --- /dev/null +++ b/var/spack/repos/builtin/packages/py-texttable/package.py @@ -0,0 +1,18 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +from spack import * + + +class PyTexttable(PythonPackage): + """Python module for creating simple ASCII tables.""" + + homepage = "https://github.com/foutaise/texttable/" + url = "https://pypi.io/packages/source/t/texttable/texttable-1.6.1.tar.gz" + + version('1.6.1', sha256='2b60a5304ccfbeac80ffae7350d7c2f5d7a24e9aab5036d0f82489746419d9b2') + + depends_on('py-setuptools', type='build') diff --git a/var/spack/repos/builtin/packages/py-typing-extensions/package.py b/var/spack/repos/builtin/packages/py-typing-extensions/package.py new file mode 100644 index 0000000000..35f1c4288d --- /dev/null +++ b/var/spack/repos/builtin/packages/py-typing-extensions/package.py @@ -0,0 +1,21 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class PyTypingExtensions(PythonPackage): + """The typing_extensions module contains both backports of these + changes as well as experimental types that will eventually be + added to the typing module, such as Protocol (see PEP 544 for + details about protocols and static duck typing).""" + + homepage = "https://github.com/python/typing/tree/master/typing_extensions" + url = "https://pypi.io/packages/source/t/typing_extensions/typing_extensions-3.7.2.tar.gz" + + version('3.7.2', sha256='fb2cd053238d33a8ec939190f30cfd736c00653a85a2919415cecf7dc3d9da71') + + depends_on('py-setuptools', type='build') + depends_on('py-typing@3.6.4:', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-typing/package.py b/var/spack/repos/builtin/packages/py-typing/package.py index 4ca09eb97e..f475c5eb2a 100644 --- a/var/spack/repos/builtin/packages/py-typing/package.py +++ b/var/spack/repos/builtin/packages/py-typing/package.py @@ -15,6 +15,7 @@ class PyTyping(PythonPackage): import_modules = ['typing'] + version('3.6.4', '5b2ade08d83be488f17b5fe587c27c74') version('3.6.1', '3fec97415bae6f742fb3c3013dedeb89') # You need Python 2.7 or 3.3+ to install the typing package -- cgit v1.2.3-60-g2f50