LBANN SW stack packages: gcc-toolchain and clang support (#28769)

* Added support to LBANN, Hydrogen, DiHydrogen, and Aluminum to capture
  a gcc-toolchain cxxflags argument and pass it to a CMAKE_CUDA_FLAG
  argument when set.  This helps deal with compiling with clang on
  systems with old base gcc installations.
* Added a dependency on py-scipy when enabling tests on LBANN.
* Updated the C++ standard for Hydrogen to C++17.
* Added a new variant +apps to enable (or disable) python packages that
  are used by applications in the LBANN repo, but are not strictly
  required for building and using LBANN.
* Added a run time dependency for both py-pytest and py-scipy so that
  they are activated in any environment.
* Added support for building LBANN, Hydrogen, and DiHydrogen with the
  IBM ESSL BLAS library.  This requires explicit identification of
  additional LAPACK libraries, since ESSL does not implement LAPACK, but
  is found by CMake.
* Fixed a bug in the LBANN dependency on OpenCV for Power architectures.
  The +powerpc variant is only required for GCC toolchains and causes
  Clang to break. Switched to only enabling when using %gcc on power.

4 files changed, 52 insertions, 7 deletions

diff --git a/var/spack/repos/builtin/packages/aluminum/package.py b/var/spack/repos/builtin/packages/aluminum/package.py
index 3c912d0652..894c8bca38 100644
--- a/var/spack/repos/builtin/packages/aluminum/package.py
+++ b/var/spack/repos/builtin/packages/aluminum/package.py
@@ -68,6 +68,10 @@ class Aluminum(CMakePackage, CudaPackage, ROCmPackage):
             '-DALUMINUM_ENABLE_ROCM:BOOL=%s' % ('+rocm' in spec)]
 
         if '+cuda' in spec:
+            if self.spec.satisfies('%clang'):
+                for flag in self.spec.compiler_flags['cxxflags']:
+                    if 'gcc-toolchain' in flag:
+                        args.append('-DCMAKE_CUDA_FLAGS=-Xcompiler={0}'.format(flag))
             if spec.satisfies('^cuda@11.0:'):
                 args.append('-DCMAKE_CUDA_STANDARD=17')
             else:
diff --git a/var/spack/repos/builtin/packages/dihydrogen/package.py b/var/spack/repos/builtin/packages/dihydrogen/package.py
index cd1c32ef04..f6b2b3b915 100644
--- a/var/spack/repos/builtin/packages/dihydrogen/package.py
+++ b/var/spack/repos/builtin/packages/dihydrogen/package.py
@@ -159,6 +159,10 @@ class Dihydrogen(CMakePackage, CudaPackage, ROCmPackage):
         ]
 
         if '+cuda' in spec:
+            if self.spec.satisfies('%clang'):
+                for flag in self.spec.compiler_flags['cxxflags']:
+                    if 'gcc-toolchain' in flag:
+                        args.append('-DCMAKE_CUDA_FLAGS=-Xcompiler={0}'.format(flag))
             if spec.satisfies('^cuda@11.0:'):
                 args.append('-DCMAKE_CUDA_STANDARD=17')
             else:
@@ -201,6 +205,14 @@ class Dihydrogen(CMakePackage, CudaPackage, ROCmPackage):
                     ' -g -fsized-deallocation -fPIC -std=c++17'.format(arch_str)
                 )
 
+        if self.spec.satisfies('^essl'):
+            # IF IBM ESSL is used it needs help finding the proper LAPACK libraries
+            args.extend([
+                '-DLAPACK_LIBRARIES=%s;-llapack;-lblas' %
+                ';'.join('-l{0}'.format(lib) for lib in self.spec['essl'].libs.names),
+                '-DBLAS_LIBRARIES=%s;-lblas' %
+                ';'.join('-l{0}'.format(lib) for lib in self.spec['essl'].libs.names)])
+
         return args
 
     def setup_build_environment(self, env):
diff --git a/var/spack/repos/builtin/packages/hydrogen/package.py b/var/spack/repos/builtin/packages/hydrogen/package.py
index 00b72ff97e..552f7f4b51 100644
--- a/var/spack/repos/builtin/packages/hydrogen/package.py
+++ b/var/spack/repos/builtin/packages/hydrogen/package.py
@@ -146,7 +146,7 @@ class Hydrogen(CMakePackage, CudaPackage, ROCmPackage):
         enable_gpu_fp16 = ('+cuda' in spec and '+half' in spec)
 
         args = [
-            '-DCMAKE_CXX_STANDARD=14',
+            '-DCMAKE_CXX_STANDARD=17',
             '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
             '-DCMAKE_INSTALL_MESSAGE:STRING=LAZY',
             '-DBUILD_SHARED_LIBS:BOOL=%s'      % ('+shared' in spec),
@@ -166,6 +166,10 @@ class Hydrogen(CMakePackage, CudaPackage, ROCmPackage):
         ]
 
         if '+cuda' in spec:
+            if self.spec.satisfies('%clang'):
+                for flag in self.spec.compiler_flags['cxxflags']:
+                    if 'gcc-toolchain' in flag:
+                        args.append('-DCMAKE_CUDA_FLAGS=-Xcompiler={0}'.format(flag))
             args.append('-DCMAKE_CUDA_STANDARD=14')
             archs = spec.variants['cuda_arch'].value
             if archs != 'none':
@@ -206,8 +210,12 @@ class Hydrogen(CMakePackage, CudaPackage, ROCmPackage):
         elif 'blas=accelerate' in spec:
             args.extend(['-DHydrogen_USE_ACCELERATE:BOOL=TRUE'])
         elif 'blas=essl' in spec:
+            # IF IBM ESSL is used it needs help finding the proper LAPACK libraries
             args.extend([
-                '-DHydrogen_USE_ESSL:BOOL=%s' % ('blas=essl' in spec)])
+                '-DLAPACK_LIBRARIES=%s;-llapack;-lblas' %
+                ';'.join('-l{0}'.format(lib) for lib in self.spec['essl'].libs.names),
+                '-DBLAS_LIBRARIES=%s;-lblas' %
+                ';'.join('-l{0}'.format(lib) for lib in self.spec['essl'].libs.names)])
 
         if '+omp_taskloops' in spec:
             args.extend([
diff --git a/var/spack/repos/builtin/packages/lbann/package.py b/var/spack/repos/builtin/packages/lbann/package.py
index ed36e37999..8fb70740bc 100644
--- a/var/spack/repos/builtin/packages/lbann/package.py
+++ b/var/spack/repos/builtin/packages/lbann/package.py
@@ -74,6 +74,7 @@ class Lbann(CMakePackage, CudaPackage, ROCmPackage):
     variant('pfe', default=True, description='Python Frontend for generating and launching models')
     variant('boost', default=False, description='Enable callbacks that use Boost libraries')
     variant('asan', default=False, description='Build with support for address-sanitizer')
+    variant('apps', default=True, description='Add python modules for standard LBANN applications')
 
     # LBANN benefits from high performance linkers, but passing these in as command
     # line options forces the linker flags to unnecessarily propagate to all
@@ -193,15 +194,16 @@ class Lbann(CMakePackage, CudaPackage, ROCmPackage):
                '+imgcodecs +imgproc +jpeg +png +tiff +fast-math ~cuda',
                when='+vision')
 
-    # Note that for Power systems we want the environment to add  +powerpc
-    depends_on('opencv@4.1.0: +powerpc', when='+vision arch=ppc64le:')
+    # Note that for Power systems we want the environment to add +powerpc
+    # When using a GCC compiler
+    depends_on('opencv@4.1.0: +powerpc', when='+vision %gcc arch=ppc64le:')
 
     depends_on('cnpy', when='+numpy')
     depends_on('nccl', when='@0.94:0.98.2 +cuda')
 
     depends_on('conduit@0.4.0: +hdf5~hdf5_compat', when='@0.94:0 +conduit')
     depends_on('conduit@0.5.0:0.6 +hdf5~hdf5_compat', when='@0.100:0.101 +conduit')
-    depends_on('conduit@0.6.0: +hdf5~hdf5_compat', when='@:0.90,0.99:')
+    depends_on('conduit@0.6.0: +hdf5~hdf5_compat~fortran~parmetis', when='@:0.90,0.99:')
 
     # LBANN can use Python in two modes 1) as part of an extensible framework
     # and 2) to drive the front end model creation and launch
@@ -215,6 +217,15 @@ class Lbann(CMakePackage, CudaPackage, ROCmPackage):
     extends("python", when='+pfe')
     depends_on('py-setuptools', type='build', when='+pfe')
     depends_on('py-argparse', type='run', when='@:0.90,0.99: +pfe ^python@:2.6,3.0:3.1')
+    depends_on('py-protobuf+cpp@3.10.0', type=('build', 'run'), when='@:0.90,0.99: +pfe')
+
+    # Add Python package dependencies to support applications in the LBANN repo
+    depends_on('py-numpy@1.16.0:', type=('build', 'run'), when='@:0.90,0.99: +pfe +apps')
+    depends_on('py-pytest', type=('test', 'run'), when='@:0.90,0.99: +pfe +apps')
+    depends_on('py-scipy', type=('test', 'run'), when='@:0.90,0.99: +pfe +apps')
+    depends_on('py-tqdm', type='run', when='@:0.90,0.99: +pfe +apps')
+
+    # Add common Python packages that are used for LBANN auxiliary tools
     depends_on('py-configparser', type='run', when='@:0.90,0.99: +pfe +extras')
     depends_on('py-graphviz@0.10.1:', type='run', when='@:0.90,0.99: +pfe +extras')
     depends_on('py-matplotlib@3.0.0:', type='run', when='@:0.90,0.99: +pfe +extras')
@@ -222,8 +233,6 @@ class Lbann(CMakePackage, CudaPackage, ROCmPackage):
     depends_on('py-onnx@1.3.0:', type='run', when='@:0.90,0.99: +pfe +extras')
     depends_on('py-pandas@0.24.1:', type='run', when='@:0.90,0.99: +pfe +extras')
     depends_on('py-texttable@1.4.0:', type='run', when='@:0.90,0.99: +pfe +extras')
-    depends_on('py-pytest', type='test', when='@:0.90,0.99: +pfe')
-    depends_on('py-protobuf+cpp@3.10.0', type=('build', 'run'), when='@:0.90,0.99: +pfe')
 
     depends_on('protobuf+shared@3.10.0', when='@:0.90,0.99:')
 
@@ -324,6 +333,10 @@ class Lbann(CMakePackage, CudaPackage, ROCmPackage):
             '-Dprotobuf_MODULE_COMPATIBLE=ON'])
 
         if '+cuda' in spec:
+            if self.spec.satisfies('%clang'):
+                for flag in self.spec.compiler_flags['cxxflags']:
+                    if 'gcc-toolchain' in flag:
+                        args.append('-DCMAKE_CUDA_FLAGS=-Xcompiler={0}'.format(flag))
             if spec.satisfies('^cuda@11.0:'):
                 args.append('-DCMAKE_CUDA_STANDARD=17')
             else:
@@ -410,6 +423,14 @@ class Lbann(CMakePackage, CudaPackage, ROCmPackage):
                         arch_str, cxxflags_str)
                 )
 
+        # IF IBM ESSL is used it needs help finding the proper LAPACK libraries
+        if self.spec.satisfies('^essl'):
+            args.extend([
+                '-DLAPACK_LIBRARIES=%s;-llapack;-lblas' %
+                ';'.join('-l{0}'.format(lib) for lib in self.spec['essl'].libs.names),
+                '-DBLAS_LIBRARIES=%s;-lblas' %
+                ';'.join('-l{0}'.format(lib) for lib in self.spec['essl'].libs.names)])
+
         return args
 
     @when('@0.91:0.93')