openmpi: options for local transport (#16831)

5 files changed, 309 insertions, 30 deletions

diff --git a/var/spack/repos/builtin/packages/fca/package.py b/var/spack/repos/builtin/packages/fca/package.py
new file mode 100644
index 0000000000..ca91a43b4e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/fca/package.py
@@ -0,0 +1,31 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Fca(Package):
+    """Legacy interface for Mellanox Fabric Collective Accelerator (FCA). FCA
+    is a MPI-integrated software package that utilizes CORE-Direct technology
+    for implementing the MPI collective communications."""
+
+    homepage = 'https://www.mellanox.com/products/fca'
+    has_code = False
+
+    version('2.5.2431')
+
+    # FCA needs to be added as an external package to SPACK. For this, the
+    # config file packages.yaml needs to be adjusted:
+    #
+    # fca:
+    #   version: [2.5.2431]
+    #   paths:
+    #     fca@2.5.2431: /opt/mellanox/fca (path to your FCA installation)
+    #   buildable: False
+
+    def install(self, spec, prefix):
+        raise InstallError(
+            self.spec.format('{name} is not installable, you need to specify '
+                             'it as an external package in packages.yaml'))
diff --git a/var/spack/repos/builtin/packages/hcoll/package.py b/var/spack/repos/builtin/packages/hcoll/package.py
new file mode 100644
index 0000000000..f39e2e7596
--- /dev/null
+++ b/var/spack/repos/builtin/packages/hcoll/package.py
@@ -0,0 +1,31 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Hcoll(Package):
+    """Modern interface for Mellanox Fabric Collective Accelerator (FCA). FCA
+    is a MPI-integrated software package that utilizes CORE-Direct technology
+    for implementing the MPI collective communications."""
+
+    homepage = 'https://www.mellanox.com/products/fca'
+    has_code = False
+
+    version('3.9.1927')
+
+    # HCOLL needs to be added as an external package to SPACK. For this, the
+    # config file packages.yaml needs to be adjusted:
+    #
+    # hcoll:
+    #   version: [3.9.1927]
+    #   paths:
+    #     hcoll@3.9.1927: /opt/mellanox/hcoll (path to your HCOLL installation)
+    #   buildable: False
+
+    def install(self, spec, prefix):
+        raise InstallError(
+            self.spec.format('{name} is not installable, you need to specify '
+                             'it as an external package in packages.yaml'))
diff --git a/var/spack/repos/builtin/packages/knem/package.py b/var/spack/repos/builtin/packages/knem/package.py
new file mode 100644
index 0000000000..473a178319
--- /dev/null
+++ b/var/spack/repos/builtin/packages/knem/package.py
@@ -0,0 +1,46 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Knem(AutotoolsPackage):
+    """KNEM is a Linux kernel module enabling high-performance intra-node MPI
+    communication for large messages."""
+
+    homepage = "http://knem.gforge.inria.fr"
+    url = "http://gforge.inria.fr/frs/download.php/37186/knem-1.1.3.tar.gz"
+    list_url = "http://knem.gforge.inria.fr/download"
+
+    maintainers = ['skosukhin']
+
+    version('1.1.3', sha256='50d3c4a20c140108b8ce47aaafd0ade0927d6f507e1b5cc690dd6bddeef30f60')
+
+    variant('hwloc', default=True,
+            description='Enable hwloc in the user-space tools')
+
+    depends_on('hwloc', when='+hwloc')
+    depends_on('pkgconfig', type='build', when='+hwloc')
+
+    # The support for hwloc was added in 0.9.1:
+    conflicts('+hwloc', when='@:0.9.0')
+
+    # Ideally, we should list all non-Linux-based platforms here:
+    conflicts('platform=darwin')
+
+    # All compilers except for gcc are in conflict:
+    for __compiler in spack.compilers.supported_compilers():
+        if __compiler != 'gcc':
+            conflicts('%{0}'.format(__compiler),
+                      msg='Linux kernel module must be compiled with gcc')
+
+    @run_before('build')
+    def override_kernel_compiler(self):
+        # Override the compiler for kernel module source files. We need
+        # this additional argument for all installation phases.
+        make.add_default_arg('CC={0}'.format(spack_cc))
+
+    def configure_args(self):
+        return self.enable_or_disable('hwloc')
diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py
index ee2f76ea19..4065e85a59 100644
--- a/var/spack/repos/builtin/packages/openmpi/package.py
+++ b/var/spack/repos/builtin/packages/openmpi/package.py
@@ -182,19 +182,25 @@ class Openmpi(AutotoolsPackage):
     variant(
         'fabrics',
         values=disjoint_sets(
-            ('auto',), ('psm', 'psm2', 'verbs', 'mxm', 'ucx', 'libfabric')
+            ('auto',),
+            ('psm', 'psm2', 'verbs',
+             'mxm', 'ucx', 'ofi',
+             'fca', 'hcoll',
+             'xpmem', 'cma', 'knem')  # shared memory transports
         ).with_non_feature_values('auto', 'none'),
         description="List of fabrics that are enabled; "
-        "'auto' lets openmpi determine",
+                    "'auto' lets openmpi determine",
     )
 
     variant(
         'schedulers',
         values=disjoint_sets(
-            ('auto',), ('alps', 'lsf', 'tm', 'slurm', 'sge', 'loadleveler')
+            ('auto',),
+            ('alps', 'lsf', 'tm',
+             'slurm', 'sge', 'loadleveler')
         ).with_non_feature_values('auto', 'none'),
         description="List of schedulers for which support is enabled; "
-        "'auto' lets openmpi determine",
+                    "'auto' lets openmpi determine",
     )
 
     # Additional support options
@@ -259,39 +265,59 @@ class Openmpi(AutotoolsPackage):
     depends_on('zlib', when='@3.0.0:')
     depends_on('valgrind~mpi', when='+memchecker')
 
-    depends_on('ucx', when='fabrics=ucx')
-    depends_on('ucx +thread_multiple', when='fabrics=ucx +thread_multiple')
-    depends_on('ucx +thread_multiple', when='@3.0.0: fabrics=ucx')
-    depends_on('libfabric', when='fabrics=libfabric')
     depends_on('opa-psm2', when='fabrics=psm2')
+    depends_on('rdma-core', when='fabrics=verbs')
     depends_on('mxm', when='fabrics=mxm')
     depends_on('binutils+libiberty', when='fabrics=mxm')
-    depends_on('rdma-core', when='fabrics=verbs')
+    depends_on('ucx', when='fabrics=ucx')
+    depends_on('ucx +thread_multiple', when='fabrics=ucx +thread_multiple')
+    depends_on('ucx +thread_multiple', when='@3.0.0: fabrics=ucx')
+    depends_on('libfabric', when='fabrics=ofi')
+    depends_on('fca', when='fabrics=fca')
+    depends_on('hcoll', when='fabrics=hcoll')
+    depends_on('xpmem', when='fabrics=xpmem')
+    depends_on('knem', when='fabrics=knem')
 
-    depends_on('slurm', when='schedulers=slurm')
     depends_on('lsf', when='schedulers=lsf')
     depends_on('openpbs', when='schedulers=tm')
+    depends_on('slurm', when='schedulers=slurm')
+
+    # CUDA support was added in 1.7
+    conflicts('+cuda', when='@:1.6')
+    # PMI support was added in 1.5.5
+    conflicts('+pmi', when='@:1.5.4')
+    # RPATH support in the wrappers was added in 1.7.4
+    conflicts('+wrapper-rpath', when='@:1.7.3')
+
+    conflicts('+cxx', when='@5:',
+              msg='C++ MPI bindings are removed in 5.0.X release')
+    conflicts('+cxx_exceptions', when='@5:',
+              msg='C++ exceptions are removed in 5.0.X release')
+
+    # PSM2 support was added in 1.10.0
+    conflicts('fabrics=psm2', when='@:1.8')
+    # MXM support was added in 1.5.4
+    conflicts('fabrics=mxm', when='@:1.5.3')
+    # libfabric (OFI) support was added in 1.10.0
+    conflicts('fabrics=ofi', when='@:1.8')
+    # fca support was added in 1.5.0 and removed in 5.0.0
+    conflicts('fabrics=fca', when='@:1.4,5:')
+    # hcoll support was added in 1.7.3:
+    conflicts('fabrics=hcoll', when='@:1.7.2')
+    # xpmem support was added in 1.7
+    conflicts('fabrics=xpmem', when='@:1.6')
+    # cma support was added in 1.7
+    conflicts('fabrics=cma', when='@:1.6')
+    # knem support was added in 1.5
+    conflicts('fabrics=knem', when='@:1.4')
 
-    conflicts('+cuda', when='@:1.6')  # CUDA support was added in 1.7
-    conflicts('fabrics=psm2', when='@:1.8')  # PSM2 support was added in 1.10.0
-    conflicts('fabrics=mxm', when='@:1.5.3')  # MXM support was added in 1.5.4
-    conflicts('+pmi', when='@:1.5.4')  # PMI support was added in 1.5.5
     conflicts('schedulers=slurm ~pmi', when='@1.5.4:',
               msg='+pmi is required for openmpi(>=1.5.5) to work with SLURM.')
     conflicts('schedulers=loadleveler', when='@3.0.0:',
               msg='The loadleveler scheduler is not supported with '
               'openmpi(>=3.0.0).')
-    conflicts('+cxx', when='@5:',
-              msg='C++ MPI bindings are removed in 5.0.X release')
-    conflicts('+cxx_exceptions', when='@5:',
-              msg='C++ exceptions are removed in 5.0.X release')
 
     filter_compiler_wrappers('openmpi/*-wrapper-data*', relative_root='share')
-    conflicts('fabrics=libfabric', when='@:1.8')  # libfabric support was added in 1.10.0
-    # It may be worth considering making libfabric an exclusive fabrics choice
-
-    # RPATH support in the wrappers was added in 1.7.4
-    conflicts('+wrapper-rpath', when='@:1.7.3')
 
     def url_for_version(self, version):
         url = "http://www.open-mpi.org/software/ompi/v{0}/downloads/openmpi-{1}.tar.bz2"
@@ -343,6 +369,17 @@ class Openmpi(AutotoolsPackage):
             join_path(self.prefix.lib, 'libmpi.{0}'.format(dso_suffix))
         ]
 
+    # Most of the following with_or_without methods might seem redundant
+    # because Spack compiler wrapper adds the required -I and -L flags, which
+    # is enough for the configure script to find them. However, we also need
+    # the flags in Libtool (lib/*.la) and pkg-config (lib/pkgconfig/*.pc).
+    # Therefore, we pass the prefixes explicitly.
+
+    def with_or_without_psm2(self, activated):
+        if not activated:
+            return '--without-psm2'
+        return '--with-psm2={0}'.format(self.spec['opa-psm2'].prefix)
+
     def with_or_without_verbs(self, activated):
         # Up through version 1.6, this option was named --with-openib.
         # In version 1.7, it was renamed to be --with-verbs.
@@ -351,16 +388,54 @@ class Openmpi(AutotoolsPackage):
             return '--without-{0}'.format(opt)
         return '--with-{0}={1}'.format(opt, self.spec['rdma-core'].prefix)
 
+    def with_or_without_mxm(self, activated):
+        if not activated:
+            return '--without-mxm'
+        return '--with-mxm={0}'.format(self.spec['mxm'].prefix)
+
+    def with_or_without_ucx(self, activated):
+        if not activated:
+            return '--without-ucx'
+        return '--with-ucx={0}'.format(self.spec['ucx'].prefix)
+
+    def with_or_without_ofi(self, activated):
+        # Up through version 3.0.3 this option was name --with-libfabric.
+        # In version 3.0.4, the old name was deprecated in favor of --with-ofi.
+        opt = 'ofi' if self.spec.satisfies('@3.0.4:') else 'libfabric'
+        if not activated:
+            return '--without-{0}'.format(opt)
+        return '--with-{0}={1}'.format(opt, self.spec['libfabric'].prefix)
+
+    def with_or_without_fca(self, activated):
+        if not activated:
+            return '--without-fca'
+        return '--with-fca={0}'.format(self.spec['fca'].prefix)
+
+    def with_or_without_hcoll(self, activated):
+        if not activated:
+            return '--without-hcoll'
+        return '--with-hcoll={0}'.format(self.spec['hcoll'].prefix)
+
+    def with_or_without_xpmem(self, activated):
+        if not activated:
+            return '--without-xpmem'
+        return '--with-xpmem={0}'.format(self.spec['xpmem'].prefix)
+
+    def with_or_without_knem(self, activated):
+        if not activated:
+            return '--without-knem'
+        return '--with-knem={0}'.format(self.spec['knem'].prefix)
+
+    def with_or_without_lsf(self, activated):
+        if not activated:
+            return '--without-lsf'
+        return '--with-lsf={0}'.format(self.spec['lsf'].prefix)
+
     def with_or_without_tm(self, activated):
         if not activated:
             return '--without-tm'
         return '--with-tm={0}'.format(self.spec['openpbs'].prefix)
 
-    def with_or_without_psm2(self, activated):
-        if not activated:
-            return '--without-psm2'
-        return '--with-psm2={0}'.format(self.spec['opa-psm2'].prefix)
-
     @run_before('autoreconf')
     def die_without_fortran(self):
         # Until we can pass variants such as +fortran through virtual
@@ -433,8 +508,13 @@ class Openmpi(AutotoolsPackage):
 
         # Fabrics
         if 'fabrics=auto' not in spec:
-            config_args.extend(self.with_or_without('fabrics',
-                                                    activation_value='prefix'))
+            config_args.extend(self.with_or_without('fabrics'))
+
+        if spec.satisfies('@2.0.0'):
+            if 'fabrics=xpmem' in spec and 'platform=cray' in spec:
+                config_args.append('--with-cray-xpmem')
+            else:
+                config_args.append('--without-cray-xpmem')
 
         # Schedulers
         if 'schedulers=auto' not in spec:
diff --git a/var/spack/repos/builtin/packages/xpmem/package.py b/var/spack/repos/builtin/packages/xpmem/package.py
new file mode 100644
index 0000000000..90fa58df06
--- /dev/null
+++ b/var/spack/repos/builtin/packages/xpmem/package.py
@@ -0,0 +1,91 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Xpmem(AutotoolsPackage):
+    """XPMEM is a Linux kernel module that enables a process to map the memory
+    of another process into its virtual address space."""
+
+    # The README file of the repository says that the development was
+    # transferred to a new repository on GitLab: http://gitlab.com/hjelmn/xpmem
+    # However, it looks like that the repository on GitHub has a more recent
+    # version of the codebase.
+    homepage = "https://github.com/hjelmn/xpmem"
+    url = "https://github.com/hjelmn/xpmem/archive/v2.6.3.tar.gz"
+    git = "https://github.com/hjelmn/xpmem.git"
+
+    maintainers = ['skosukhin']
+
+    version('master', branch='master')
+
+    # Versions starting 2.6.4 are neither tagged nor released in the repo
+    # (the choice of commits is based on the commit history of
+    # 'kernel/xpmem_private.h'):
+    version('2.6.5', commit='4efeed9cbaabe971f3766d67cb108e2c3316d4b8')
+    version('2.6.4', commit='522054850e4d1479d69f50f7190d1548bf9749fd')
+
+    # Released versions:
+    version('2.6.3', sha256='ee239a32269f33234cdbdb94db29c12287862934c0784328d34aff82a9fa8b54')
+    version('2.6.2', sha256='2c1a93b4cb20ed73c2093435a7afec513e0e797aa1e49d4d964cc6bdae89d65b')
+
+    variant('kernel-module', default=True,
+            description='Enable building the kernel module')
+
+    depends_on('autoconf', type='build')
+    depends_on('automake', type='build')
+    depends_on('libtool',  type='build')
+    depends_on('m4',       type='build')
+
+    # It will become possible to disable the kernel module only starting 2.6.6:
+    # https://github.com/hjelmn/xpmem/pull/24
+    conflicts('~kernel-module', when='@:2.6.5')
+
+    # Ideally, we should list all non-Linux-based platforms here:
+    conflicts('+kernel-module', when='platform=darwin')
+
+    # All compilers except for gcc are in conflict with +kernel-module:
+    for __compiler in spack.compilers.supported_compilers():
+        if __compiler != 'gcc':
+            conflicts('+kernel-module',
+                      when='%{0}'.format(__compiler),
+                      msg='Linux kernel module must be compiled with gcc')
+
+    def autoreconf(self, spec, prefix):
+        Executable('./autogen.sh')()
+
+    @run_before('build')
+    def override_kernel_compiler(self):
+        # Override the compiler for kernel module source files. We need
+        # this additional argument for all installation phases.
+        if '+kernel-module' in self.spec:
+            make.add_default_arg('CC={0}'.format(spack_cc))
+
+    def configure_args(self):
+        args = []
+
+        if '~kernel-module' in self.spec:
+            # The kernel module is enabled by default. An attempt of explicit
+            # enabling with '--enable-kernel-module' disables the module.
+            args.extend('--disable-kernel-module')
+
+        if self.spec.satisfies('@:2.6.5'):
+            fmt = self.spec.format
+            # The following arguments will not be needed starting 2.6.6:
+            # https://github.com/hjelmn/xpmem/pull/18
+            args.extend([
+                fmt('--with-default-prefix={prefix}'),
+                fmt('--with-module={prefix.share}/Modules/{name}/{version}')])
+
+        return args
+
+    @when('@:2.6.5')
+    def install(self, spec, prefix):
+        with working_dir(self.build_directory):
+            # Override the hardcoded prefix for 'cray-xpmem.conf'
+            make('ldsoconfdir={0}'.format(
+                self.spec.prefix.etc.join('ld.so.conf.d')),
+                *self.install_targets)