From 01b277ad0fe15e6f42f9f403126da5de68994737 Mon Sep 17 00:00:00 2001 From: Sergey Kosukhin Date: Wed, 17 Jun 2020 21:41:24 +0200 Subject: openmpi: options for local transport (#16831) --- var/spack/repos/builtin/packages/fca/package.py | 31 +++++ var/spack/repos/builtin/packages/hcoll/package.py | 31 +++++ var/spack/repos/builtin/packages/knem/package.py | 46 +++++++ .../repos/builtin/packages/openmpi/package.py | 140 ++++++++++++++++----- var/spack/repos/builtin/packages/xpmem/package.py | 91 ++++++++++++++ 5 files changed, 309 insertions(+), 30 deletions(-) create mode 100644 var/spack/repos/builtin/packages/fca/package.py create mode 100644 var/spack/repos/builtin/packages/hcoll/package.py create mode 100644 var/spack/repos/builtin/packages/knem/package.py create mode 100644 var/spack/repos/builtin/packages/xpmem/package.py (limited to 'var') diff --git a/var/spack/repos/builtin/packages/fca/package.py b/var/spack/repos/builtin/packages/fca/package.py new file mode 100644 index 0000000000..ca91a43b4e --- /dev/null +++ b/var/spack/repos/builtin/packages/fca/package.py @@ -0,0 +1,31 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class Fca(Package): + """Legacy interface for Mellanox Fabric Collective Accelerator (FCA). FCA + is a MPI-integrated software package that utilizes CORE-Direct technology + for implementing the MPI collective communications.""" + + homepage = 'https://www.mellanox.com/products/fca' + has_code = False + + version('2.5.2431') + + # FCA needs to be added as an external package to SPACK. For this, the + # config file packages.yaml needs to be adjusted: + # + # fca: + # version: [2.5.2431] + # paths: + # fca@2.5.2431: /opt/mellanox/fca (path to your FCA installation) + # buildable: False + + def install(self, spec, prefix): + raise InstallError( + self.spec.format('{name} is not installable, you need to specify ' + 'it as an external package in packages.yaml')) diff --git a/var/spack/repos/builtin/packages/hcoll/package.py b/var/spack/repos/builtin/packages/hcoll/package.py new file mode 100644 index 0000000000..f39e2e7596 --- /dev/null +++ b/var/spack/repos/builtin/packages/hcoll/package.py @@ -0,0 +1,31 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class Hcoll(Package): + """Modern interface for Mellanox Fabric Collective Accelerator (FCA). FCA + is a MPI-integrated software package that utilizes CORE-Direct technology + for implementing the MPI collective communications.""" + + homepage = 'https://www.mellanox.com/products/fca' + has_code = False + + version('3.9.1927') + + # HCOLL needs to be added as an external package to SPACK. For this, the + # config file packages.yaml needs to be adjusted: + # + # hcoll: + # version: [3.9.1927] + # paths: + # hcoll@3.9.1927: /opt/mellanox/hcoll (path to your HCOLL installation) + # buildable: False + + def install(self, spec, prefix): + raise InstallError( + self.spec.format('{name} is not installable, you need to specify ' + 'it as an external package in packages.yaml')) diff --git a/var/spack/repos/builtin/packages/knem/package.py b/var/spack/repos/builtin/packages/knem/package.py new file mode 100644 index 0000000000..473a178319 --- /dev/null +++ b/var/spack/repos/builtin/packages/knem/package.py @@ -0,0 +1,46 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class Knem(AutotoolsPackage): + """KNEM is a Linux kernel module enabling high-performance intra-node MPI + communication for large messages.""" + + homepage = "http://knem.gforge.inria.fr" + url = "http://gforge.inria.fr/frs/download.php/37186/knem-1.1.3.tar.gz" + list_url = "http://knem.gforge.inria.fr/download" + + maintainers = ['skosukhin'] + + version('1.1.3', sha256='50d3c4a20c140108b8ce47aaafd0ade0927d6f507e1b5cc690dd6bddeef30f60') + + variant('hwloc', default=True, + description='Enable hwloc in the user-space tools') + + depends_on('hwloc', when='+hwloc') + depends_on('pkgconfig', type='build', when='+hwloc') + + # The support for hwloc was added in 0.9.1: + conflicts('+hwloc', when='@:0.9.0') + + # Ideally, we should list all non-Linux-based platforms here: + conflicts('platform=darwin') + + # All compilers except for gcc are in conflict: + for __compiler in spack.compilers.supported_compilers(): + if __compiler != 'gcc': + conflicts('%{0}'.format(__compiler), + msg='Linux kernel module must be compiled with gcc') + + @run_before('build') + def override_kernel_compiler(self): + # Override the compiler for kernel module source files. We need + # this additional argument for all installation phases. + make.add_default_arg('CC={0}'.format(spack_cc)) + + def configure_args(self): + return self.enable_or_disable('hwloc') diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index ee2f76ea19..4065e85a59 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -182,19 +182,25 @@ class Openmpi(AutotoolsPackage): variant( 'fabrics', values=disjoint_sets( - ('auto',), ('psm', 'psm2', 'verbs', 'mxm', 'ucx', 'libfabric') + ('auto',), + ('psm', 'psm2', 'verbs', + 'mxm', 'ucx', 'ofi', + 'fca', 'hcoll', + 'xpmem', 'cma', 'knem') # shared memory transports ).with_non_feature_values('auto', 'none'), description="List of fabrics that are enabled; " - "'auto' lets openmpi determine", + "'auto' lets openmpi determine", ) variant( 'schedulers', values=disjoint_sets( - ('auto',), ('alps', 'lsf', 'tm', 'slurm', 'sge', 'loadleveler') + ('auto',), + ('alps', 'lsf', 'tm', + 'slurm', 'sge', 'loadleveler') ).with_non_feature_values('auto', 'none'), description="List of schedulers for which support is enabled; " - "'auto' lets openmpi determine", + "'auto' lets openmpi determine", ) # Additional support options @@ -259,39 +265,59 @@ class Openmpi(AutotoolsPackage): depends_on('zlib', when='@3.0.0:') depends_on('valgrind~mpi', when='+memchecker') - depends_on('ucx', when='fabrics=ucx') - depends_on('ucx +thread_multiple', when='fabrics=ucx +thread_multiple') - depends_on('ucx +thread_multiple', when='@3.0.0: fabrics=ucx') - depends_on('libfabric', when='fabrics=libfabric') depends_on('opa-psm2', when='fabrics=psm2') + depends_on('rdma-core', when='fabrics=verbs') depends_on('mxm', when='fabrics=mxm') depends_on('binutils+libiberty', when='fabrics=mxm') - depends_on('rdma-core', when='fabrics=verbs') + depends_on('ucx', when='fabrics=ucx') + depends_on('ucx +thread_multiple', when='fabrics=ucx +thread_multiple') + depends_on('ucx +thread_multiple', when='@3.0.0: fabrics=ucx') + depends_on('libfabric', when='fabrics=ofi') + depends_on('fca', when='fabrics=fca') + depends_on('hcoll', when='fabrics=hcoll') + depends_on('xpmem', when='fabrics=xpmem') + depends_on('knem', when='fabrics=knem') - depends_on('slurm', when='schedulers=slurm') depends_on('lsf', when='schedulers=lsf') depends_on('openpbs', when='schedulers=tm') + depends_on('slurm', when='schedulers=slurm') + + # CUDA support was added in 1.7 + conflicts('+cuda', when='@:1.6') + # PMI support was added in 1.5.5 + conflicts('+pmi', when='@:1.5.4') + # RPATH support in the wrappers was added in 1.7.4 + conflicts('+wrapper-rpath', when='@:1.7.3') + + conflicts('+cxx', when='@5:', + msg='C++ MPI bindings are removed in 5.0.X release') + conflicts('+cxx_exceptions', when='@5:', + msg='C++ exceptions are removed in 5.0.X release') + + # PSM2 support was added in 1.10.0 + conflicts('fabrics=psm2', when='@:1.8') + # MXM support was added in 1.5.4 + conflicts('fabrics=mxm', when='@:1.5.3') + # libfabric (OFI) support was added in 1.10.0 + conflicts('fabrics=ofi', when='@:1.8') + # fca support was added in 1.5.0 and removed in 5.0.0 + conflicts('fabrics=fca', when='@:1.4,5:') + # hcoll support was added in 1.7.3: + conflicts('fabrics=hcoll', when='@:1.7.2') + # xpmem support was added in 1.7 + conflicts('fabrics=xpmem', when='@:1.6') + # cma support was added in 1.7 + conflicts('fabrics=cma', when='@:1.6') + # knem support was added in 1.5 + conflicts('fabrics=knem', when='@:1.4') - conflicts('+cuda', when='@:1.6') # CUDA support was added in 1.7 - conflicts('fabrics=psm2', when='@:1.8') # PSM2 support was added in 1.10.0 - conflicts('fabrics=mxm', when='@:1.5.3') # MXM support was added in 1.5.4 - conflicts('+pmi', when='@:1.5.4') # PMI support was added in 1.5.5 conflicts('schedulers=slurm ~pmi', when='@1.5.4:', msg='+pmi is required for openmpi(>=1.5.5) to work with SLURM.') conflicts('schedulers=loadleveler', when='@3.0.0:', msg='The loadleveler scheduler is not supported with ' 'openmpi(>=3.0.0).') - conflicts('+cxx', when='@5:', - msg='C++ MPI bindings are removed in 5.0.X release') - conflicts('+cxx_exceptions', when='@5:', - msg='C++ exceptions are removed in 5.0.X release') filter_compiler_wrappers('openmpi/*-wrapper-data*', relative_root='share') - conflicts('fabrics=libfabric', when='@:1.8') # libfabric support was added in 1.10.0 - # It may be worth considering making libfabric an exclusive fabrics choice - - # RPATH support in the wrappers was added in 1.7.4 - conflicts('+wrapper-rpath', when='@:1.7.3') def url_for_version(self, version): url = "http://www.open-mpi.org/software/ompi/v{0}/downloads/openmpi-{1}.tar.bz2" @@ -343,6 +369,17 @@ class Openmpi(AutotoolsPackage): join_path(self.prefix.lib, 'libmpi.{0}'.format(dso_suffix)) ] + # Most of the following with_or_without methods might seem redundant + # because Spack compiler wrapper adds the required -I and -L flags, which + # is enough for the configure script to find them. However, we also need + # the flags in Libtool (lib/*.la) and pkg-config (lib/pkgconfig/*.pc). + # Therefore, we pass the prefixes explicitly. + + def with_or_without_psm2(self, activated): + if not activated: + return '--without-psm2' + return '--with-psm2={0}'.format(self.spec['opa-psm2'].prefix) + def with_or_without_verbs(self, activated): # Up through version 1.6, this option was named --with-openib. # In version 1.7, it was renamed to be --with-verbs. @@ -351,16 +388,54 @@ class Openmpi(AutotoolsPackage): return '--without-{0}'.format(opt) return '--with-{0}={1}'.format(opt, self.spec['rdma-core'].prefix) + def with_or_without_mxm(self, activated): + if not activated: + return '--without-mxm' + return '--with-mxm={0}'.format(self.spec['mxm'].prefix) + + def with_or_without_ucx(self, activated): + if not activated: + return '--without-ucx' + return '--with-ucx={0}'.format(self.spec['ucx'].prefix) + + def with_or_without_ofi(self, activated): + # Up through version 3.0.3 this option was name --with-libfabric. + # In version 3.0.4, the old name was deprecated in favor of --with-ofi. + opt = 'ofi' if self.spec.satisfies('@3.0.4:') else 'libfabric' + if not activated: + return '--without-{0}'.format(opt) + return '--with-{0}={1}'.format(opt, self.spec['libfabric'].prefix) + + def with_or_without_fca(self, activated): + if not activated: + return '--without-fca' + return '--with-fca={0}'.format(self.spec['fca'].prefix) + + def with_or_without_hcoll(self, activated): + if not activated: + return '--without-hcoll' + return '--with-hcoll={0}'.format(self.spec['hcoll'].prefix) + + def with_or_without_xpmem(self, activated): + if not activated: + return '--without-xpmem' + return '--with-xpmem={0}'.format(self.spec['xpmem'].prefix) + + def with_or_without_knem(self, activated): + if not activated: + return '--without-knem' + return '--with-knem={0}'.format(self.spec['knem'].prefix) + + def with_or_without_lsf(self, activated): + if not activated: + return '--without-lsf' + return '--with-lsf={0}'.format(self.spec['lsf'].prefix) + def with_or_without_tm(self, activated): if not activated: return '--without-tm' return '--with-tm={0}'.format(self.spec['openpbs'].prefix) - def with_or_without_psm2(self, activated): - if not activated: - return '--without-psm2' - return '--with-psm2={0}'.format(self.spec['opa-psm2'].prefix) - @run_before('autoreconf') def die_without_fortran(self): # Until we can pass variants such as +fortran through virtual @@ -433,8 +508,13 @@ class Openmpi(AutotoolsPackage): # Fabrics if 'fabrics=auto' not in spec: - config_args.extend(self.with_or_without('fabrics', - activation_value='prefix')) + config_args.extend(self.with_or_without('fabrics')) + + if spec.satisfies('@2.0.0'): + if 'fabrics=xpmem' in spec and 'platform=cray' in spec: + config_args.append('--with-cray-xpmem') + else: + config_args.append('--without-cray-xpmem') # Schedulers if 'schedulers=auto' not in spec: diff --git a/var/spack/repos/builtin/packages/xpmem/package.py b/var/spack/repos/builtin/packages/xpmem/package.py new file mode 100644 index 0000000000..90fa58df06 --- /dev/null +++ b/var/spack/repos/builtin/packages/xpmem/package.py @@ -0,0 +1,91 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class Xpmem(AutotoolsPackage): + """XPMEM is a Linux kernel module that enables a process to map the memory + of another process into its virtual address space.""" + + # The README file of the repository says that the development was + # transferred to a new repository on GitLab: http://gitlab.com/hjelmn/xpmem + # However, it looks like that the repository on GitHub has a more recent + # version of the codebase. + homepage = "https://github.com/hjelmn/xpmem" + url = "https://github.com/hjelmn/xpmem/archive/v2.6.3.tar.gz" + git = "https://github.com/hjelmn/xpmem.git" + + maintainers = ['skosukhin'] + + version('master', branch='master') + + # Versions starting 2.6.4 are neither tagged nor released in the repo + # (the choice of commits is based on the commit history of + # 'kernel/xpmem_private.h'): + version('2.6.5', commit='4efeed9cbaabe971f3766d67cb108e2c3316d4b8') + version('2.6.4', commit='522054850e4d1479d69f50f7190d1548bf9749fd') + + # Released versions: + version('2.6.3', sha256='ee239a32269f33234cdbdb94db29c12287862934c0784328d34aff82a9fa8b54') + version('2.6.2', sha256='2c1a93b4cb20ed73c2093435a7afec513e0e797aa1e49d4d964cc6bdae89d65b') + + variant('kernel-module', default=True, + description='Enable building the kernel module') + + depends_on('autoconf', type='build') + depends_on('automake', type='build') + depends_on('libtool', type='build') + depends_on('m4', type='build') + + # It will become possible to disable the kernel module only starting 2.6.6: + # https://github.com/hjelmn/xpmem/pull/24 + conflicts('~kernel-module', when='@:2.6.5') + + # Ideally, we should list all non-Linux-based platforms here: + conflicts('+kernel-module', when='platform=darwin') + + # All compilers except for gcc are in conflict with +kernel-module: + for __compiler in spack.compilers.supported_compilers(): + if __compiler != 'gcc': + conflicts('+kernel-module', + when='%{0}'.format(__compiler), + msg='Linux kernel module must be compiled with gcc') + + def autoreconf(self, spec, prefix): + Executable('./autogen.sh')() + + @run_before('build') + def override_kernel_compiler(self): + # Override the compiler for kernel module source files. We need + # this additional argument for all installation phases. + if '+kernel-module' in self.spec: + make.add_default_arg('CC={0}'.format(spack_cc)) + + def configure_args(self): + args = [] + + if '~kernel-module' in self.spec: + # The kernel module is enabled by default. An attempt of explicit + # enabling with '--enable-kernel-module' disables the module. + args.extend('--disable-kernel-module') + + if self.spec.satisfies('@:2.6.5'): + fmt = self.spec.format + # The following arguments will not be needed starting 2.6.6: + # https://github.com/hjelmn/xpmem/pull/18 + args.extend([ + fmt('--with-default-prefix={prefix}'), + fmt('--with-module={prefix.share}/Modules/{name}/{version}')]) + + return args + + @when('@:2.6.5') + def install(self, spec, prefix): + with working_dir(self.build_directory): + # Override the hardcoded prefix for 'cray-xpmem.conf' + make('ldsoconfdir={0}'.format( + self.spec.prefix.etc.join('ld.so.conf.d')), + *self.install_targets) -- cgit v1.2.3-60-g2f50