From 6ececa697673528df404a3f3f0e32b4eaf83b6b7 Mon Sep 17 00:00:00 2001 From: Nithin Senthil Kumar Date: Sun, 16 Aug 2020 09:56:22 -0400 Subject: New packages Mvapich2x and Mvapich2-GDR (#17883) * Adding new packages Mvapich2x and Mvapich2-GDR which can be installed only via binary mirrors * Added docstring descriptions to both packages * Removed variant wrapper for cuda dependencies * Fixed multiple flake8 errors * Updated APIs to pass unit tests * Updated APIs for MVAPICH2-X package and fixed flake8 warnings for MVAPICH2-GDR * Changed url back to single line * Removed extra parantesis around URL string Co-authored-by: nithintsk --- .../repos/builtin/packages/mvapich2-gdr/package.py | 197 ++++++++++++++++++ .../repos/builtin/packages/mvapich2x/package.py | 227 +++++++++++++++++++++ 2 files changed, 424 insertions(+) create mode 100755 var/spack/repos/builtin/packages/mvapich2-gdr/package.py create mode 100755 var/spack/repos/builtin/packages/mvapich2x/package.py diff --git a/var/spack/repos/builtin/packages/mvapich2-gdr/package.py b/var/spack/repos/builtin/packages/mvapich2-gdr/package.py new file mode 100755 index 0000000000..1a0fe8def6 --- /dev/null +++ b/var/spack/repos/builtin/packages/mvapich2-gdr/package.py @@ -0,0 +1,197 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os.path +import sys + + +class Mvapich2Gdr(AutotoolsPackage): + """MVAPICH2-GDR is an optimized version of the MVAPICH2 MPI library for + GPU-enabled HPC and Deep Learning Applications. MVAPICH2-GDR is not + installable from source and is only available through a binary mirror. + If you do not find the binary you're looking for, send us an email at + mvapich@cse.ohio-state.edu. The binary mirror url is: + http://mvapich.cse.ohio-state.edu:8080/download/mvapich/spack-mirror/mvapich2-gdr/ + """ + + homepage = 'http://mvapich.cse.ohio-state.edu' + url = 'http://mvapich.cse.ohio-state.edu:8080/download/mvapich/spack-mirror/mvapich2-gdr/mvapich2-gdr-2.3.4.tar.gz' + + maintainers = ['nithintsk', 'harisubramoni'] + + version('2.3.4', sha256='ed78101e6bb807e979213006ee5f20ff466369b01f96b6d1cf0c471baf7e35aa') + version('2.3.3', sha256='9b7b5dd235dbf85099fba3b6f1ccb49bb755923efed66ddc335921f44cb1b8a8') + + provides('mpi') + provides('mpi@:3.1') + + variant( + 'process_managers', + description='The process manager to activate.', + default='mpirun', + values=('slurm', 'mpirun', 'pbs', 'jsrun'), + multi=False + ) + + variant( + 'distribution', + description='The type of fabric distribution.', + default='stock-ofed', + values=('stock-ofed', 'mofed4.5', 'mofed4.6', 'mofed4.7', 'mofed5.0'), + multi=False + ) + + variant( + 'pmi_version', + description='The pmi version to be used with slurm. ' + 'Is ignored if set for mpirun or jsrun. ' + 'jsrun uses pmix regardless of chosen option.', + default='pmi1', + values=('pmi1', 'pmi2', 'pmix'), + multi=False + ) + + variant( + 'mcast', + description='Enable/Disable support for mcast', + default=True + ) + + variant( + 'openacc', + description='Enable/Disable support for openacc', + default=False + ) + + variant( + 'core_direct', + description='Enable/Disable support for core_direct', + default=False + ) + + depends_on('bison@3.4.2', type='build') + depends_on('libpciaccess@0.13.5', when=(sys.platform != 'darwin')) + depends_on('libxml2@2.9.10') + depends_on('cuda@9.2.88:10.2.89') + depends_on('pmix@3.1.3', when='pmi_version=pmix') + + filter_compiler_wrappers( + 'mpicc', 'mpicxx', 'mpif77', 'mpif90', 'mpifort', relative_root='bin' + ) + + @property + def libs(self): + query_parameters = self.spec.last_query.extra_parameters + libraries = ['libmpi'] + + if 'cxx' in query_parameters: + libraries = ['libmpicxx'] + libraries + + return find_libraries( + libraries, root=self.prefix, shared=True, recursive=True + ) + + @property + def process_manager_options(self): + spec = self.spec + + opts = [] + + if '~mcast' in spec: + opts.append('--disable-mcast') + + if '+core_direct' in spec: + opts.append('--with-core-direct') + + if '+openacc' in spec: + opts.append('--enable-openacc') + + # See: http://slurm.schedmd.com/mpi_guide.html#mvapich2 + if 'process_managers=slurm' in spec: + opts.append('--with-pm=slurm') + if 'pmi_version=pmi1' in spec: + opts.append('--with-pmi=pmi1') + if 'pmi_version=pmi2' in spec: + opts.append('--with-pmi=pmi2') + if 'pmi_version=pmix' in spec: + opts.append('--with-pmi=pmix') + opts.append('--with-pmix={0}'.format(spec['pmix'].prefix)) + + elif 'process_managers=pbs' in spec: + opts.append([ + '--with-pm=hydra', + '--with-pbs=/opt/pbs' + ]) + if '~mcast' in spec: + opts.append('--disable-mcast') + if '+core_direct' in spec: + opts.append('--with-core-direct') + if '+openacc' in spec: + opts.append('--enable-openacc') + + elif 'process_managers=jsrun' in spec: + opts.append([ + '--with-pmi=pmix', + '--with-pmix={0}'.format(spec['pmix'].prefix), + '--with-pm=jsm' + ]) + + return opts + + def setup_build_environment(self, env): + # mvapich2 configure fails when F90 and F90FLAGS are set + env.unset('F90') + env.unset('F90FLAGS') + + def setup_run_environment(self, env): + if 'pmi_version=pmi1' in self.spec: + env.set('SLURM_MPI_TYPE', 'pmi1') + if 'pmi_version=pmi2' in self.spec: + env.set('SLURM_MPI_TYPE', 'pmi2') + if 'pmi_version=pmix' in self.spec: + env.set('SLURM_MPI_TYPE', 'pmix') + + # Because MPI functions as a compiler, we need to treat it as one and + # add its compiler paths to the run environment. + self.setup_compiler_environment(env) + + def setup_dependent_build_environment(self, env, dependent_spec): + self.setup_compiler_environment(env) + + # use the Spack compiler wrappers under MPI + env.set('MPICH_CC', spack_cc) + env.set('MPICH_CXX', spack_cxx) + env.set('MPICH_F77', spack_f77) + env.set('MPICH_F90', spack_fc) + env.set('MPICH_FC', spack_fc) + + def setup_compiler_environment(self, env): + env.set('MPICC', join_path(self.prefix.bin, 'mpicc')) + env.set('MPICXX', join_path(self.prefix.bin, 'mpicxx')) + env.set('MPIF77', join_path(self.prefix.bin, 'mpif77')) + env.set('MPIF90', join_path(self.prefix.bin, 'mpif90')) + + def setup_dependent_package(self, module, dependent_spec): + self.spec.mpicc = join_path(self.prefix.bin, 'mpicc') + self.spec.mpicxx = join_path(self.prefix.bin, 'mpicxx') + self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') + self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77') + self.spec.mpicxx_shared_libs = [ + os.path.join(self.prefix.lib, 'libmpicxx.{0}'.format(dso_suffix)), + os.path.join(self.prefix.lib, 'libmpi.{0}'.format(dso_suffix)) + ] + + def configure_args(self): + args = ['--enable-cuda', + '--disable-hybrid', + '--with-ch3-rank-bits=32', + '--disable-gl', + '--without-hydra-ckpointlib', + '--disable-static', + '--enable-shared', + '--disable-rdma-cm' + ] + args.extend(self.process_manager_options) + return args diff --git a/var/spack/repos/builtin/packages/mvapich2x/package.py b/var/spack/repos/builtin/packages/mvapich2x/package.py new file mode 100755 index 0000000000..e164acc53c --- /dev/null +++ b/var/spack/repos/builtin/packages/mvapich2x/package.py @@ -0,0 +1,227 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os.path +import sys + + +class Mvapich2x(AutotoolsPackage): + """MVAPICH2-X is the advanced version of the MVAPICH2 MPI library with + enhanced features (UMR, ODP, DC, Core-Direct, SHARP, XPMEM), OSU INAM + (InifniBand Network Monitoring and Analysis),PGAS (OpenSHMEM, UPC, UPC++, + and CAF), and MPI+PGAS programming models with unified communication + runtime. MVAPICH2-X is not installable from source and is only available + through a binary mirror. If you do not find the binary you're looking for, + send us an email at mvapich@cse.ohio-state.edu. The binary mirror url is: + http://mvapich.cse.ohio-state.edu:8080/download/mvapich/spack-mirror/mvapich2x/ + """ + + homepage = "http://mvapich.cse.ohio-state.edu" + url = "http://mvapich.cse.ohio-state.edu:8080/download/mvapich/spack-mirror/mvapich2x/mvapich2x-2.3.tar.gz" + + maintainers = ['nithintsk', 'harisubramoni'] + + version('2.3', sha256='fc47070e2e9fac09b97022be2320200d732a0a4a820a2b51532b88f8ded14536', preferred=True) + version('2.3rc3', sha256='85a9f1ea1a837d487e356f021ef6f3a4661ad270a0c5f54777b362ee4d45166f') + + provides('mpi') + provides('mpi@:3.1') + + variant( + 'feature', + description=('Feature descriptions are specified at: ' + 'https://mvapich.cse.ohio-state.edu/downloads/'), + default='basic', + values=('basic', 'basic-xpmem', 'advanced', 'advanced-xpmem'), + multi=False + ) + + variant( + 'process_managers', + description='List of the process managers to activate', + default='mpirun', + values=('slurm', 'mpirun', 'pbs', 'jsrun'), + multi=False + ) + + variant( + 'distribution', + description='The type of distribution of the fabric.', + default='stock-ofed', + values=('stock-ofed', 'mofed4.5', 'mofed4.6', 'mofed4.7', 'mofed5.0', + 'ifs10.6', 'ifs10.9'), + multi=False + ) + + variant( + 'pmi_version', + description=('The pmi version to be used with slurm. This variant is ' + 'IGNORED if set for mpirun or jsrun. jsrun uses pmix ' + 'regardless of chosen option.'), + default='pmi1', + values=('pmi1', 'pmi2', 'pmix'), + multi=False + ) + + depends_on('bison@3.4.2', type='build') + depends_on('libpciaccess@0.13.5', when=(sys.platform != 'darwin')) + depends_on('libxml2@2.9.10') + depends_on('pmix@3.1.3', when='pmi_version=pmix') + + filter_compiler_wrappers( + 'mpicc', 'mpicxx', 'mpif77', 'mpif90', 'mpifort', relative_root='bin' + ) + + @property + def libs(self): + query_parameters = self.spec.last_query.extra_parameters + libraries = ['libmpi'] + + if 'cxx' in query_parameters: + libraries = ['libmpicxx'] + libraries + + return find_libraries( + libraries, root=self.prefix, shared=True, recursive=True + ) + + @property + def process_feature_options(self): + spec = self.spec + opts = [] + + if 'feature=basic' in spec: + opts = ['--enable-mcast', '--enable-hybrid', '--enable-mpit-tool', + '--enable-mpit-pvars=mv2'] + elif 'feature=basic-xpmem' in spec: + opts = ['--enable-mcast', '--enable-hybrid', '--enable-mpit-tool', + '--enable-mpit-pvars=mv2', '--with-xpmem=/opt/xpmem/'] + elif 'feature=advanced' in spec: + opts = ['--enable-mcast', '--enable-hybrid', '--enable-mpit-tool', + '--enable-mpit-pvars=mv2', '--with-core-direct', + '--enable-dc', '--enable-umr'] + elif 'feature=advanced-xpmem' in spec: + opts = ['--enable-mcast', '--enable-hybrid', '--enable-mpit-tool', + '--enable-mpit-pvars=mv2', '--with-core-direct', + '--enable-dc', '--enable-umr', '--with-xpmem=/opt/xpmem/'] + return opts + + @property + def distribution_options(self): + opts = [] + if ('distribution=ifs10.6' in self.spec or + 'distribution=ifs10.9' in self.spec): + opts = ["--with-device=ch3:psm"] + else: + opts = ["--with-device=ch3:mrail", "--with-rdma=gen2"] + return opts + + @property + def process_manager_options(self): + spec = self.spec + opts = [] + # See: http://slurm.schedmd.com/mpi_guide.html#mvapich2 + if 'process_managers=slurm' in spec: + opts = [ + '--with-ch3-rank-bits=32', + '--with-pm=slurm' + ] + if 'pmi_version=pmi1' in spec: + opts.append('--with-pmi=pmi1') + if 'pmi_version=pmi2' in spec: + opts.append('--with-pmi=pmi2') + if 'pmi_version=pmix' in spec: + opts.append('--with-pmi=pmix') + opts.append('--with-pmix={0}'.format(spec['pmix'].prefix)) + elif 'process_managers=pbs' in spec: + opts = ['--with-ch3-rank-bits=32', '--with-pbs=/opt/pbs', + '--with-pm=hydra'] + elif 'process_managers=jsrun' in spec: + opts = ['--with-ch3-rank-bits=32', '--with-pmi=pmix', + '--with-pmix={0}'.format(['pmix'].prefix), + '--with-pm=jsm'] + opts.append('--disable-gl') + return opts + + @property + def construct_ldflags(self): + # LDFLAGS contributed by the process manager + spec = self.spec + xpmem_ldflags = '' + if ('feature=basic-xpmem' in spec or 'feature=advanced-xpmem' in spec): + xpmem_ldflags = (' -Wl,-rpath,/opt/xpmem/lib ' + '-L/opt/xpmem/lib -lxpmem') + + # Add default LDFLAGS and combine together + LDFLAGS = 'LDFLAGS=-Wl,-rpath,XORIGIN/placeholder' + LDFLAGS = LDFLAGS + xpmem_ldflags + return LDFLAGS + + @property + def construct_cflags(self): + # CFLAGS contributed by the feature + spec = self.spec + cflags = 'CFLAGS=' + if ('feature=basic-xpmem' in spec or 'feature=advanced-xpmem' in spec): + cflags = cflags + '-I/opt/xpmem/include' + return cflags + + def setup_build_environment(self, env): + # mvapich2 configure fails when F90 and F90FLAGS are set + env.unset('F90') + env.unset('F90FLAGS') + + def setup_run_environment(self, env): + if 'pmi_version=pmi1' in self.spec: + set('SLURM_MPI_TYPE', 'pmi1') + if 'pmi_version=pmi2' in self.spec: + set('SLURM_MPI_TYPE', 'pmi2') + if 'pmi_version=pmix' in self.spec: + set('SLURM_MPI_TYPE', 'pmix') + + # Because MPI functions as a compiler, we need to treat it as one and + # add its compiler paths to the run environment. + self.setup_compiler_environment(env) + + def setup_dependent_build_environment(self, env, dependent_spec): + self.setup_compiler_environment(env) + + # use the Spack compiler wrappers under MPI + env.set('MPICH_CC', spack_cc) + env.set('MPICH_CXX', spack_cxx) + env.set('MPICH_F77', spack_f77) + env.set('MPICH_F90', spack_fc) + env.set('MPICH_FC', spack_fc) + + def setup_compiler_environment(self, env): + # For Cray MPIs, the regular compiler wrappers *are* the MPI wrappers. + # Cray MPIs always have cray in the module name, e.g. "cray-mvapich" + env.set('MPICC', join_path(self.prefix.bin, 'mpicc')) + env.set('MPICXX', join_path(self.prefix.bin, 'mpicxx')) + env.set('MPIF77', join_path(self.prefix.bin, 'mpif77')) + env.set('MPIF90', join_path(self.prefix.bin, 'mpif90')) + + def setup_dependent_package(self, module, dependent_spec): + self.spec.mpicc = join_path(self.prefix.bin, 'mpicc') + self.spec.mpicxx = join_path(self.prefix.bin, 'mpicxx') + self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') + self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77') + self.spec.mpicxx_shared_libs = [ + os.path.join(self.prefix.lib, 'libmpicxx.{0}'.format(dso_suffix)), + os.path.join(self.prefix.lib, 'libmpi.{0}'.format(dso_suffix)) + ] + + def configure_args(self): + args = [ + '--enable-ucr', + '--disable-static', + '--enable-shared', + '--disable-rdma-cm', + '--without-hydra-ckpointlib' + ] + args.extend(self.process_manager_options) + args.extend(self.distribution_options) + args.append(self.construct_cflags) + args.append(self.construct_ldflags) + return args -- cgit v1.2.3-60-g2f50