New packages Mvapich2x and Mvapich2-GDR (#17883)

* Adding new packages Mvapich2x and Mvapich2-GDR which can be installed only via binary mirrors

* Added docstring descriptions to both packages

* Removed variant wrapper for cuda dependencies

* Fixed multiple flake8 errors

* Updated APIs to pass unit tests

* Updated APIs for MVAPICH2-X package and fixed flake8 warnings for MVAPICH2-GDR

* Changed url back to single line

* Removed extra parantesis around URL string

Co-authored-by: nithintsk <nithintsk@github.com>

2 files changed, 424 insertions, 0 deletions

diff --git a/var/spack/repos/builtin/packages/mvapich2-gdr/package.py b/var/spack/repos/builtin/packages/mvapich2-gdr/package.py
new file mode 100755
index 0000000000..1a0fe8def6
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mvapich2-gdr/package.py
@@ -0,0 +1,197 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os.path
+import sys
+
+
+class Mvapich2Gdr(AutotoolsPackage):
+    """MVAPICH2-GDR is an optimized version of the MVAPICH2 MPI library for
+    GPU-enabled HPC and Deep Learning Applications. MVAPICH2-GDR is not
+    installable from source and is only available through a binary mirror.
+    If you do not find the binary you're looking for, send us an email at
+    mvapich@cse.ohio-state.edu. The binary mirror url is:
+    http://mvapich.cse.ohio-state.edu:8080/download/mvapich/spack-mirror/mvapich2-gdr/
+    """
+
+    homepage = 'http://mvapich.cse.ohio-state.edu'
+    url      = 'http://mvapich.cse.ohio-state.edu:8080/download/mvapich/spack-mirror/mvapich2-gdr/mvapich2-gdr-2.3.4.tar.gz'
+
+    maintainers = ['nithintsk', 'harisubramoni']
+
+    version('2.3.4', sha256='ed78101e6bb807e979213006ee5f20ff466369b01f96b6d1cf0c471baf7e35aa')
+    version('2.3.3', sha256='9b7b5dd235dbf85099fba3b6f1ccb49bb755923efed66ddc335921f44cb1b8a8')
+
+    provides('mpi')
+    provides('mpi@:3.1')
+
+    variant(
+        'process_managers',
+        description='The process manager to activate.',
+        default='mpirun',
+        values=('slurm', 'mpirun', 'pbs', 'jsrun'),
+        multi=False
+    )
+
+    variant(
+        'distribution',
+        description='The type of fabric distribution.',
+        default='stock-ofed',
+        values=('stock-ofed', 'mofed4.5', 'mofed4.6', 'mofed4.7', 'mofed5.0'),
+        multi=False
+    )
+
+    variant(
+        'pmi_version',
+        description='The pmi version to be used with slurm. '
+                    'Is ignored if set for mpirun or jsrun. '
+                    'jsrun uses pmix regardless of chosen option.',
+        default='pmi1',
+        values=('pmi1', 'pmi2', 'pmix'),
+        multi=False
+    )
+
+    variant(
+        'mcast',
+        description='Enable/Disable support for mcast',
+        default=True
+    )
+
+    variant(
+        'openacc',
+        description='Enable/Disable support for openacc',
+        default=False
+    )
+
+    variant(
+        'core_direct',
+        description='Enable/Disable support for core_direct',
+        default=False
+    )
+
+    depends_on('bison@3.4.2', type='build')
+    depends_on('libpciaccess@0.13.5', when=(sys.platform != 'darwin'))
+    depends_on('libxml2@2.9.10')
+    depends_on('cuda@9.2.88:10.2.89')
+    depends_on('pmix@3.1.3', when='pmi_version=pmix')
+
+    filter_compiler_wrappers(
+        'mpicc', 'mpicxx', 'mpif77', 'mpif90', 'mpifort', relative_root='bin'
+    )
+
+    @property
+    def libs(self):
+        query_parameters = self.spec.last_query.extra_parameters
+        libraries = ['libmpi']
+
+        if 'cxx' in query_parameters:
+            libraries = ['libmpicxx'] + libraries
+
+        return find_libraries(
+            libraries, root=self.prefix, shared=True, recursive=True
+        )
+
+    @property
+    def process_manager_options(self):
+        spec = self.spec
+
+        opts = []
+
+        if '~mcast' in spec:
+            opts.append('--disable-mcast')
+
+        if '+core_direct' in spec:
+            opts.append('--with-core-direct')
+
+        if '+openacc' in spec:
+            opts.append('--enable-openacc')
+
+        # See: http://slurm.schedmd.com/mpi_guide.html#mvapich2
+        if 'process_managers=slurm' in spec:
+            opts.append('--with-pm=slurm')
+            if 'pmi_version=pmi1' in spec:
+                opts.append('--with-pmi=pmi1')
+            if 'pmi_version=pmi2' in spec:
+                opts.append('--with-pmi=pmi2')
+            if 'pmi_version=pmix' in spec:
+                opts.append('--with-pmi=pmix')
+                opts.append('--with-pmix={0}'.format(spec['pmix'].prefix))
+
+        elif 'process_managers=pbs' in spec:
+            opts.append([
+                '--with-pm=hydra',
+                '--with-pbs=/opt/pbs'
+            ])
+            if '~mcast' in spec:
+                opts.append('--disable-mcast')
+            if '+core_direct' in spec:
+                opts.append('--with-core-direct')
+            if '+openacc' in spec:
+                opts.append('--enable-openacc')
+
+        elif 'process_managers=jsrun' in spec:
+            opts.append([
+                '--with-pmi=pmix',
+                '--with-pmix={0}'.format(spec['pmix'].prefix),
+                '--with-pm=jsm'
+            ])
+
+        return opts
+
+    def setup_build_environment(self, env):
+        # mvapich2 configure fails when F90 and F90FLAGS are set
+        env.unset('F90')
+        env.unset('F90FLAGS')
+
+    def setup_run_environment(self, env):
+        if 'pmi_version=pmi1' in self.spec:
+            env.set('SLURM_MPI_TYPE', 'pmi1')
+        if 'pmi_version=pmi2' in self.spec:
+            env.set('SLURM_MPI_TYPE', 'pmi2')
+        if 'pmi_version=pmix' in self.spec:
+            env.set('SLURM_MPI_TYPE', 'pmix')
+
+        # Because MPI functions as a compiler, we need to treat it as one and
+        # add its compiler paths to the run environment.
+        self.setup_compiler_environment(env)
+
+    def setup_dependent_build_environment(self, env, dependent_spec):
+        self.setup_compiler_environment(env)
+
+        # use the Spack compiler wrappers under MPI
+        env.set('MPICH_CC', spack_cc)
+        env.set('MPICH_CXX', spack_cxx)
+        env.set('MPICH_F77', spack_f77)
+        env.set('MPICH_F90', spack_fc)
+        env.set('MPICH_FC', spack_fc)
+
+    def setup_compiler_environment(self, env):
+        env.set('MPICC',  join_path(self.prefix.bin, 'mpicc'))
+        env.set('MPICXX', join_path(self.prefix.bin, 'mpicxx'))
+        env.set('MPIF77', join_path(self.prefix.bin, 'mpif77'))
+        env.set('MPIF90', join_path(self.prefix.bin, 'mpif90'))
+
+    def setup_dependent_package(self, module, dependent_spec):
+        self.spec.mpicc  = join_path(self.prefix.bin, 'mpicc')
+        self.spec.mpicxx = join_path(self.prefix.bin, 'mpicxx')
+        self.spec.mpifc  = join_path(self.prefix.bin, 'mpif90')
+        self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77')
+        self.spec.mpicxx_shared_libs = [
+            os.path.join(self.prefix.lib, 'libmpicxx.{0}'.format(dso_suffix)),
+            os.path.join(self.prefix.lib, 'libmpi.{0}'.format(dso_suffix))
+        ]
+
+    def configure_args(self):
+        args = ['--enable-cuda',
+                '--disable-hybrid',
+                '--with-ch3-rank-bits=32',
+                '--disable-gl',
+                '--without-hydra-ckpointlib',
+                '--disable-static',
+                '--enable-shared',
+                '--disable-rdma-cm'
+                ]
+        args.extend(self.process_manager_options)
+        return args
diff --git a/var/spack/repos/builtin/packages/mvapich2x/package.py b/var/spack/repos/builtin/packages/mvapich2x/package.py
new file mode 100755
index 0000000000..e164acc53c
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mvapich2x/package.py
@@ -0,0 +1,227 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os.path
+import sys
+
+
+class Mvapich2x(AutotoolsPackage):
+    """MVAPICH2-X is the advanced version of the MVAPICH2 MPI library with
+    enhanced features (UMR, ODP, DC, Core-Direct, SHARP, XPMEM), OSU INAM
+    (InifniBand Network Monitoring and Analysis),PGAS (OpenSHMEM, UPC, UPC++,
+    and CAF), and MPI+PGAS programming models with unified communication
+    runtime. MVAPICH2-X is not installable from source and is only available
+    through a binary mirror. If you do not find the binary you're looking for,
+    send us an email at mvapich@cse.ohio-state.edu. The binary mirror url is:
+    http://mvapich.cse.ohio-state.edu:8080/download/mvapich/spack-mirror/mvapich2x/
+    """
+
+    homepage = "http://mvapich.cse.ohio-state.edu"
+    url      = "http://mvapich.cse.ohio-state.edu:8080/download/mvapich/spack-mirror/mvapich2x/mvapich2x-2.3.tar.gz"
+
+    maintainers = ['nithintsk', 'harisubramoni']
+
+    version('2.3', sha256='fc47070e2e9fac09b97022be2320200d732a0a4a820a2b51532b88f8ded14536', preferred=True)
+    version('2.3rc3', sha256='85a9f1ea1a837d487e356f021ef6f3a4661ad270a0c5f54777b362ee4d45166f')
+
+    provides('mpi')
+    provides('mpi@:3.1')
+
+    variant(
+        'feature',
+        description=('Feature descriptions are specified at: '
+                     'https://mvapich.cse.ohio-state.edu/downloads/'),
+        default='basic',
+        values=('basic', 'basic-xpmem', 'advanced', 'advanced-xpmem'),
+        multi=False
+    )
+
+    variant(
+        'process_managers',
+        description='List of the process managers to activate',
+        default='mpirun',
+        values=('slurm', 'mpirun', 'pbs', 'jsrun'),
+        multi=False
+    )
+
+    variant(
+        'distribution',
+        description='The type of distribution of the fabric.',
+        default='stock-ofed',
+        values=('stock-ofed', 'mofed4.5', 'mofed4.6', 'mofed4.7', 'mofed5.0',
+                'ifs10.6', 'ifs10.9'),
+        multi=False
+    )
+
+    variant(
+        'pmi_version',
+        description=('The pmi version to be used with slurm. This variant is '
+                     'IGNORED if set for mpirun or jsrun. jsrun uses pmix '
+                     'regardless of chosen option.'),
+        default='pmi1',
+        values=('pmi1', 'pmi2', 'pmix'),
+        multi=False
+    )
+
+    depends_on('bison@3.4.2', type='build')
+    depends_on('libpciaccess@0.13.5', when=(sys.platform != 'darwin'))
+    depends_on('libxml2@2.9.10')
+    depends_on('pmix@3.1.3', when='pmi_version=pmix')
+
+    filter_compiler_wrappers(
+        'mpicc', 'mpicxx', 'mpif77', 'mpif90', 'mpifort', relative_root='bin'
+    )
+
+    @property
+    def libs(self):
+        query_parameters = self.spec.last_query.extra_parameters
+        libraries = ['libmpi']
+
+        if 'cxx' in query_parameters:
+            libraries = ['libmpicxx'] + libraries
+
+        return find_libraries(
+            libraries, root=self.prefix, shared=True, recursive=True
+        )
+
+    @property
+    def process_feature_options(self):
+        spec = self.spec
+        opts = []
+
+        if 'feature=basic' in spec:
+            opts = ['--enable-mcast', '--enable-hybrid', '--enable-mpit-tool',
+                    '--enable-mpit-pvars=mv2']
+        elif 'feature=basic-xpmem' in spec:
+            opts = ['--enable-mcast', '--enable-hybrid', '--enable-mpit-tool',
+                    '--enable-mpit-pvars=mv2', '--with-xpmem=/opt/xpmem/']
+        elif 'feature=advanced' in spec:
+            opts = ['--enable-mcast', '--enable-hybrid', '--enable-mpit-tool',
+                    '--enable-mpit-pvars=mv2', '--with-core-direct',
+                    '--enable-dc', '--enable-umr']
+        elif 'feature=advanced-xpmem' in spec:
+            opts = ['--enable-mcast', '--enable-hybrid', '--enable-mpit-tool',
+                    '--enable-mpit-pvars=mv2', '--with-core-direct',
+                    '--enable-dc', '--enable-umr', '--with-xpmem=/opt/xpmem/']
+        return opts
+
+    @property
+    def distribution_options(self):
+        opts = []
+        if ('distribution=ifs10.6' in self.spec or
+            'distribution=ifs10.9' in self.spec):
+            opts = ["--with-device=ch3:psm"]
+        else:
+            opts = ["--with-device=ch3:mrail", "--with-rdma=gen2"]
+        return opts
+
+    @property
+    def process_manager_options(self):
+        spec = self.spec
+        opts = []
+        # See: http://slurm.schedmd.com/mpi_guide.html#mvapich2
+        if 'process_managers=slurm' in spec:
+            opts = [
+                '--with-ch3-rank-bits=32',
+                '--with-pm=slurm'
+            ]
+            if 'pmi_version=pmi1' in spec:
+                opts.append('--with-pmi=pmi1')
+            if 'pmi_version=pmi2' in spec:
+                opts.append('--with-pmi=pmi2')
+            if 'pmi_version=pmix' in spec:
+                opts.append('--with-pmi=pmix')
+                opts.append('--with-pmix={0}'.format(spec['pmix'].prefix))
+        elif 'process_managers=pbs' in spec:
+            opts = ['--with-ch3-rank-bits=32', '--with-pbs=/opt/pbs',
+                    '--with-pm=hydra']
+        elif 'process_managers=jsrun' in spec:
+            opts = ['--with-ch3-rank-bits=32', '--with-pmi=pmix',
+                    '--with-pmix={0}'.format(['pmix'].prefix),
+                    '--with-pm=jsm']
+        opts.append('--disable-gl')
+        return opts
+
+    @property
+    def construct_ldflags(self):
+        # LDFLAGS contributed by the process manager
+        spec = self.spec
+        xpmem_ldflags = ''
+        if ('feature=basic-xpmem' in spec or 'feature=advanced-xpmem' in spec):
+            xpmem_ldflags = (' -Wl,-rpath,/opt/xpmem/lib '
+                             '-L/opt/xpmem/lib -lxpmem')
+
+        # Add default LDFLAGS and combine together
+        LDFLAGS = 'LDFLAGS=-Wl,-rpath,XORIGIN/placeholder'
+        LDFLAGS = LDFLAGS + xpmem_ldflags
+        return LDFLAGS
+
+    @property
+    def construct_cflags(self):
+        # CFLAGS contributed by the feature
+        spec = self.spec
+        cflags = 'CFLAGS='
+        if ('feature=basic-xpmem' in spec or 'feature=advanced-xpmem' in spec):
+            cflags = cflags + '-I/opt/xpmem/include'
+        return cflags
+
+    def setup_build_environment(self, env):
+        # mvapich2 configure fails when F90 and F90FLAGS are set
+        env.unset('F90')
+        env.unset('F90FLAGS')
+
+    def setup_run_environment(self, env):
+        if 'pmi_version=pmi1' in self.spec:
+            set('SLURM_MPI_TYPE', 'pmi1')
+        if 'pmi_version=pmi2' in self.spec:
+            set('SLURM_MPI_TYPE', 'pmi2')
+        if 'pmi_version=pmix' in self.spec:
+            set('SLURM_MPI_TYPE', 'pmix')
+
+        # Because MPI functions as a compiler, we need to treat it as one and
+        # add its compiler paths to the run environment.
+        self.setup_compiler_environment(env)
+
+    def setup_dependent_build_environment(self, env, dependent_spec):
+        self.setup_compiler_environment(env)
+
+        # use the Spack compiler wrappers under MPI
+        env.set('MPICH_CC', spack_cc)
+        env.set('MPICH_CXX', spack_cxx)
+        env.set('MPICH_F77', spack_f77)
+        env.set('MPICH_F90', spack_fc)
+        env.set('MPICH_FC', spack_fc)
+
+    def setup_compiler_environment(self, env):
+        # For Cray MPIs, the regular compiler wrappers *are* the MPI wrappers.
+        # Cray MPIs always have cray in the module name, e.g. "cray-mvapich"
+        env.set('MPICC',  join_path(self.prefix.bin, 'mpicc'))
+        env.set('MPICXX', join_path(self.prefix.bin, 'mpicxx'))
+        env.set('MPIF77', join_path(self.prefix.bin, 'mpif77'))
+        env.set('MPIF90', join_path(self.prefix.bin, 'mpif90'))
+
+    def setup_dependent_package(self, module, dependent_spec):
+        self.spec.mpicc  = join_path(self.prefix.bin, 'mpicc')
+        self.spec.mpicxx = join_path(self.prefix.bin, 'mpicxx')
+        self.spec.mpifc  = join_path(self.prefix.bin, 'mpif90')
+        self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77')
+        self.spec.mpicxx_shared_libs = [
+            os.path.join(self.prefix.lib, 'libmpicxx.{0}'.format(dso_suffix)),
+            os.path.join(self.prefix.lib, 'libmpi.{0}'.format(dso_suffix))
+        ]
+
+    def configure_args(self):
+        args = [
+            '--enable-ucr',
+            '--disable-static',
+            '--enable-shared',
+            '--disable-rdma-cm',
+            '--without-hydra-ckpointlib'
+        ]
+        args.extend(self.process_manager_options)
+        args.extend(self.distribution_options)
+        args.append(self.construct_cflags)
+        args.append(self.construct_ldflags)
+        return args