gptune: add variants (mpispawn, hypre) (#27733)

Co-authored-by: Adam J. Stewart <ajstewart426@gmail.com>

1 files changed, 115 insertions, 86 deletions

diff --git a/var/spack/repos/builtin/packages/gptune/package.py b/var/spack/repos/builtin/packages/gptune/package.py
index eeebd6f40d..d96c86b074 100644
--- a/var/spack/repos/builtin/packages/gptune/package.py
+++ b/var/spack/repos/builtin/packages/gptune/package.py
@@ -18,7 +18,9 @@ class Gptune(CMakePackage):
 
     version('master', branch='master')
 
-    variant('app', default=False, description='Build all HPC application examples')
+    variant('superlu', default=False, description='Build the SuperLU_DIST example')
+    variant('hypre', default=False, description='Build the Hypre example')
+    variant('mpispawn', default=True, description='MPI spawning-based interface')
 
     depends_on('mpi', type=('build', 'link', 'run'))
     depends_on('cmake@3.3:', type='build')
@@ -48,9 +50,14 @@ class Gptune(CMakePackage):
     depends_on('pygmo', type=('build', 'run'))
     depends_on('openturns', type=('build', 'run'))
 
-    depends_on('superlu-dist@develop', when='+app', type=('build', 'run'))
+    depends_on('superlu-dist@develop', when='+superlu', type=('build', 'run'))
+    depends_on('hypre+gptune@2.19.0', when='+hypre', type=('build', 'run'))
 
-    conflicts('openmpi@:3')
+    depends_on('openmpi@4:', when='+mpispawn', type=('build', 'run'))
+    conflicts('mpich', when='+mpispawn')
+    conflicts('spectrum-mpi', when='+mpispawn')
+    conflicts('cray-mpich', when='+mpispawn')
+    conflicts('gcc@:7')
 
     def cmake_args(self):
         spec = self.spec
@@ -59,6 +66,7 @@ class Gptune(CMakePackage):
             fc_flags.append('-fallow-argument-mismatch')
 
         args = [
+            '-DGPTUNE_INSTALL_PATH=%s' % site_packages_dir,
             '-DTPL_BLAS_LIBRARIES=%s' % spec['blas'].libs.joined(";"),
             '-DTPL_LAPACK_LIBRARIES=%s' % spec['lapack'].libs.joined(";"),
             '-DTPL_SCALAPACK_LIBRARIES=%s' % spec['scalapack'].
@@ -81,96 +89,117 @@ class Gptune(CMakePackage):
     def cache_test_sources(self):
         """Copy the example source files after the package is installed to an
         install test subdirectory for use during `spack test run`."""
-        self.cache_extra_test_sources([self.examples_src_dir, self.src_dir])
+        self.cache_extra_test_sources([self.examples_src_dir])
+
+    def setup_run_environment(self, env):
+        env.set('GPTUNE_INSTALL_PATH', site_packages_dir)
 
     def test(self):
         spec = self.spec
         comp_name = self.compiler.name
         comp_version = str(self.compiler.version).replace('.', ',')
-        test_dir = join_path(self.install_test_root, self.examples_src_dir)
+        test_dir = join_path(self.test_suite.current_test_cache_dir,
+                             self.examples_src_dir)
 
-        if '+app' in spec:
-            superludriver = join_path(spec['superlu-dist'].prefix.bin, 'pddrive_spawn')
+        if '+superlu' in spec:
+            superludriver = join_path(spec['superlu-dist'].prefix.lib,
+                                      'EXAMPLE/pddrive_spawn')
             op = ['-r', superludriver, '.']
             # copy superlu-dist executables to the correct place
-            with working_dir(join_path(test_dir, 'SuperLU_DIST'), create=False):
-                self.run_test('rm', options=['-rf', 'superlu_dist'], work_dir='.')
-                self.run_test('git', options=['clone', 'https://github.com/xiaoyeli/superlu_dist.git'], work_dir='.')
-                self.run_test('mkdir', options=['-p',
-                                                'build'], work_dir='./superlu_dist')
-                self.run_test('mkdir', options=['-p', 'EXAMPLE'],
-                              work_dir='./superlu_dist/build')
-                self.run_test('cp', options=op, work_dir='./superlu_dist/build/EXAMPLE')
-
-        with working_dir(self.install_test_root, create=False):
-            cdir = join_path(self.prefix, 'gptuneclcm')
-            self.run_test('cp', options=['-r', cdir, '.'], work_dir='.')
-            self.run_test('rm', options=['-rf', 'build'], work_dir='.')
-            self.run_test('mv', options=['gptuneclcm', 'build'], work_dir='.')
-
-            with open('{0}/run_env.sh'.format(self.install_test_root), 'w') as envfile:
-                envfile.write('if [[ $NERSC_HOST = "cori" ]]; then\n')
-                envfile.write('    export machine=cori\n')
-                envfile.write('elif [[ $(uname -s) = "Darwin" ]]; then\n')
-                envfile.write('    export machine=mac\n')
-                envfile.write('elif [[ $(dnsdomainname) = ' +
-                              '"summit.olcf.ornl.gov" ]]; then\n')
-                envfile.write('    export machine=summit\n')
-                envfile.write('elif [[ $(cat /etc/os-release | grep "PRETTY_NAME") ==' +
-                              ' *"Ubuntu"* || $(cat /etc/os-release | grep' +
-                              ' "PRETTY_NAME") == *"Debian"* ]]; then\n')
-                envfile.write('    export machine=unknownlinux\n')
-                envfile.write('fi\n')
-                envfile.write('export GPTUNEROOT=$PWD\n')
-                envfile.write('export MPIRUN={0}\n'.format
-                              (which(spec['mpi'].prefix.bin + '/mpirun')))
-                envfile.write('export proc=$(spack arch)\n')
-                envfile.write('export mpi={0}\n'.format(spec['mpi'].name))
-                envfile.write('export compiler={0}\n'.format(comp_name))
-                envfile.write('export nodes={0} \n'.format(self.nodes))
-                envfile.write('export cores={0} \n'.format(self.cores))
-                envfile.write('export ModuleEnv=$machine-$proc-$mpi-$compiler \n')
-                envfile.write('software_json=$(echo ",\\\"software_configuration\\\":' +
-                              '{\\\"' + spec['blas'].name +
-                              '\\\":{\\\"version_split\\\":' +
-                              ' [' + str(spec['blas'].versions).replace('.', ',') +
-                              ']},\\\"' + spec['mpi'].name +
-                              '\\\":{\\\"version_split\\\": [' +
-                              str(spec['mpi'].versions).replace('.', ',') + ']},\\\"' +
-                              spec['scalapack'].name +
-                              '\\\":{\\\"version_split\\\": [' +
-                              str(spec['scalapack'].versions).replace('.', ',') +
-                              ']},\\\"' +
-                              str(comp_name) + '\\\":{\\\"version_split\\\": [' +
-                              str(comp_version) + ']}}") \n')
-                envfile.write('loadable_software_json=$(echo ",\\\"loadable_software_' +
-                              'configurations\\\":{\\\"' + spec['blas'].name +
-                              '\\\":{\\\"version_split\\\": [' +
-                              str(spec['blas'].versions).replace('.', ',') +
-                              ']},\\\"' + spec['mpi'].name +
-                              '\\\":{\\\"version_split\\\": [' +
-                              str(spec['mpi'].versions).replace('.', ',') + ']},\\\"' +
-                              spec['scalapack'].name +
-                              '\\\":{\\\"version_split\\\": [' +
-                              str(spec['scalapack'].versions).replace('.', ',') +
-                              ']},\\\"' + str(comp_name) +
-                              '\\\":{\\\"version_split\\\": ['
-                              + str(comp_version) + ']}}") \n')
-                envfile.write('machine_json=$(echo ",\\\"machine_configuration\\\":' +
-                              '{\\\"machine_name\\\":\\\"$machine\\\",\\\"$proc\\\":' +
-                              '{\\\"nodes\\\":$nodes,\\\"cores\\\":$cores}}") \n')
-                envfile.write('loadable_machine_json=$(echo ",\\\"loadable_machine_' +
-                              'configurations\\\":{\\\"$machine\\\":{\\\"$proc\\\":' +
-                              '{\\\"nodes\\\":$nodes,\\\"cores\\\":$cores}}}") \n')
-
-        if '+app' in spec:
-            apps = ['GPTune-Demo', 'SuperLU_DIST', 'SuperLU_DIST_RCI',
-                    'Scalapack-PDGEQRF', 'Scalapack-PDGEQRF_RCI']
-        else:
-            apps = ['GPTune-Demo', 'Scalapack-PDGEQRF', 'Scalapack-PDGEQRF_RCI']
+            wd = join_path(test_dir, 'SuperLU_DIST')
+            self.run_test('rm', options=['-rf', 'superlu_dist'], work_dir=wd)
+            self.run_test('git', options=['clone', 'https://github.com/xiaoyeli/superlu_dist.git'], work_dir=wd)
+            self.run_test('mkdir', options=['-p',
+                                            'build'], work_dir=wd + '/superlu_dist')
+            self.run_test('mkdir', options=['-p', 'EXAMPLE'],
+                          work_dir=wd + '/superlu_dist/build')
+            self.run_test('cp', options=op, work_dir=wd + '/superlu_dist/build/EXAMPLE')
+
+        if '+hypre' in spec:
+            hypredriver = join_path(spec['hypre'].prefix.bin, 'ij')
+            op = ['-r', hypredriver, '.']
+            # copy superlu-dist executables to the correct place
+            wd = join_path(test_dir, 'Hypre')
+            self.run_test('rm', options=['-rf', 'hypre'], work_dir=wd)
+            self.run_test('git', options=['clone', 'https://github.com/hypre-space/hypre.git'], work_dir=wd)
+            self.run_test('cp', options=op, work_dir=wd + '/hypre/src/test/')
+
+        wd = self.test_suite.current_test_cache_dir
+        with open('{0}/run_env.sh'.format(wd), 'w') as envfile:
+            envfile.write('if [[ $NERSC_HOST = "cori" ]]; then\n')
+            envfile.write('    export machine=cori\n')
+            envfile.write('elif [[ $(uname -s) = "Darwin" ]]; then\n')
+            envfile.write('    export machine=mac\n')
+            envfile.write('elif [[ $(dnsdomainname) = ' +
+                          '"summit.olcf.ornl.gov" ]]; then\n')
+            envfile.write('    export machine=summit\n')
+            envfile.write('elif [[ $(cat /etc/os-release | grep "PRETTY_NAME") ==' +
+                          ' *"Ubuntu"* || $(cat /etc/os-release | grep' +
+                          ' "PRETTY_NAME") == *"Debian"* ]]; then\n')
+            envfile.write('    export machine=unknownlinux\n')
+            envfile.write('fi\n')
+            envfile.write('export GPTUNEROOT=$PWD\n')
+            envfile.write('export MPIRUN={0}\n'.format
+                          (which(spec['mpi'].prefix.bin + '/mpirun')))
+            envfile.write('export PYTHONPATH={0}:$PYTHONPATH\n'.format
+                          (site_packages_dir + '/gptune'))
+            envfile.write('export proc=$(spack arch)\n')
+            envfile.write('export mpi={0}\n'.format(spec['mpi'].name))
+            envfile.write('export compiler={0}\n'.format(comp_name))
+            envfile.write('export nodes={0} \n'.format(self.nodes))
+            envfile.write('export cores={0} \n'.format(self.cores))
+            envfile.write('export ModuleEnv=$machine-$proc-$mpi-$compiler \n')
+            envfile.write('software_json=$(echo ",\\\"software_configuration\\\":' +
+                          '{\\\"' + spec['blas'].name +
+                          '\\\":{\\\"version_split\\\":' +
+                          ' [' + str(spec['blas'].versions).replace('.', ',') +
+                          ']},\\\"' + spec['mpi'].name +
+                          '\\\":{\\\"version_split\\\": [' +
+                          str(spec['mpi'].versions).replace('.', ',') + ']},\\\"' +
+                          spec['scalapack'].name +
+                          '\\\":{\\\"version_split\\\": [' +
+                          str(spec['scalapack'].versions).replace('.', ',') +
+                          ']},\\\"' +
+                          str(comp_name) + '\\\":{\\\"version_split\\\": [' +
+                          str(comp_version) + ']}}") \n')
+            envfile.write('loadable_software_json=$(echo ",\\\"loadable_software_' +
+                          'configurations\\\":{\\\"' + spec['blas'].name +
+                          '\\\":{\\\"version_split\\\": [' +
+                          str(spec['blas'].versions).replace('.', ',') +
+                          ']},\\\"' + spec['mpi'].name +
+                          '\\\":{\\\"version_split\\\": [' +
+                          str(spec['mpi'].versions).replace('.', ',') + ']},\\\"' +
+                          spec['scalapack'].name +
+                          '\\\":{\\\"version_split\\\": [' +
+                          str(spec['scalapack'].versions).replace('.', ',') +
+                          ']},\\\"' + str(comp_name) +
+                          '\\\":{\\\"version_split\\\": ['
+                          + str(comp_version) + ']}}") \n')
+            envfile.write('machine_json=$(echo ",\\\"machine_configuration\\\":' +
+                          '{\\\"machine_name\\\":\\\"$machine\\\",\\\"$proc\\\":' +
+                          '{\\\"nodes\\\":$nodes,\\\"cores\\\":$cores}}") \n')
+            envfile.write('loadable_machine_json=$(echo ",\\\"loadable_machine_' +
+                          'configurations\\\":{\\\"$machine\\\":{\\\"$proc\\\":' +
+                          '{\\\"nodes\\\":$nodes,\\\"cores\\\":$cores}}}") \n')
+
+        # copy the environment configuration files to non-cache directories
+        op = ['run_env.sh', site_packages_dir + '/gptune/.']
+        self.run_test('cp', options=op, work_dir=wd)
+        op = ['run_env.sh', self.install_test_root + '/.']
+        self.run_test('cp', options=op, work_dir=wd)
+
+        apps = ['Scalapack-PDGEQRF_RCI']
+        if '+mpispawn' in spec:
+            apps = apps + ['GPTune-Demo', 'Scalapack-PDGEQRF']
+        if '+superlu' in spec:
+            apps = apps + ['SuperLU_DIST_RCI']
+            if '+mpispawn' in spec:
+                apps = apps + ['SuperLU_DIST']
+        if '+hypre' in spec:
+            if '+mpispawn' in spec:
+                apps = apps + ['Hypre']
 
         for app in apps:
-            with working_dir(join_path(test_dir, app), create=False):
-                # PDGEQRF with GPTune
-                self.run_test('bash', options=['run_examples.sh'], work_dir='.',
-                              purpose='gptune smoke test for {0}'.format(app))
+            wd = join_path(test_dir, app)
+            self.run_test('bash', options=['run_examples.sh'], work_dir=wd,
+                          purpose='gptune smoke test for {0}'.format(app))