gptune: add variants (mpispawn, hypre) (#27733)

Co-authored-by: Adam J. Stewart <ajstewart426@gmail.com>

5 files changed, 626 insertions, 86 deletions

diff --git a/var/spack/repos/builtin/packages/gptune/package.py b/var/spack/repos/builtin/packages/gptune/package.py
index eeebd6f40d..d96c86b074 100644
--- a/var/spack/repos/builtin/packages/gptune/package.py
+++ b/var/spack/repos/builtin/packages/gptune/package.py
@@ -18,7 +18,9 @@ class Gptune(CMakePackage):
 
     version('master', branch='master')
 
-    variant('app', default=False, description='Build all HPC application examples')
+    variant('superlu', default=False, description='Build the SuperLU_DIST example')
+    variant('hypre', default=False, description='Build the Hypre example')
+    variant('mpispawn', default=True, description='MPI spawning-based interface')
 
     depends_on('mpi', type=('build', 'link', 'run'))
     depends_on('cmake@3.3:', type='build')
@@ -48,9 +50,14 @@ class Gptune(CMakePackage):
     depends_on('pygmo', type=('build', 'run'))
     depends_on('openturns', type=('build', 'run'))
 
-    depends_on('superlu-dist@develop', when='+app', type=('build', 'run'))
+    depends_on('superlu-dist@develop', when='+superlu', type=('build', 'run'))
+    depends_on('hypre+gptune@2.19.0', when='+hypre', type=('build', 'run'))
 
-    conflicts('openmpi@:3')
+    depends_on('openmpi@4:', when='+mpispawn', type=('build', 'run'))
+    conflicts('mpich', when='+mpispawn')
+    conflicts('spectrum-mpi', when='+mpispawn')
+    conflicts('cray-mpich', when='+mpispawn')
+    conflicts('gcc@:7')
 
     def cmake_args(self):
         spec = self.spec
@@ -59,6 +66,7 @@ class Gptune(CMakePackage):
             fc_flags.append('-fallow-argument-mismatch')
 
         args = [
+            '-DGPTUNE_INSTALL_PATH=%s' % site_packages_dir,
             '-DTPL_BLAS_LIBRARIES=%s' % spec['blas'].libs.joined(";"),
             '-DTPL_LAPACK_LIBRARIES=%s' % spec['lapack'].libs.joined(";"),
             '-DTPL_SCALAPACK_LIBRARIES=%s' % spec['scalapack'].
@@ -81,96 +89,117 @@ class Gptune(CMakePackage):
     def cache_test_sources(self):
         """Copy the example source files after the package is installed to an
         install test subdirectory for use during `spack test run`."""
-        self.cache_extra_test_sources([self.examples_src_dir, self.src_dir])
+        self.cache_extra_test_sources([self.examples_src_dir])
+
+    def setup_run_environment(self, env):
+        env.set('GPTUNE_INSTALL_PATH', site_packages_dir)
 
     def test(self):
         spec = self.spec
         comp_name = self.compiler.name
         comp_version = str(self.compiler.version).replace('.', ',')
-        test_dir = join_path(self.install_test_root, self.examples_src_dir)
+        test_dir = join_path(self.test_suite.current_test_cache_dir,
+                             self.examples_src_dir)
 
-        if '+app' in spec:
-            superludriver = join_path(spec['superlu-dist'].prefix.bin, 'pddrive_spawn')
+        if '+superlu' in spec:
+            superludriver = join_path(spec['superlu-dist'].prefix.lib,
+                                      'EXAMPLE/pddrive_spawn')
             op = ['-r', superludriver, '.']
             # copy superlu-dist executables to the correct place
-            with working_dir(join_path(test_dir, 'SuperLU_DIST'), create=False):
-                self.run_test('rm', options=['-rf', 'superlu_dist'], work_dir='.')
-                self.run_test('git', options=['clone', 'https://github.com/xiaoyeli/superlu_dist.git'], work_dir='.')
-                self.run_test('mkdir', options=['-p',
-                                                'build'], work_dir='./superlu_dist')
-                self.run_test('mkdir', options=['-p', 'EXAMPLE'],
-                              work_dir='./superlu_dist/build')
-                self.run_test('cp', options=op, work_dir='./superlu_dist/build/EXAMPLE')
-
-        with working_dir(self.install_test_root, create=False):
-            cdir = join_path(self.prefix, 'gptuneclcm')
-            self.run_test('cp', options=['-r', cdir, '.'], work_dir='.')
-            self.run_test('rm', options=['-rf', 'build'], work_dir='.')
-            self.run_test('mv', options=['gptuneclcm', 'build'], work_dir='.')
-
-            with open('{0}/run_env.sh'.format(self.install_test_root), 'w') as envfile:
-                envfile.write('if [[ $NERSC_HOST = "cori" ]]; then\n')
-                envfile.write('    export machine=cori\n')
-                envfile.write('elif [[ $(uname -s) = "Darwin" ]]; then\n')
-                envfile.write('    export machine=mac\n')
-                envfile.write('elif [[ $(dnsdomainname) = ' +
-                              '"summit.olcf.ornl.gov" ]]; then\n')
-                envfile.write('    export machine=summit\n')
-                envfile.write('elif [[ $(cat /etc/os-release | grep "PRETTY_NAME") ==' +
-                              ' *"Ubuntu"* || $(cat /etc/os-release | grep' +
-                              ' "PRETTY_NAME") == *"Debian"* ]]; then\n')
-                envfile.write('    export machine=unknownlinux\n')
-                envfile.write('fi\n')
-                envfile.write('export GPTUNEROOT=$PWD\n')
-                envfile.write('export MPIRUN={0}\n'.format
-                              (which(spec['mpi'].prefix.bin + '/mpirun')))
-                envfile.write('export proc=$(spack arch)\n')
-                envfile.write('export mpi={0}\n'.format(spec['mpi'].name))
-                envfile.write('export compiler={0}\n'.format(comp_name))
-                envfile.write('export nodes={0} \n'.format(self.nodes))
-                envfile.write('export cores={0} \n'.format(self.cores))
-                envfile.write('export ModuleEnv=$machine-$proc-$mpi-$compiler \n')
-                envfile.write('software_json=$(echo ",\\\"software_configuration\\\":' +
-                              '{\\\"' + spec['blas'].name +
-                              '\\\":{\\\"version_split\\\":' +
-                              ' [' + str(spec['blas'].versions).replace('.', ',') +
-                              ']},\\\"' + spec['mpi'].name +
-                              '\\\":{\\\"version_split\\\": [' +
-                              str(spec['mpi'].versions).replace('.', ',') + ']},\\\"' +
-                              spec['scalapack'].name +
-                              '\\\":{\\\"version_split\\\": [' +
-                              str(spec['scalapack'].versions).replace('.', ',') +
-                              ']},\\\"' +
-                              str(comp_name) + '\\\":{\\\"version_split\\\": [' +
-                              str(comp_version) + ']}}") \n')
-                envfile.write('loadable_software_json=$(echo ",\\\"loadable_software_' +
-                              'configurations\\\":{\\\"' + spec['blas'].name +
-                              '\\\":{\\\"version_split\\\": [' +
-                              str(spec['blas'].versions).replace('.', ',') +
-                              ']},\\\"' + spec['mpi'].name +
-                              '\\\":{\\\"version_split\\\": [' +
-                              str(spec['mpi'].versions).replace('.', ',') + ']},\\\"' +
-                              spec['scalapack'].name +
-                              '\\\":{\\\"version_split\\\": [' +
-                              str(spec['scalapack'].versions).replace('.', ',') +
-                              ']},\\\"' + str(comp_name) +
-                              '\\\":{\\\"version_split\\\": ['
-                              + str(comp_version) + ']}}") \n')
-                envfile.write('machine_json=$(echo ",\\\"machine_configuration\\\":' +
-                              '{\\\"machine_name\\\":\\\"$machine\\\",\\\"$proc\\\":' +
-                              '{\\\"nodes\\\":$nodes,\\\"cores\\\":$cores}}") \n')
-                envfile.write('loadable_machine_json=$(echo ",\\\"loadable_machine_' +
-                              'configurations\\\":{\\\"$machine\\\":{\\\"$proc\\\":' +
-                              '{\\\"nodes\\\":$nodes,\\\"cores\\\":$cores}}}") \n')
-
-        if '+app' in spec:
-            apps = ['GPTune-Demo', 'SuperLU_DIST', 'SuperLU_DIST_RCI',
-                    'Scalapack-PDGEQRF', 'Scalapack-PDGEQRF_RCI']
-        else:
-            apps = ['GPTune-Demo', 'Scalapack-PDGEQRF', 'Scalapack-PDGEQRF_RCI']
+            wd = join_path(test_dir, 'SuperLU_DIST')
+            self.run_test('rm', options=['-rf', 'superlu_dist'], work_dir=wd)
+            self.run_test('git', options=['clone', 'https://github.com/xiaoyeli/superlu_dist.git'], work_dir=wd)
+            self.run_test('mkdir', options=['-p',
+                                            'build'], work_dir=wd + '/superlu_dist')
+            self.run_test('mkdir', options=['-p', 'EXAMPLE'],
+                          work_dir=wd + '/superlu_dist/build')
+            self.run_test('cp', options=op, work_dir=wd + '/superlu_dist/build/EXAMPLE')
+
+        if '+hypre' in spec:
+            hypredriver = join_path(spec['hypre'].prefix.bin, 'ij')
+            op = ['-r', hypredriver, '.']
+            # copy superlu-dist executables to the correct place
+            wd = join_path(test_dir, 'Hypre')
+            self.run_test('rm', options=['-rf', 'hypre'], work_dir=wd)
+            self.run_test('git', options=['clone', 'https://github.com/hypre-space/hypre.git'], work_dir=wd)
+            self.run_test('cp', options=op, work_dir=wd + '/hypre/src/test/')
+
+        wd = self.test_suite.current_test_cache_dir
+        with open('{0}/run_env.sh'.format(wd), 'w') as envfile:
+            envfile.write('if [[ $NERSC_HOST = "cori" ]]; then\n')
+            envfile.write('    export machine=cori\n')
+            envfile.write('elif [[ $(uname -s) = "Darwin" ]]; then\n')
+            envfile.write('    export machine=mac\n')
+            envfile.write('elif [[ $(dnsdomainname) = ' +
+                          '"summit.olcf.ornl.gov" ]]; then\n')
+            envfile.write('    export machine=summit\n')
+            envfile.write('elif [[ $(cat /etc/os-release | grep "PRETTY_NAME") ==' +
+                          ' *"Ubuntu"* || $(cat /etc/os-release | grep' +
+                          ' "PRETTY_NAME") == *"Debian"* ]]; then\n')
+            envfile.write('    export machine=unknownlinux\n')
+            envfile.write('fi\n')
+            envfile.write('export GPTUNEROOT=$PWD\n')
+            envfile.write('export MPIRUN={0}\n'.format
+                          (which(spec['mpi'].prefix.bin + '/mpirun')))
+            envfile.write('export PYTHONPATH={0}:$PYTHONPATH\n'.format
+                          (site_packages_dir + '/gptune'))
+            envfile.write('export proc=$(spack arch)\n')
+            envfile.write('export mpi={0}\n'.format(spec['mpi'].name))
+            envfile.write('export compiler={0}\n'.format(comp_name))
+            envfile.write('export nodes={0} \n'.format(self.nodes))
+            envfile.write('export cores={0} \n'.format(self.cores))
+            envfile.write('export ModuleEnv=$machine-$proc-$mpi-$compiler \n')
+            envfile.write('software_json=$(echo ",\\\"software_configuration\\\":' +
+                          '{\\\"' + spec['blas'].name +
+                          '\\\":{\\\"version_split\\\":' +
+                          ' [' + str(spec['blas'].versions).replace('.', ',') +
+                          ']},\\\"' + spec['mpi'].name +
+                          '\\\":{\\\"version_split\\\": [' +
+                          str(spec['mpi'].versions).replace('.', ',') + ']},\\\"' +
+                          spec['scalapack'].name +
+                          '\\\":{\\\"version_split\\\": [' +
+                          str(spec['scalapack'].versions).replace('.', ',') +
+                          ']},\\\"' +
+                          str(comp_name) + '\\\":{\\\"version_split\\\": [' +
+                          str(comp_version) + ']}}") \n')
+            envfile.write('loadable_software_json=$(echo ",\\\"loadable_software_' +
+                          'configurations\\\":{\\\"' + spec['blas'].name +
+                          '\\\":{\\\"version_split\\\": [' +
+                          str(spec['blas'].versions).replace('.', ',') +
+                          ']},\\\"' + spec['mpi'].name +
+                          '\\\":{\\\"version_split\\\": [' +
+                          str(spec['mpi'].versions).replace('.', ',') + ']},\\\"' +
+                          spec['scalapack'].name +
+                          '\\\":{\\\"version_split\\\": [' +
+                          str(spec['scalapack'].versions).replace('.', ',') +
+                          ']},\\\"' + str(comp_name) +
+                          '\\\":{\\\"version_split\\\": ['
+                          + str(comp_version) + ']}}") \n')
+            envfile.write('machine_json=$(echo ",\\\"machine_configuration\\\":' +
+                          '{\\\"machine_name\\\":\\\"$machine\\\",\\\"$proc\\\":' +
+                          '{\\\"nodes\\\":$nodes,\\\"cores\\\":$cores}}") \n')
+            envfile.write('loadable_machine_json=$(echo ",\\\"loadable_machine_' +
+                          'configurations\\\":{\\\"$machine\\\":{\\\"$proc\\\":' +
+                          '{\\\"nodes\\\":$nodes,\\\"cores\\\":$cores}}}") \n')
+
+        # copy the environment configuration files to non-cache directories
+        op = ['run_env.sh', site_packages_dir + '/gptune/.']
+        self.run_test('cp', options=op, work_dir=wd)
+        op = ['run_env.sh', self.install_test_root + '/.']
+        self.run_test('cp', options=op, work_dir=wd)
+
+        apps = ['Scalapack-PDGEQRF_RCI']
+        if '+mpispawn' in spec:
+            apps = apps + ['GPTune-Demo', 'Scalapack-PDGEQRF']
+        if '+superlu' in spec:
+            apps = apps + ['SuperLU_DIST_RCI']
+            if '+mpispawn' in spec:
+                apps = apps + ['SuperLU_DIST']
+        if '+hypre' in spec:
+            if '+mpispawn' in spec:
+                apps = apps + ['Hypre']
 
         for app in apps:
-            with working_dir(join_path(test_dir, app), create=False):
-                # PDGEQRF with GPTune
-                self.run_test('bash', options=['run_examples.sh'], work_dir='.',
-                              purpose='gptune smoke test for {0}'.format(app))
+            wd = join_path(test_dir, app)
+            self.run_test('bash', options=['run_examples.sh'], work_dir=wd,
+                          purpose='gptune smoke test for {0}'.format(app))
diff --git a/var/spack/repos/builtin/packages/hypre/ij_gptune.patch b/var/spack/repos/builtin/packages/hypre/ij_gptune.patch
new file mode 100644
index 0000000000..36dbf44afb
--- /dev/null
+++ b/var/spack/repos/builtin/packages/hypre/ij_gptune.patch
@@ -0,0 +1,440 @@
+diff --git a/src/test/ij.c b/src/test/ij.c
+old mode 100644
+new mode 100755
+index fcda91898..c43dcf477
+--- a/src/test/ij.c
++++ b/src/test/ij.c
+@@ -85,10 +85,81 @@ extern HYPRE_Int hypre_FlexGMRESModifyPCDefault(void *precond_data, HYPRE_Int it
+ #endif
+ #define SECOND_TIME 0
+ 
++
++HYPRE_Int
++hypre_fPrintTiming(FILE* fp, const char     *heading,
++                   MPI_Comm        comm  )
++{
++   HYPRE_Int  ierr = 0;
++
++   HYPRE_Real  local_wall_time;
++   HYPRE_Real  local_cpu_time;
++   HYPRE_Real  wall_time;
++   HYPRE_Real  cpu_time;
++   HYPRE_Real  wall_mflops;
++   HYPRE_Real  cpu_mflops;
++
++   HYPRE_Int     i;
++   HYPRE_Int     myrank;
++
++   if (hypre_global_timing == NULL)
++      return ierr;
++
++   hypre_MPI_Comm_rank(comm, &myrank );
++
++   /* print heading */
++   if (myrank == 0)
++   {
++      hypre_fprintf(fp, "=============================================\n");
++      hypre_fprintf(fp, "%s:\n", heading);
++      hypre_fprintf(fp, "=============================================\n");
++   }
++
++   for (i = 0; i < (hypre_global_timing -> size); i++)
++   {
++      if (hypre_TimingNumRegs(i) > 0)
++      {
++         local_wall_time = hypre_TimingWallTime(i);
++         local_cpu_time  = hypre_TimingCPUTime(i);
++         hypre_MPI_Allreduce(&local_wall_time, &wall_time, 1,
++                       hypre_MPI_REAL, hypre_MPI_MAX, comm);
++         hypre_MPI_Allreduce(&local_cpu_time, &cpu_time, 1,
++                       hypre_MPI_REAL, hypre_MPI_MAX, comm);
++
++         if (myrank == 0)
++         {
++            hypre_fprintf(fp, "%s:\n", hypre_TimingName(i));
++
++            /* print wall clock info */
++            hypre_fprintf(fp, "  wall clock time = %f seconds\n", wall_time);
++            if (wall_time)
++               wall_mflops = hypre_TimingFLOPS(i) / wall_time / 1.0E6;
++            else
++               wall_mflops = 0.0;
++            hypre_fprintf(fp, "  wall MFLOPS     = %f\n", wall_mflops);
++
++            /* print CPU clock info */
++            hypre_fprintf(fp, "  cpu clock time  = %f seconds\n", cpu_time);
++            if (cpu_time)
++               cpu_mflops = hypre_TimingFLOPS(i) / cpu_time / 1.0E6;
++            else
++               cpu_mflops = 0.0;
++            hypre_fprintf(fp, "  cpu MFLOPS      = %f\n\n", cpu_mflops);
++         }
++      }
++   }
++
++   return ierr;
++}
++
++
++
++
+ hypre_int
+ main( hypre_int argc,
+       char *argv[] )
+ {
++   FILE *fplog;
+    HYPRE_Int           arg_index;
+    HYPRE_Int           print_usage;
+    HYPRE_Int           sparsity_known = 0;
+@@ -414,7 +485,9 @@ main( hypre_int argc,
+    size_t mempool_max_cached_bytes = 2000LL * 1024 * 1024;
+ 
+    /* Initialize MPI */
+-   hypre_MPI_Init(&argc, &argv);
++   hypre_MPI_Init(&argc, &argv);MPI_Comm parent; MPI_Comm_get_parent(&parent);
++      
++
+ 
+    hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
+    hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid );
+@@ -481,6 +554,11 @@ main( hypre_int argc,
+          build_matrix_type      = 3;
+          build_matrix_arg_index = arg_index;
+       }
++      else if ( strcmp(argv[arg_index], "-logfile") == 0 )
++      {
++         arg_index++;
++         fplog = fopen(argv[arg_index++], "w");
++      }      
+       else if ( strcmp(argv[arg_index], "-27pt") == 0 )
+       {
+          arg_index++;
+@@ -2068,7 +2146,7 @@ main( hypre_int argc,
+    HYPRE_Init();
+ 
+    hypre_EndTiming(time_index);
+-   hypre_PrintTiming("Hypre init times", hypre_MPI_COMM_WORLD);
++   hypre_fPrintTiming(fplog, "Hypre init times", hypre_MPI_COMM_WORLD);
+    hypre_FinalizeTiming(time_index);
+    hypre_ClearTiming();
+ 
+@@ -2182,7 +2260,7 @@ main( hypre_int argc,
+       local_num_cols = (HYPRE_Int)(last_local_col - first_local_col + 1);
+    }
+    hypre_EndTiming(time_index);
+-   hypre_PrintTiming("Generate Matrix", hypre_MPI_COMM_WORLD);
++   hypre_fPrintTiming(fplog, "Generate Matrix", hypre_MPI_COMM_WORLD);
+    hypre_FinalizeTiming(time_index);
+    hypre_ClearTiming();
+ 
+@@ -2404,7 +2482,7 @@ main( hypre_int argc,
+       ierr += HYPRE_IJMatrixAssemble( ij_A );
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("IJ Matrix Setup", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "IJ Matrix Setup", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -3032,7 +3110,7 @@ main( hypre_int argc,
+    }
+ 
+    hypre_EndTiming(time_index);
+-   hypre_PrintTiming("IJ Vector Setup", hypre_MPI_COMM_WORLD);
++   hypre_fPrintTiming(fplog, "IJ Vector Setup", hypre_MPI_COMM_WORLD);
+    hypre_FinalizeTiming(time_index);
+    hypre_ClearTiming();
+ 
+@@ -3124,7 +3202,7 @@ main( hypre_int argc,
+ #endif
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("MatVec Test", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "MatVec Test", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -3190,7 +3268,7 @@ main( hypre_int argc,
+       HYPRE_ParCSRHybridSetup(amg_solver, parcsr_A, b, x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -3200,7 +3278,7 @@ main( hypre_int argc,
+       HYPRE_ParCSRHybridSolve(amg_solver, parcsr_A, b, x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -3422,7 +3500,7 @@ main( hypre_int argc,
+ #endif
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -3448,7 +3526,7 @@ main( hypre_int argc,
+ #endif
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -3638,7 +3716,7 @@ main( hypre_int argc,
+       HYPRE_BoomerAMGSetup(amg_solver, parcsr_A, b, x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -3648,7 +3726,7 @@ main( hypre_int argc,
+       HYPRE_BoomerAMGSolve(amg_solver, parcsr_A, b, x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -4112,7 +4190,7 @@ main( hypre_int argc,
+       HYPRE_PCGSetup(pcg_solver, (HYPRE_Matrix)parcsr_A,
+                      (HYPRE_Vector)b, (HYPRE_Vector)x);
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -4123,7 +4201,7 @@ main( hypre_int argc,
+                      (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -4140,7 +4218,7 @@ main( hypre_int argc,
+                      (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -4151,7 +4229,7 @@ main( hypre_int argc,
+                      (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ #endif
+@@ -4550,7 +4628,7 @@ main( hypre_int argc,
+           *                     (HYPRE_Vector)b, (HYPRE_Vector)x); */
+ 
+          hypre_EndTiming(time_index);
+-         hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
++         hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD);
+          hypre_FinalizeTiming(time_index);
+          hypre_ClearTiming();
+ 
+@@ -4610,7 +4688,7 @@ main( hypre_int argc,
+          HYPRE_LOBPCGSolve(lobpcg_solver, constraints, eigenvectors, eigenvalues );
+ 
+          hypre_EndTiming(time_index);
+-         hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
++         hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD);
+          hypre_FinalizeTiming(time_index);
+          hypre_ClearTiming();
+ 
+@@ -4942,7 +5020,7 @@ main( hypre_int argc,
+                                (HYPRE_Vector)x);
+ 
+          hypre_EndTiming(time_index);
+-         hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
++         hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD);
+          hypre_FinalizeTiming(time_index);
+          hypre_ClearTiming();
+ 
+@@ -4980,7 +5058,7 @@ main( hypre_int argc,
+          HYPRE_LOBPCGSolve(pcg_solver, constraints, eigenvectors, eigenvalues);
+ 
+          hypre_EndTiming(time_index);
+-         hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
++         hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD);
+          hypre_FinalizeTiming(time_index);
+          hypre_ClearTiming();
+ 
+@@ -5490,7 +5568,7 @@ main( hypre_int argc,
+          (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -5501,7 +5579,7 @@ main( hypre_int argc,
+          (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -5723,7 +5801,7 @@ main( hypre_int argc,
+          (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -5734,7 +5812,7 @@ main( hypre_int argc,
+          (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -6060,7 +6138,7 @@ main( hypre_int argc,
+          (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -6071,7 +6149,7 @@ main( hypre_int argc,
+          (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -6397,7 +6475,7 @@ main( hypre_int argc,
+                           (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -6408,7 +6486,7 @@ main( hypre_int argc,
+                           (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -6723,7 +6801,7 @@ main( hypre_int argc,
+                          (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -6734,7 +6812,7 @@ main( hypre_int argc,
+                          (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -6930,7 +7008,7 @@ main( hypre_int argc,
+                       (HYPRE_Vector)x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -6941,7 +7019,7 @@ main( hypre_int argc,
+                       (HYPRE_Vector)x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -7073,7 +7151,7 @@ main( hypre_int argc,
+       HYPRE_MGRSetup(mgr_solver, parcsr_A, b, x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -7084,7 +7162,7 @@ main( hypre_int argc,
+       HYPRE_MGRSolve(mgr_solver, parcsr_A, b, x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -7170,7 +7248,7 @@ main( hypre_int argc,
+       HYPRE_ILUSetup(ilu_solver, parcsr_A, b, x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -7181,7 +7259,7 @@ main( hypre_int argc,
+       HYPRE_ILUSolve(ilu_solver, parcsr_A, b, x);
+ 
+       hypre_EndTiming(time_index);
+-      hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
++      hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD);
+       hypre_FinalizeTiming(time_index);
+       hypre_ClearTiming();
+ 
+@@ -7286,7 +7364,13 @@ main( hypre_int argc,
+    /* Finalize Hypre */
+    HYPRE_Finalize();
+ 
++
++   fflush(fplog);
++   fclose(fplog);
++
+    /* Finalize MPI */
++   if(parent!=MPI_COMM_NULL)
++   MPI_Comm_disconnect(&parent);
+    hypre_MPI_Finalize();
+ 
+    /* when using cuda-memcheck --leak-check full, uncomment this */
diff --git a/var/spack/repos/builtin/packages/hypre/package.py b/var/spack/repos/builtin/packages/hypre/package.py
index dff8419df8..d4dc0d6a39 100644
--- a/var/spack/repos/builtin/packages/hypre/package.py
+++ b/var/spack/repos/builtin/packages/hypre/package.py
@@ -68,6 +68,11 @@ class Hypre(AutotoolsPackage, CudaPackage):
     variant('unified-memory', default=False, description='Use unified memory')
     variant('fortran', default=True,
             description='Enables fortran bindings')
+    variant('gptune', default=False,
+            description='Add the GPTune hookup code')
+
+    # Patch to add gptune hookup codes
+    patch('ij_gptune.patch', when='+gptune@2.19.0')
 
     # Patch to add ppc64le in config.guess
     patch('ibm-ppc64le.patch', when='@:2.11.1')
@@ -88,6 +93,7 @@ class Hypre(AutotoolsPackage, CudaPackage):
 
     conflicts('+cuda', when='+int64')
     conflicts('+unified-memory', when='~cuda')
+    conflicts('+gptune', when='~mpi')
 
     # Patch to build shared libraries on Darwin does not apply to
     # versions before 2.13.0
@@ -228,6 +234,10 @@ class Hypre(AutotoolsPackage, CudaPackage):
                 sstruct('-in', 'test/sstruct.in.default', '-solver', '40',
                         '-rhsone')
             make("install")
+            if '+gptune' in self.spec:
+                make("test")
+                self.run_test('mkdir', options=['-p', self.prefix.bin])
+                self.run_test('cp', options=['test/ij', self.prefix.bin + '/.'])
 
     extra_install_tests = join_path('src', 'examples')
 
diff --git a/var/spack/repos/builtin/packages/superlu-dist/CMAKE_INSTALL_LIBDIR.patch b/var/spack/repos/builtin/packages/superlu-dist/CMAKE_INSTALL_LIBDIR.patch
new file mode 100644
index 0000000000..d3d8311863
--- /dev/null
+++ b/var/spack/repos/builtin/packages/superlu-dist/CMAKE_INSTALL_LIBDIR.patch
@@ -0,0 +1,59 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 2b2fdf2..f89cf4c 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -456,15 +456,6 @@ endif()
+ #target_compile_features(SuperLU_DIST PUBLIC cxx_std_11)
+ 
+ # Generate various configure files with proper definitions
+-# configure_file(${CMAKE_SOURCE_DIR}/make.inc.in ${CMAKE_BINARY_DIR}/make.inc)
+-configure_file(${SuperLU_DIST_SOURCE_DIR}/make.inc.in ${SuperLU_DIST_SOURCE_DIR}/make.inc)
+-
+-configure_file(${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h.in ${SuperLU_DIST_BINARY_DIR}/SRC/superlu_dist_config.h)
+-configure_file(${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h.in ${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h)
+-
+-# Following is to configure a file for FORTRAN code
+-configure_file(${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h.in ${SuperLU_DIST_BINARY_DIR}/FORTRAN/superlu_dist_config.h)
+-
+ 
+ # Add pkg-config support
+ if(IS_ABSOLUTE ${CMAKE_INSTALL_LIBDIR})
+@@ -476,4 +467,13 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/superlu_dist.pc.in ${CMAKE_CURRENT_BI
+ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/superlu_dist.pc
+ 	DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
+ 
++# configure_file(${CMAKE_SOURCE_DIR}/make.inc.in ${CMAKE_BINARY_DIR}/make.inc)
++configure_file(${SuperLU_DIST_SOURCE_DIR}/make.inc.in ${SuperLU_DIST_SOURCE_DIR}/make.inc)
++
++configure_file(${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h.in ${SuperLU_DIST_BINARY_DIR}/SRC/superlu_dist_config.h)
++configure_file(${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h.in ${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h)
++
++# Following is to configure a file for FORTRAN code
++configure_file(${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h.in ${SuperLU_DIST_BINARY_DIR}/FORTRAN/superlu_dist_config.h)
++
+ #message("MPI_Fortran_LINK_FLAGS '${MPI_Fortran_LINK_FLAGS}'")
+diff --git a/make.inc.in b/make.inc.in
+index 0beb461..860c0bf 100644
+--- a/make.inc.in
++++ b/make.inc.in
+@@ -18,7 +18,8 @@
+ #
+ SuperLUroot = ${CMAKE_INSTALL_PREFIX}
+ #DSUPERLULIB = $(SuperLUroot)/SRC/${PROJECT_NAME_LIB_EXPORT}
+-DSUPERLULIB = $(SuperLUroot)/@CMAKE_INSTALL_LIBDIR@/${PROJECT_NAME_LIB_EXPORT}
++#DSUPERLULIB = $(SuperLUroot)/@CMAKE_INSTALL_LIBDIR@/${PROJECT_NAME_LIB_EXPORT}
++DSUPERLULIB = @pkgconfig_libdir@/${PROJECT_NAME_LIB_EXPORT}
+ INCLUDEDIR  = $(SuperLUroot)/@CMAKE_INSTALL_INCLUDEDIR@
+ 
+ XSDK_INDEX_SIZE = @XSDK_INDEX_SIZE@
+@@ -29,7 +30,8 @@ HAVE_CUDA       = @HAVE_CUDA@
+ 
+ XSDK_ENABLE_Fortran = @XSDK_ENABLE_Fortran@
+ ifeq ($(XSDK_ENABLE_Fortran),ON)
+-  DFORTRANLIB = $(SuperLUroot)/@CMAKE_INSTALL_LIBDIR@/${PROJECT_NAME_LIB_FORTRAN}
++#  DFORTRANLIB = $(SuperLUroot)/@CMAKE_INSTALL_LIBDIR@/${PROJECT_NAME_LIB_FORTRAN}
++  DFORTRANLIB = @pkgconfig_libdir@/${PROJECT_NAME_LIB_FORTRAN}
+   LIBS = $(DFORTRANLIB) $(DSUPERLULIB) ${BLAS_LIB_EXPORT} -lm
+   LIBS += ${EXTRA_FLIB_EXPORT}
+ else
+
diff --git a/var/spack/repos/builtin/packages/superlu-dist/package.py b/var/spack/repos/builtin/packages/superlu-dist/package.py
index 9677446893..7af879c45f 100644
--- a/var/spack/repos/builtin/packages/superlu-dist/package.py
+++ b/var/spack/repos/builtin/packages/superlu-dist/package.py
@@ -20,6 +20,7 @@ class SuperluDist(CMakePackage, CudaPackage, ROCmPackage):
 
     version('develop', branch='master')
     version('amd', branch='amd')
+    version('7.2.0', sha256='20b60bd8a3d88031c9ce6511ae9700b7a8dcf12e2fd704e74b1af762b3468b8c')
     version('7.1.1', sha256='558053b3d4a56eb661c4f04d4fcab6604018ce5db97115394c161b56c9c278ff')
     version('7.1.0', sha256='edbea877562be95fb22c7de1ff484f18685bec4baa8e4f703c414d3c035d4a66')
     version('6.4.0', sha256='cb9c0b2ba4c28e5ed5817718ba19ae1dd63ccd30bc44c8b8252b54f5f04a44cc')
@@ -58,6 +59,7 @@ class SuperluDist(CMakePackage, CudaPackage, ROCmPackage):
     patch('xl-611.patch', when='@:6.1.1 %xl')
     patch('xl-611.patch', when='@:6.1.1 %xl_r')
     patch('superlu-cray-ftn-case.patch', when='@7.1.1 %cce')
+    patch('CMAKE_INSTALL_LIBDIR.patch', when='@7.0.0:7.2.0')
 
     def cmake_args(self):
         spec = self.spec