From 80f92cfddeb5aa835ec5853596127bbf5cbc9318 Mon Sep 17 00:00:00 2001 From: liuyangzhuan Date: Fri, 28 Jan 2022 01:31:56 -0800 Subject: gptune: add variants (mpispawn, hypre) (#27733) Co-authored-by: Adam J. Stewart --- var/spack/repos/builtin/packages/gptune/package.py | 201 ++++++---- .../repos/builtin/packages/hypre/ij_gptune.patch | 440 +++++++++++++++++++++ var/spack/repos/builtin/packages/hypre/package.py | 10 + .../superlu-dist/CMAKE_INSTALL_LIBDIR.patch | 59 +++ .../repos/builtin/packages/superlu-dist/package.py | 2 + 5 files changed, 626 insertions(+), 86 deletions(-) create mode 100644 var/spack/repos/builtin/packages/hypre/ij_gptune.patch create mode 100644 var/spack/repos/builtin/packages/superlu-dist/CMAKE_INSTALL_LIBDIR.patch diff --git a/var/spack/repos/builtin/packages/gptune/package.py b/var/spack/repos/builtin/packages/gptune/package.py index eeebd6f40d..d96c86b074 100644 --- a/var/spack/repos/builtin/packages/gptune/package.py +++ b/var/spack/repos/builtin/packages/gptune/package.py @@ -18,7 +18,9 @@ class Gptune(CMakePackage): version('master', branch='master') - variant('app', default=False, description='Build all HPC application examples') + variant('superlu', default=False, description='Build the SuperLU_DIST example') + variant('hypre', default=False, description='Build the Hypre example') + variant('mpispawn', default=True, description='MPI spawning-based interface') depends_on('mpi', type=('build', 'link', 'run')) depends_on('cmake@3.3:', type='build') @@ -48,9 +50,14 @@ class Gptune(CMakePackage): depends_on('pygmo', type=('build', 'run')) depends_on('openturns', type=('build', 'run')) - depends_on('superlu-dist@develop', when='+app', type=('build', 'run')) + depends_on('superlu-dist@develop', when='+superlu', type=('build', 'run')) + depends_on('hypre+gptune@2.19.0', when='+hypre', type=('build', 'run')) - conflicts('openmpi@:3') + depends_on('openmpi@4:', when='+mpispawn', type=('build', 'run')) + conflicts('mpich', when='+mpispawn') + conflicts('spectrum-mpi', when='+mpispawn') + conflicts('cray-mpich', when='+mpispawn') + conflicts('gcc@:7') def cmake_args(self): spec = self.spec @@ -59,6 +66,7 @@ class Gptune(CMakePackage): fc_flags.append('-fallow-argument-mismatch') args = [ + '-DGPTUNE_INSTALL_PATH=%s' % site_packages_dir, '-DTPL_BLAS_LIBRARIES=%s' % spec['blas'].libs.joined(";"), '-DTPL_LAPACK_LIBRARIES=%s' % spec['lapack'].libs.joined(";"), '-DTPL_SCALAPACK_LIBRARIES=%s' % spec['scalapack']. @@ -81,96 +89,117 @@ class Gptune(CMakePackage): def cache_test_sources(self): """Copy the example source files after the package is installed to an install test subdirectory for use during `spack test run`.""" - self.cache_extra_test_sources([self.examples_src_dir, self.src_dir]) + self.cache_extra_test_sources([self.examples_src_dir]) + + def setup_run_environment(self, env): + env.set('GPTUNE_INSTALL_PATH', site_packages_dir) def test(self): spec = self.spec comp_name = self.compiler.name comp_version = str(self.compiler.version).replace('.', ',') - test_dir = join_path(self.install_test_root, self.examples_src_dir) + test_dir = join_path(self.test_suite.current_test_cache_dir, + self.examples_src_dir) - if '+app' in spec: - superludriver = join_path(spec['superlu-dist'].prefix.bin, 'pddrive_spawn') + if '+superlu' in spec: + superludriver = join_path(spec['superlu-dist'].prefix.lib, + 'EXAMPLE/pddrive_spawn') op = ['-r', superludriver, '.'] # copy superlu-dist executables to the correct place - with working_dir(join_path(test_dir, 'SuperLU_DIST'), create=False): - self.run_test('rm', options=['-rf', 'superlu_dist'], work_dir='.') - self.run_test('git', options=['clone', 'https://github.com/xiaoyeli/superlu_dist.git'], work_dir='.') - self.run_test('mkdir', options=['-p', - 'build'], work_dir='./superlu_dist') - self.run_test('mkdir', options=['-p', 'EXAMPLE'], - work_dir='./superlu_dist/build') - self.run_test('cp', options=op, work_dir='./superlu_dist/build/EXAMPLE') - - with working_dir(self.install_test_root, create=False): - cdir = join_path(self.prefix, 'gptuneclcm') - self.run_test('cp', options=['-r', cdir, '.'], work_dir='.') - self.run_test('rm', options=['-rf', 'build'], work_dir='.') - self.run_test('mv', options=['gptuneclcm', 'build'], work_dir='.') - - with open('{0}/run_env.sh'.format(self.install_test_root), 'w') as envfile: - envfile.write('if [[ $NERSC_HOST = "cori" ]]; then\n') - envfile.write(' export machine=cori\n') - envfile.write('elif [[ $(uname -s) = "Darwin" ]]; then\n') - envfile.write(' export machine=mac\n') - envfile.write('elif [[ $(dnsdomainname) = ' + - '"summit.olcf.ornl.gov" ]]; then\n') - envfile.write(' export machine=summit\n') - envfile.write('elif [[ $(cat /etc/os-release | grep "PRETTY_NAME") ==' + - ' *"Ubuntu"* || $(cat /etc/os-release | grep' + - ' "PRETTY_NAME") == *"Debian"* ]]; then\n') - envfile.write(' export machine=unknownlinux\n') - envfile.write('fi\n') - envfile.write('export GPTUNEROOT=$PWD\n') - envfile.write('export MPIRUN={0}\n'.format - (which(spec['mpi'].prefix.bin + '/mpirun'))) - envfile.write('export proc=$(spack arch)\n') - envfile.write('export mpi={0}\n'.format(spec['mpi'].name)) - envfile.write('export compiler={0}\n'.format(comp_name)) - envfile.write('export nodes={0} \n'.format(self.nodes)) - envfile.write('export cores={0} \n'.format(self.cores)) - envfile.write('export ModuleEnv=$machine-$proc-$mpi-$compiler \n') - envfile.write('software_json=$(echo ",\\\"software_configuration\\\":' + - '{\\\"' + spec['blas'].name + - '\\\":{\\\"version_split\\\":' + - ' [' + str(spec['blas'].versions).replace('.', ',') + - ']},\\\"' + spec['mpi'].name + - '\\\":{\\\"version_split\\\": [' + - str(spec['mpi'].versions).replace('.', ',') + ']},\\\"' + - spec['scalapack'].name + - '\\\":{\\\"version_split\\\": [' + - str(spec['scalapack'].versions).replace('.', ',') + - ']},\\\"' + - str(comp_name) + '\\\":{\\\"version_split\\\": [' + - str(comp_version) + ']}}") \n') - envfile.write('loadable_software_json=$(echo ",\\\"loadable_software_' + - 'configurations\\\":{\\\"' + spec['blas'].name + - '\\\":{\\\"version_split\\\": [' + - str(spec['blas'].versions).replace('.', ',') + - ']},\\\"' + spec['mpi'].name + - '\\\":{\\\"version_split\\\": [' + - str(spec['mpi'].versions).replace('.', ',') + ']},\\\"' + - spec['scalapack'].name + - '\\\":{\\\"version_split\\\": [' + - str(spec['scalapack'].versions).replace('.', ',') + - ']},\\\"' + str(comp_name) + - '\\\":{\\\"version_split\\\": [' - + str(comp_version) + ']}}") \n') - envfile.write('machine_json=$(echo ",\\\"machine_configuration\\\":' + - '{\\\"machine_name\\\":\\\"$machine\\\",\\\"$proc\\\":' + - '{\\\"nodes\\\":$nodes,\\\"cores\\\":$cores}}") \n') - envfile.write('loadable_machine_json=$(echo ",\\\"loadable_machine_' + - 'configurations\\\":{\\\"$machine\\\":{\\\"$proc\\\":' + - '{\\\"nodes\\\":$nodes,\\\"cores\\\":$cores}}}") \n') - - if '+app' in spec: - apps = ['GPTune-Demo', 'SuperLU_DIST', 'SuperLU_DIST_RCI', - 'Scalapack-PDGEQRF', 'Scalapack-PDGEQRF_RCI'] - else: - apps = ['GPTune-Demo', 'Scalapack-PDGEQRF', 'Scalapack-PDGEQRF_RCI'] + wd = join_path(test_dir, 'SuperLU_DIST') + self.run_test('rm', options=['-rf', 'superlu_dist'], work_dir=wd) + self.run_test('git', options=['clone', 'https://github.com/xiaoyeli/superlu_dist.git'], work_dir=wd) + self.run_test('mkdir', options=['-p', + 'build'], work_dir=wd + '/superlu_dist') + self.run_test('mkdir', options=['-p', 'EXAMPLE'], + work_dir=wd + '/superlu_dist/build') + self.run_test('cp', options=op, work_dir=wd + '/superlu_dist/build/EXAMPLE') + + if '+hypre' in spec: + hypredriver = join_path(spec['hypre'].prefix.bin, 'ij') + op = ['-r', hypredriver, '.'] + # copy superlu-dist executables to the correct place + wd = join_path(test_dir, 'Hypre') + self.run_test('rm', options=['-rf', 'hypre'], work_dir=wd) + self.run_test('git', options=['clone', 'https://github.com/hypre-space/hypre.git'], work_dir=wd) + self.run_test('cp', options=op, work_dir=wd + '/hypre/src/test/') + + wd = self.test_suite.current_test_cache_dir + with open('{0}/run_env.sh'.format(wd), 'w') as envfile: + envfile.write('if [[ $NERSC_HOST = "cori" ]]; then\n') + envfile.write(' export machine=cori\n') + envfile.write('elif [[ $(uname -s) = "Darwin" ]]; then\n') + envfile.write(' export machine=mac\n') + envfile.write('elif [[ $(dnsdomainname) = ' + + '"summit.olcf.ornl.gov" ]]; then\n') + envfile.write(' export machine=summit\n') + envfile.write('elif [[ $(cat /etc/os-release | grep "PRETTY_NAME") ==' + + ' *"Ubuntu"* || $(cat /etc/os-release | grep' + + ' "PRETTY_NAME") == *"Debian"* ]]; then\n') + envfile.write(' export machine=unknownlinux\n') + envfile.write('fi\n') + envfile.write('export GPTUNEROOT=$PWD\n') + envfile.write('export MPIRUN={0}\n'.format + (which(spec['mpi'].prefix.bin + '/mpirun'))) + envfile.write('export PYTHONPATH={0}:$PYTHONPATH\n'.format + (site_packages_dir + '/gptune')) + envfile.write('export proc=$(spack arch)\n') + envfile.write('export mpi={0}\n'.format(spec['mpi'].name)) + envfile.write('export compiler={0}\n'.format(comp_name)) + envfile.write('export nodes={0} \n'.format(self.nodes)) + envfile.write('export cores={0} \n'.format(self.cores)) + envfile.write('export ModuleEnv=$machine-$proc-$mpi-$compiler \n') + envfile.write('software_json=$(echo ",\\\"software_configuration\\\":' + + '{\\\"' + spec['blas'].name + + '\\\":{\\\"version_split\\\":' + + ' [' + str(spec['blas'].versions).replace('.', ',') + + ']},\\\"' + spec['mpi'].name + + '\\\":{\\\"version_split\\\": [' + + str(spec['mpi'].versions).replace('.', ',') + ']},\\\"' + + spec['scalapack'].name + + '\\\":{\\\"version_split\\\": [' + + str(spec['scalapack'].versions).replace('.', ',') + + ']},\\\"' + + str(comp_name) + '\\\":{\\\"version_split\\\": [' + + str(comp_version) + ']}}") \n') + envfile.write('loadable_software_json=$(echo ",\\\"loadable_software_' + + 'configurations\\\":{\\\"' + spec['blas'].name + + '\\\":{\\\"version_split\\\": [' + + str(spec['blas'].versions).replace('.', ',') + + ']},\\\"' + spec['mpi'].name + + '\\\":{\\\"version_split\\\": [' + + str(spec['mpi'].versions).replace('.', ',') + ']},\\\"' + + spec['scalapack'].name + + '\\\":{\\\"version_split\\\": [' + + str(spec['scalapack'].versions).replace('.', ',') + + ']},\\\"' + str(comp_name) + + '\\\":{\\\"version_split\\\": [' + + str(comp_version) + ']}}") \n') + envfile.write('machine_json=$(echo ",\\\"machine_configuration\\\":' + + '{\\\"machine_name\\\":\\\"$machine\\\",\\\"$proc\\\":' + + '{\\\"nodes\\\":$nodes,\\\"cores\\\":$cores}}") \n') + envfile.write('loadable_machine_json=$(echo ",\\\"loadable_machine_' + + 'configurations\\\":{\\\"$machine\\\":{\\\"$proc\\\":' + + '{\\\"nodes\\\":$nodes,\\\"cores\\\":$cores}}}") \n') + + # copy the environment configuration files to non-cache directories + op = ['run_env.sh', site_packages_dir + '/gptune/.'] + self.run_test('cp', options=op, work_dir=wd) + op = ['run_env.sh', self.install_test_root + '/.'] + self.run_test('cp', options=op, work_dir=wd) + + apps = ['Scalapack-PDGEQRF_RCI'] + if '+mpispawn' in spec: + apps = apps + ['GPTune-Demo', 'Scalapack-PDGEQRF'] + if '+superlu' in spec: + apps = apps + ['SuperLU_DIST_RCI'] + if '+mpispawn' in spec: + apps = apps + ['SuperLU_DIST'] + if '+hypre' in spec: + if '+mpispawn' in spec: + apps = apps + ['Hypre'] for app in apps: - with working_dir(join_path(test_dir, app), create=False): - # PDGEQRF with GPTune - self.run_test('bash', options=['run_examples.sh'], work_dir='.', - purpose='gptune smoke test for {0}'.format(app)) + wd = join_path(test_dir, app) + self.run_test('bash', options=['run_examples.sh'], work_dir=wd, + purpose='gptune smoke test for {0}'.format(app)) diff --git a/var/spack/repos/builtin/packages/hypre/ij_gptune.patch b/var/spack/repos/builtin/packages/hypre/ij_gptune.patch new file mode 100644 index 0000000000..36dbf44afb --- /dev/null +++ b/var/spack/repos/builtin/packages/hypre/ij_gptune.patch @@ -0,0 +1,440 @@ +diff --git a/src/test/ij.c b/src/test/ij.c +old mode 100644 +new mode 100755 +index fcda91898..c43dcf477 +--- a/src/test/ij.c ++++ b/src/test/ij.c +@@ -85,10 +85,81 @@ extern HYPRE_Int hypre_FlexGMRESModifyPCDefault(void *precond_data, HYPRE_Int it + #endif + #define SECOND_TIME 0 + ++ ++HYPRE_Int ++hypre_fPrintTiming(FILE* fp, const char *heading, ++ MPI_Comm comm ) ++{ ++ HYPRE_Int ierr = 0; ++ ++ HYPRE_Real local_wall_time; ++ HYPRE_Real local_cpu_time; ++ HYPRE_Real wall_time; ++ HYPRE_Real cpu_time; ++ HYPRE_Real wall_mflops; ++ HYPRE_Real cpu_mflops; ++ ++ HYPRE_Int i; ++ HYPRE_Int myrank; ++ ++ if (hypre_global_timing == NULL) ++ return ierr; ++ ++ hypre_MPI_Comm_rank(comm, &myrank ); ++ ++ /* print heading */ ++ if (myrank == 0) ++ { ++ hypre_fprintf(fp, "=============================================\n"); ++ hypre_fprintf(fp, "%s:\n", heading); ++ hypre_fprintf(fp, "=============================================\n"); ++ } ++ ++ for (i = 0; i < (hypre_global_timing -> size); i++) ++ { ++ if (hypre_TimingNumRegs(i) > 0) ++ { ++ local_wall_time = hypre_TimingWallTime(i); ++ local_cpu_time = hypre_TimingCPUTime(i); ++ hypre_MPI_Allreduce(&local_wall_time, &wall_time, 1, ++ hypre_MPI_REAL, hypre_MPI_MAX, comm); ++ hypre_MPI_Allreduce(&local_cpu_time, &cpu_time, 1, ++ hypre_MPI_REAL, hypre_MPI_MAX, comm); ++ ++ if (myrank == 0) ++ { ++ hypre_fprintf(fp, "%s:\n", hypre_TimingName(i)); ++ ++ /* print wall clock info */ ++ hypre_fprintf(fp, " wall clock time = %f seconds\n", wall_time); ++ if (wall_time) ++ wall_mflops = hypre_TimingFLOPS(i) / wall_time / 1.0E6; ++ else ++ wall_mflops = 0.0; ++ hypre_fprintf(fp, " wall MFLOPS = %f\n", wall_mflops); ++ ++ /* print CPU clock info */ ++ hypre_fprintf(fp, " cpu clock time = %f seconds\n", cpu_time); ++ if (cpu_time) ++ cpu_mflops = hypre_TimingFLOPS(i) / cpu_time / 1.0E6; ++ else ++ cpu_mflops = 0.0; ++ hypre_fprintf(fp, " cpu MFLOPS = %f\n\n", cpu_mflops); ++ } ++ } ++ } ++ ++ return ierr; ++} ++ ++ ++ ++ + hypre_int + main( hypre_int argc, + char *argv[] ) + { ++ FILE *fplog; + HYPRE_Int arg_index; + HYPRE_Int print_usage; + HYPRE_Int sparsity_known = 0; +@@ -414,7 +485,9 @@ main( hypre_int argc, + size_t mempool_max_cached_bytes = 2000LL * 1024 * 1024; + + /* Initialize MPI */ +- hypre_MPI_Init(&argc, &argv); ++ hypre_MPI_Init(&argc, &argv);MPI_Comm parent; MPI_Comm_get_parent(&parent); ++ ++ + + hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs ); + hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid ); +@@ -481,6 +554,11 @@ main( hypre_int argc, + build_matrix_type = 3; + build_matrix_arg_index = arg_index; + } ++ else if ( strcmp(argv[arg_index], "-logfile") == 0 ) ++ { ++ arg_index++; ++ fplog = fopen(argv[arg_index++], "w"); ++ } + else if ( strcmp(argv[arg_index], "-27pt") == 0 ) + { + arg_index++; +@@ -2068,7 +2146,7 @@ main( hypre_int argc, + HYPRE_Init(); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Hypre init times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Hypre init times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -2182,7 +2260,7 @@ main( hypre_int argc, + local_num_cols = (HYPRE_Int)(last_local_col - first_local_col + 1); + } + hypre_EndTiming(time_index); +- hypre_PrintTiming("Generate Matrix", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Generate Matrix", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -2404,7 +2482,7 @@ main( hypre_int argc, + ierr += HYPRE_IJMatrixAssemble( ij_A ); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("IJ Matrix Setup", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "IJ Matrix Setup", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -3032,7 +3110,7 @@ main( hypre_int argc, + } + + hypre_EndTiming(time_index); +- hypre_PrintTiming("IJ Vector Setup", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "IJ Vector Setup", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -3124,7 +3202,7 @@ main( hypre_int argc, + #endif + + hypre_EndTiming(time_index); +- hypre_PrintTiming("MatVec Test", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "MatVec Test", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -3190,7 +3268,7 @@ main( hypre_int argc, + HYPRE_ParCSRHybridSetup(amg_solver, parcsr_A, b, x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -3200,7 +3278,7 @@ main( hypre_int argc, + HYPRE_ParCSRHybridSolve(amg_solver, parcsr_A, b, x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -3422,7 +3500,7 @@ main( hypre_int argc, + #endif + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -3448,7 +3526,7 @@ main( hypre_int argc, + #endif + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -3638,7 +3716,7 @@ main( hypre_int argc, + HYPRE_BoomerAMGSetup(amg_solver, parcsr_A, b, x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -3648,7 +3726,7 @@ main( hypre_int argc, + HYPRE_BoomerAMGSolve(amg_solver, parcsr_A, b, x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -4112,7 +4190,7 @@ main( hypre_int argc, + HYPRE_PCGSetup(pcg_solver, (HYPRE_Matrix)parcsr_A, + (HYPRE_Vector)b, (HYPRE_Vector)x); + hypre_EndTiming(time_index); +- hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -4123,7 +4201,7 @@ main( hypre_int argc, + (HYPRE_Vector)b, (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -4140,7 +4218,7 @@ main( hypre_int argc, + (HYPRE_Vector)b, (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -4151,7 +4229,7 @@ main( hypre_int argc, + (HYPRE_Vector)b, (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + #endif +@@ -4550,7 +4628,7 @@ main( hypre_int argc, + * (HYPRE_Vector)b, (HYPRE_Vector)x); */ + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -4610,7 +4688,7 @@ main( hypre_int argc, + HYPRE_LOBPCGSolve(lobpcg_solver, constraints, eigenvectors, eigenvalues ); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -4942,7 +5020,7 @@ main( hypre_int argc, + (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -4980,7 +5058,7 @@ main( hypre_int argc, + HYPRE_LOBPCGSolve(pcg_solver, constraints, eigenvectors, eigenvalues); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -5490,7 +5568,7 @@ main( hypre_int argc, + (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -5501,7 +5579,7 @@ main( hypre_int argc, + (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -5723,7 +5801,7 @@ main( hypre_int argc, + (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -5734,7 +5812,7 @@ main( hypre_int argc, + (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -6060,7 +6138,7 @@ main( hypre_int argc, + (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -6071,7 +6149,7 @@ main( hypre_int argc, + (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -6397,7 +6475,7 @@ main( hypre_int argc, + (HYPRE_Vector)b, (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -6408,7 +6486,7 @@ main( hypre_int argc, + (HYPRE_Vector)b, (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -6723,7 +6801,7 @@ main( hypre_int argc, + (HYPRE_Vector)b, (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -6734,7 +6812,7 @@ main( hypre_int argc, + (HYPRE_Vector)b, (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -6930,7 +7008,7 @@ main( hypre_int argc, + (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -6941,7 +7019,7 @@ main( hypre_int argc, + (HYPRE_Vector)x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -7073,7 +7151,7 @@ main( hypre_int argc, + HYPRE_MGRSetup(mgr_solver, parcsr_A, b, x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -7084,7 +7162,7 @@ main( hypre_int argc, + HYPRE_MGRSolve(mgr_solver, parcsr_A, b, x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -7170,7 +7248,7 @@ main( hypre_int argc, + HYPRE_ILUSetup(ilu_solver, parcsr_A, b, x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Setup phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -7181,7 +7259,7 @@ main( hypre_int argc, + HYPRE_ILUSolve(ilu_solver, parcsr_A, b, x); + + hypre_EndTiming(time_index); +- hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD); ++ hypre_fPrintTiming(fplog, "Solve phase times", hypre_MPI_COMM_WORLD); + hypre_FinalizeTiming(time_index); + hypre_ClearTiming(); + +@@ -7286,7 +7364,13 @@ main( hypre_int argc, + /* Finalize Hypre */ + HYPRE_Finalize(); + ++ ++ fflush(fplog); ++ fclose(fplog); ++ + /* Finalize MPI */ ++ if(parent!=MPI_COMM_NULL) ++ MPI_Comm_disconnect(&parent); + hypre_MPI_Finalize(); + + /* when using cuda-memcheck --leak-check full, uncomment this */ diff --git a/var/spack/repos/builtin/packages/hypre/package.py b/var/spack/repos/builtin/packages/hypre/package.py index dff8419df8..d4dc0d6a39 100644 --- a/var/spack/repos/builtin/packages/hypre/package.py +++ b/var/spack/repos/builtin/packages/hypre/package.py @@ -68,6 +68,11 @@ class Hypre(AutotoolsPackage, CudaPackage): variant('unified-memory', default=False, description='Use unified memory') variant('fortran', default=True, description='Enables fortran bindings') + variant('gptune', default=False, + description='Add the GPTune hookup code') + + # Patch to add gptune hookup codes + patch('ij_gptune.patch', when='+gptune@2.19.0') # Patch to add ppc64le in config.guess patch('ibm-ppc64le.patch', when='@:2.11.1') @@ -88,6 +93,7 @@ class Hypre(AutotoolsPackage, CudaPackage): conflicts('+cuda', when='+int64') conflicts('+unified-memory', when='~cuda') + conflicts('+gptune', when='~mpi') # Patch to build shared libraries on Darwin does not apply to # versions before 2.13.0 @@ -228,6 +234,10 @@ class Hypre(AutotoolsPackage, CudaPackage): sstruct('-in', 'test/sstruct.in.default', '-solver', '40', '-rhsone') make("install") + if '+gptune' in self.spec: + make("test") + self.run_test('mkdir', options=['-p', self.prefix.bin]) + self.run_test('cp', options=['test/ij', self.prefix.bin + '/.']) extra_install_tests = join_path('src', 'examples') diff --git a/var/spack/repos/builtin/packages/superlu-dist/CMAKE_INSTALL_LIBDIR.patch b/var/spack/repos/builtin/packages/superlu-dist/CMAKE_INSTALL_LIBDIR.patch new file mode 100644 index 0000000000..d3d8311863 --- /dev/null +++ b/var/spack/repos/builtin/packages/superlu-dist/CMAKE_INSTALL_LIBDIR.patch @@ -0,0 +1,59 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 2b2fdf2..f89cf4c 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -456,15 +456,6 @@ endif() + #target_compile_features(SuperLU_DIST PUBLIC cxx_std_11) + + # Generate various configure files with proper definitions +-# configure_file(${CMAKE_SOURCE_DIR}/make.inc.in ${CMAKE_BINARY_DIR}/make.inc) +-configure_file(${SuperLU_DIST_SOURCE_DIR}/make.inc.in ${SuperLU_DIST_SOURCE_DIR}/make.inc) +- +-configure_file(${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h.in ${SuperLU_DIST_BINARY_DIR}/SRC/superlu_dist_config.h) +-configure_file(${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h.in ${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h) +- +-# Following is to configure a file for FORTRAN code +-configure_file(${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h.in ${SuperLU_DIST_BINARY_DIR}/FORTRAN/superlu_dist_config.h) +- + + # Add pkg-config support + if(IS_ABSOLUTE ${CMAKE_INSTALL_LIBDIR}) +@@ -476,4 +467,13 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/superlu_dist.pc.in ${CMAKE_CURRENT_BI + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/superlu_dist.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + ++# configure_file(${CMAKE_SOURCE_DIR}/make.inc.in ${CMAKE_BINARY_DIR}/make.inc) ++configure_file(${SuperLU_DIST_SOURCE_DIR}/make.inc.in ${SuperLU_DIST_SOURCE_DIR}/make.inc) ++ ++configure_file(${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h.in ${SuperLU_DIST_BINARY_DIR}/SRC/superlu_dist_config.h) ++configure_file(${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h.in ${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h) ++ ++# Following is to configure a file for FORTRAN code ++configure_file(${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h.in ${SuperLU_DIST_BINARY_DIR}/FORTRAN/superlu_dist_config.h) ++ + #message("MPI_Fortran_LINK_FLAGS '${MPI_Fortran_LINK_FLAGS}'") +diff --git a/make.inc.in b/make.inc.in +index 0beb461..860c0bf 100644 +--- a/make.inc.in ++++ b/make.inc.in +@@ -18,7 +18,8 @@ + # + SuperLUroot = ${CMAKE_INSTALL_PREFIX} + #DSUPERLULIB = $(SuperLUroot)/SRC/${PROJECT_NAME_LIB_EXPORT} +-DSUPERLULIB = $(SuperLUroot)/@CMAKE_INSTALL_LIBDIR@/${PROJECT_NAME_LIB_EXPORT} ++#DSUPERLULIB = $(SuperLUroot)/@CMAKE_INSTALL_LIBDIR@/${PROJECT_NAME_LIB_EXPORT} ++DSUPERLULIB = @pkgconfig_libdir@/${PROJECT_NAME_LIB_EXPORT} + INCLUDEDIR = $(SuperLUroot)/@CMAKE_INSTALL_INCLUDEDIR@ + + XSDK_INDEX_SIZE = @XSDK_INDEX_SIZE@ +@@ -29,7 +30,8 @@ HAVE_CUDA = @HAVE_CUDA@ + + XSDK_ENABLE_Fortran = @XSDK_ENABLE_Fortran@ + ifeq ($(XSDK_ENABLE_Fortran),ON) +- DFORTRANLIB = $(SuperLUroot)/@CMAKE_INSTALL_LIBDIR@/${PROJECT_NAME_LIB_FORTRAN} ++# DFORTRANLIB = $(SuperLUroot)/@CMAKE_INSTALL_LIBDIR@/${PROJECT_NAME_LIB_FORTRAN} ++ DFORTRANLIB = @pkgconfig_libdir@/${PROJECT_NAME_LIB_FORTRAN} + LIBS = $(DFORTRANLIB) $(DSUPERLULIB) ${BLAS_LIB_EXPORT} -lm + LIBS += ${EXTRA_FLIB_EXPORT} + else + diff --git a/var/spack/repos/builtin/packages/superlu-dist/package.py b/var/spack/repos/builtin/packages/superlu-dist/package.py index 9677446893..7af879c45f 100644 --- a/var/spack/repos/builtin/packages/superlu-dist/package.py +++ b/var/spack/repos/builtin/packages/superlu-dist/package.py @@ -20,6 +20,7 @@ class SuperluDist(CMakePackage, CudaPackage, ROCmPackage): version('develop', branch='master') version('amd', branch='amd') + version('7.2.0', sha256='20b60bd8a3d88031c9ce6511ae9700b7a8dcf12e2fd704e74b1af762b3468b8c') version('7.1.1', sha256='558053b3d4a56eb661c4f04d4fcab6604018ce5db97115394c161b56c9c278ff') version('7.1.0', sha256='edbea877562be95fb22c7de1ff484f18685bec4baa8e4f703c414d3c035d4a66') version('6.4.0', sha256='cb9c0b2ba4c28e5ed5817718ba19ae1dd63ccd30bc44c8b8252b54f5f04a44cc') @@ -58,6 +59,7 @@ class SuperluDist(CMakePackage, CudaPackage, ROCmPackage): patch('xl-611.patch', when='@:6.1.1 %xl') patch('xl-611.patch', when='@:6.1.1 %xl_r') patch('superlu-cray-ftn-case.patch', when='@7.1.1 %cce') + patch('CMAKE_INSTALL_LIBDIR.patch', when='@7.0.0:7.2.0') def cmake_args(self): spec = self.spec -- cgit v1.2.3-60-g2f50