diff options
Diffstat (limited to 'var/spack/repos/builtin/packages/mfem/test_builds.sh')
| -rwxr-xr-x | var/spack/repos/builtin/packages/mfem/test_builds.sh | 120 |
1 files changed, 88 insertions, 32 deletions
diff --git a/var/spack/repos/builtin/packages/mfem/test_builds.sh b/var/spack/repos/builtin/packages/mfem/test_builds.sh index cb658dd59c..be0d27bc0f 100755 --- a/var/spack/repos/builtin/packages/mfem/test_builds.sh +++ b/var/spack/repos/builtin/packages/mfem/test_builds.sh @@ -14,9 +14,9 @@ rocm_arch="gfx908" spack_jobs='' # spack_jobs='-j 128' -mfem='mfem@4.6.0'${compiler} +mfem='mfem@4.7.0'${compiler} # mfem_dev='mfem@develop'${compiler} -mfem_dev='mfem@4.6.0'${compiler} +mfem_dev='mfem@4.7.0'${compiler} backends='+occa+raja+libceed' backends_specs='^occa~cuda ^raja~openmp' @@ -31,44 +31,60 @@ petsc_spec_rocm='^petsc+rocm+mumps' strumpack_spec='^strumpack~slate~openmp~cuda' strumpack_cuda_spec='^strumpack+cuda~slate~openmp' strumpack_rocm_spec='^strumpack+rocm~slate~openmp~cuda' -# superlu specs with cuda and rocm -superlu_cuda_spec='^superlu-dist+cuda' -superlu_rocm_spec='^superlu-dist+rocm' +# superlu specs with cpu, cuda and rocm +# - v8.2.1 on CPU and GPU stalls in ex11p; works when superlu::PARMETIS is +# replaced with superlu::METIS_AT_PLUS_A, at least on CPU +superlu_spec='^superlu-dist@8.1.2' +superlu_cuda_spec='^superlu-dist@8.1.2+cuda' +superlu_rocm_spec='^superlu-dist@8.1.2+rocm' +# FMS spec +fms_spec='^libfms+conduit' builds=( # preferred version: ${mfem} ${mfem}'~mpi~metis~zlib' - ${mfem}"$backends"'+superlu-dist+strumpack+suite-sparse+petsc+slepc+gslib \ - +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \ - '"$backends_specs $strumpack_spec $petsc_spec $conduit_spec" + # TODO: add back "+fms $fms_spec" when the FMS unit test is fixed + ${mfem}"$backends"'+superlu-dist+strumpack+mumps+suite-sparse+petsc+slepc \ + +gslib+sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo \ + +hiop \ + '"$backends_specs $superlu_spec $strumpack_spec $petsc_spec"' \ + '"$conduit_spec" + # TODO: add back "+fms $fms_spec" when the FMS unit test is fixed ${mfem}'~mpi \ '"$backends"'+suite-sparse+sundials+gslib+mpfr+netcdf \ +zlib+gnutls+libunwind+conduit+ginkgo+hiop \ '"$backends_specs $conduit_spec"' ^sundials~mpi' + ${mfem}' precision=single +mumps+petsc '"$petsc_spec" # develop version, shared builds: ${mfem_dev}'+shared~static' ${mfem_dev}'+shared~static~mpi~metis~zlib' # NOTE: Shared build with +gslib works on mac but not on linux # TODO: add back '+gslib' when the above NOTE is addressed. + # TODO: add back "+fms $fms_spec" when the FMS unit test is fixed ${mfem_dev}'+shared~static \ - '"$backends"'+superlu-dist+strumpack+suite-sparse+petsc+slepc \ + '"$backends"'+superlu-dist+strumpack+mumps+suite-sparse+petsc+slepc \ +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \ - '"$backends_specs $strumpack_spec $petsc_spec $conduit_spec" + '"$backends_specs $superlu_spec $strumpack_spec $petsc_spec"' \ + '"$conduit_spec" # NOTE: Shared build with +gslib works on mac but not on linux # TODO: add back '+gslib' when the above NOTE is addressed. + # TODO: add back "+fms $fms_spec" when the FMS unit test is fixed ${mfem_dev}'+shared~static~mpi \ '"$backends"'+suite-sparse+sundials+mpfr+netcdf \ +zlib+gnutls+libunwind+conduit+ginkgo+hiop \ '"$backends_specs $conduit_spec"' ^sundials~mpi' + ${mfem_dev}'+shared~static precision=single +mumps+petsc '"$petsc_spec" ) builds2=( # preferred version ${mfem}"$backends $backends_specs" - ${mfem}'+superlu-dist' + ${mfem}' precision=single' + ${mfem}'+superlu-dist'" $superlu_spec" ${mfem}'+strumpack'" $strumpack_spec" + ${mfem}'+mumps' ${mfem}'+suite-sparse~mpi' ${mfem}'+suite-sparse' ${mfem}'+sundials~mpi ^sundials~mpi' @@ -81,6 +97,8 @@ builds2=( ${mfem}'+gnutls' ${mfem}'+conduit~mpi'" $conduit_spec" ${mfem}'+conduit'" $conduit_spec" + # TODO: uncomment next line when the FMS unit test is fixed + # ${mfem}'+fms'" $fms_spec" ${mfem}'+umpire' ${mfem}'+petsc'" $petsc_spec" ${mfem}'+petsc+slepc'" $petsc_spec" @@ -93,8 +111,10 @@ builds2=( # # develop version ${mfem_dev}"$backends $backends_specs" - ${mfem_dev}'+superlu-dist' + ${mfem_dev}' precision=single' + ${mfem_dev}'+superlu-dist'" $superlu_spec" ${mfem_dev}'+strumpack'" $strumpack_spec" + ${mfem_dev}'+mumps' ${mfem_dev}'+suite-sparse~mpi' ${mfem_dev}'+suite-sparse' ${mfem_dev}'+sundials~mpi ^sundials~mpi' @@ -107,6 +127,8 @@ builds2=( ${mfem_dev}'+gnutls' ${mfem_dev}'+conduit~mpi'" $conduit_spec" ${mfem_dev}'+conduit'" $conduit_spec" + # TODO: uncomment next line when the FMS unit test is fixed + # ${mfem_dev}'+fms'" $fms_spec" ${mfem_dev}'+umpire' ${mfem_dev}'+petsc'" $petsc_spec" ${mfem_dev}'+petsc+slepc'" $petsc_spec" @@ -134,25 +156,37 @@ builds_cuda=( # hypre without cuda: # NOTE: PETSc tests may need PETSC_OPTIONS="-use_gpu_aware_mpi 0" # TODO: restore '+libceed' when the libCEED CUDA unit tests take less time. - # TODO: remove "^hiop+shared" when the default static build is fixed. ${mfem}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \ +superlu-dist+strumpack+suite-sparse+gslib+petsc+slepc \ - +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \ - ^raja+cuda+openmp ^hiop+shared'" $strumpack_cuda_spec"' \ + +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo \ + ^raja+cuda+openmp'" $strumpack_cuda_spec"' \ '"$superlu_cuda_spec $petsc_spec_cuda $conduit_spec" + ${mfem}'+cuda cuda_arch='"${cuda_arch}"' +raja+umpire' + + # hiop needs older versions of raja, umpire, etc + # TODO: combine this spec with the above spec when the combined spec works. + ${mfem}'+cuda cuda_arch='"${cuda_arch}"' +hiop' + # hypre with cuda: # TODO: restore '+libceed' when the libCEED CUDA unit tests take less time. # TODO: add back "+petsc+slepc $petsc_spec_cuda" when it works. # NOTE: PETSc tests may need PETSC_OPTIONS="-use_gpu_aware_mpi 0" # TODO: add back "+sundials" when it's supported with '^hypre+cuda'. - # TODO: remove "^hiop+shared" when the default static build is fixed. ${mfem}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \ +superlu-dist+strumpack+suite-sparse+gslib \ - +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \ - ^raja+cuda+openmp ^hiop+shared ^hypre+cuda \ + +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo \ + ^raja+cuda+openmp ^hypre+cuda \ '" $strumpack_cuda_spec $superlu_cuda_spec $conduit_spec" + ${mfem}'+cuda cuda_arch='"${cuda_arch}"' +raja+umpire ^hypre+cuda' + + # hiop needs older versions of raja, umpire, etc + # TODO: combine this spec with the above spec when the combined spec works. + ${mfem}'+cuda cuda_arch='"${cuda_arch}"' +hiop ^hypre+cuda' + + ${mfem}' precision=single +cuda cuda_arch='"${cuda_arch}"' ^hypre+cuda' + # # same builds as above with ${mfem_dev} # @@ -171,24 +205,36 @@ builds_cuda=( # hypre without cuda: # NOTE: PETSc tests may need PETSC_OPTIONS="-use_gpu_aware_mpi 0" # TODO: restore '+libceed' when the libCEED CUDA unit tests take less time. - # TODO: remove "^hiop+shared" when the default static build is fixed. ${mfem_dev}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \ +superlu-dist+strumpack+suite-sparse+gslib+petsc+slepc \ - +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \ - ^raja+cuda+openmp ^hiop+shared'" $strumpack_cuda_spec"' \ + +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo \ + ^raja+cuda+openmp'" $strumpack_cuda_spec"' \ '"$superlu_cuda_spec $petsc_spec_cuda $conduit_spec" + ${mfem_dev}'+cuda cuda_arch='"${cuda_arch}"' +raja+umpire' + + # hiop needs older versions of raja, umpire, etc + # TODO: combine this spec with the above spec when the combined spec works. + ${mfem_dev}'+cuda cuda_arch='"${cuda_arch}"' +hiop' + # hypre with cuda: # TODO: restore '+libceed' when the libCEED CUDA unit tests take less time. # TODO: add back "+petsc+slepc $petsc_spec_cuda" when it works. # NOTE: PETSc tests may need PETSC_OPTIONS="-use_gpu_aware_mpi 0" # TODO: add back "+sundials" when it's supported with '^hypre+cuda'. - # TODO: remove "^hiop+shared" when the default static build is fixed. ${mfem_dev}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \ +superlu-dist+strumpack+suite-sparse+gslib \ - +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \ - ^raja+cuda+openmp ^hiop+shared ^hypre+cuda \ + +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo \ + ^raja+cuda+openmp ^hypre+cuda \ '"$strumpack_cuda_spec $superlu_cuda_spec $conduit_spec" + + ${mfem_dev}'+cuda cuda_arch='"${cuda_arch}"' +raja+umpire ^hypre+cuda' + + # hiop needs older versions of raja, umpire, etc + # TODO: combine this spec with the above spec when the combined spec works. + ${mfem_dev}'+cuda cuda_arch='"${cuda_arch}"' +hiop ^hypre+cuda' + + ${mfem_dev}' precision=single +cuda cuda_arch='"${cuda_arch}"' ^hypre+cuda' ) @@ -204,27 +250,35 @@ builds_rocm=( ^raja+rocm~openmp ^occa~cuda~openmp ^hypre+rocm' # hypre without rocm: - # TODO: add back '+hiop' when it is no longer linked with tcmalloc* through - # its magma dependency. - # TODO: add back '+ginkgo' when the Ginkgo example works. ${mfem}'+rocm+openmp+raja+occa+libceed amdgpu_target='"${rocm_arch}"' \ +superlu-dist+strumpack+suite-sparse+gslib+petsc+slepc \ - +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \ + +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo \ ^raja+rocm~openmp ^occa~cuda'" $strumpack_rocm_spec"' \ '"$superlu_rocm_spec $petsc_spec_rocm $conduit_spec" + ${mfem}'+rocm amdgpu_target='"${rocm_arch}"' +raja+umpire' + + # hiop needs older versions of raja, umpire, etc + # TODO: combine this spec with the above spec when the combined spec works. + ${mfem}'+rocm amdgpu_target='"${rocm_arch}"' +hiop' + # hypre with rocm: # TODO: add back "+petsc+slepc $petsc_spec_rocm" when it works. - # TODO: add back '+hiop' when it is no longer linked with tcmalloc* through - # its magma dependency. - # TODO: add back '+ginkgo' when the Ginkgo example works. # TODO: add back "+sundials" when it's supported with '^hypre+rocm'. ${mfem}'+rocm+openmp+raja+occa+libceed amdgpu_target='"${rocm_arch}"' \ +superlu-dist+strumpack+suite-sparse+gslib \ - +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \ + +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo \ ^raja+rocm~openmp ^occa~cuda ^hypre+rocm \ '"$strumpack_rocm_spec $superlu_rocm_spec $conduit_spec" + ${mfem}'+rocm amdgpu_target='"${rocm_arch}"' +raja+umpire ^hypre+rocm' + + # hiop needs older versions of raja, umpire, etc + # TODO: combine this spec with the above spec when the combined spec works. + ${mfem}'+rocm amdgpu_target='"${rocm_arch}"' +hiop ^hypre+rocm' + + ${mfem}' precision=single +rocm amdgpu_target='"${rocm_arch}"' ^hypre+rocm' + # # same builds as above with ${mfem_dev} # @@ -244,6 +298,8 @@ run_builds=("${builds[@]}" "${builds2[@]}") # PETSc CUDA tests on Lassen need this: # export PETSC_OPTIONS="-use_gpu_aware_mpi 0" +# STRUMPACK forces "^openblas threads=openmp" when using openblas: +export OMP_NUM_THREADS=1 # spack files to clean in "$mfem_src_dir" when using 'dev-build' clean_files=( |