From f9e9ecd0c1d0e020bfd545ab1f7ba25267ace7f7 Mon Sep 17 00:00:00 2001 From: Terry Cojean Date: Mon, 14 Nov 2022 19:33:11 +0100 Subject: Ginkgo 1.5.0 version, new MPI variant, related fixes (#33838) * Ginkgo 1.5.0 release, new MPI variant * Fix ROCTHRUST/ROCPRIM issues * Fix deal.II issue with Ginkgo 1.5.0 * Also fix hipRAND+rocRAND RPATH settings * Turn off CCACHE for spack builds. Co-authored-by: Veselin Dobrev Co-authored-by: Veselin Dobrev --- var/spack/repos/builtin/packages/dealii/package.py | 8 +++++++ var/spack/repos/builtin/packages/ginkgo/package.py | 26 +++++++++++++++------- .../rocrand/hiprand_prefer_samedir_rocrand.patch | 13 +++++++++++ .../repos/builtin/packages/rocrand/package.py | 8 ++++++- 4 files changed, 46 insertions(+), 9 deletions(-) create mode 100644 var/spack/repos/builtin/packages/rocrand/hiprand_prefer_samedir_rocrand.patch (limited to 'var') diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index b4cade21df..5b6f16b0ef 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -265,6 +265,14 @@ class Dealii(CMakePackage, CudaPackage): when="@9.4.0 ^python", ) + # Fix issues with the FIND_GINKGO module for the newer Ginkgo versions + # https://github.com/dealii/dealii/pull/14413 + patch( + "https://github.com/dealii/dealii/commit/df6c5de8d6785fce701c10575982858f3aeb4cbd.patch?full_index=1", + sha256="c9884ebb0fe379c539012a225d8bcdcfe288edec8dc9d319fbfd64d8fbafba8e", + when="@:9.4.0+ginkgo ^ginkgo@1.5.0:", + ) + # Check for sufficiently modern versions conflicts("cxxstd=11", when="@9.3:") diff --git a/var/spack/repos/builtin/packages/ginkgo/package.py b/var/spack/repos/builtin/packages/ginkgo/package.py index e2ebbe1377..8700e7ea62 100644 --- a/var/spack/repos/builtin/packages/ginkgo/package.py +++ b/var/spack/repos/builtin/packages/ginkgo/package.py @@ -22,9 +22,9 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage): version("develop", branch="develop") version("master", branch="master") - version("1.5.0.glu", branch="glu") + version("1.5.0", commit="234594c92b58e2384dfb43c2d08e7f43e2b58e7a", preferred=True) # v1.5.0 version("1.5.0.glu_experimental", branch="glu_experimental") - version("1.4.0", commit="f811917c1def4d0fcd8db3fe5c948ce13409e28e", preferred=True) # v1.4.0 + version("1.4.0", commit="f811917c1def4d0fcd8db3fe5c948ce13409e28e") # v1.4.0 version("1.3.0", commit="4678668c66f634169def81620a85c9a20b7cec78") # v1.3.0 version("1.2.0", commit="b4be2be961fd5db45c3d02b5e004d73550722e31") # v1.2.0 version("1.1.1", commit="08d2c5200d3c78015ac8a4fd488bafe1e4240cf5") # v1.1.1 @@ -37,6 +37,7 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage): variant("oneapi", default=False, description="Build with oneAPI support") variant("develtools", default=False, description="Compile with develtools enabled") variant("hwloc", default=False, description="Enable HWLOC support") + variant("mpi", default=False, description="Enable MPI support") variant( "build_type", default="Release", @@ -46,12 +47,16 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage): depends_on("cmake@3.9:", type="build") depends_on("cuda@9:", when="+cuda") + depends_on("mpi", when="+mpi") - depends_on("rocthrust", type="build", when="+rocm") - depends_on("hipsparse", type="link", when="+rocm") - depends_on("hipblas", type="link", when="+rocm") - depends_on("rocrand", type="link", when="+rocm") - depends_on("hwloc@2.1:", type="link", when="+hwloc") + depends_on("rocthrust", when="+rocm") + depends_on("hipsparse", when="+rocm") + depends_on("hipblas", when="+rocm") + depends_on("rocrand", when="+rocm") + # ROCPRIM is not a direct dependency, but until we have reviewed our CMake + # setup for rocthrust, this needs to also be added here. + depends_on("rocprim", when="+rocm") + depends_on("hwloc@2.1:", when="+hwloc") depends_on("googletest", type="test") depends_on("numactl", type="test", when="+hwloc") @@ -61,6 +66,7 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage): conflicts("%gcc@:5.2.9") conflicts("+rocm", when="@:1.1.1") + conflicts("+mpi", when="@:1.4.0") conflicts("+cuda", when="+rocm") conflicts("+openmp", when="+oneapi") @@ -69,6 +75,7 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage): conflicts("^hipblas@4.1.0:", when="@:1.3.0") conflicts("^hipsparse@4.1.0:", when="@:1.3.0") conflicts("^rocthrust@4.1.0:", when="@:1.3.0") + conflicts("^rocprim@4.1.0:", when="@:1.3.0") # Skip smoke tests if compatible hardware isn't found patch("1.4.0_skip_invalid_smoke_tests.patch", when="@master") @@ -110,17 +117,19 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage): args = [ from_variant("GINKGO_BUILD_CUDA", "cuda"), from_variant("GINKGO_BUILD_HIP", "rocm"), + from_variant("GINKGO_BUILD_DPCPP", "oneapi"), from_variant("GINKGO_BUILD_OMP", "openmp"), + from_variant("GINKGO_BUILD_MPI", "mpi"), from_variant("BUILD_SHARED_LIBS", "shared"), from_variant("GINKGO_JACOBI_FULL_OPTIMIZATIONS", "full_optimizations"), from_variant("GINKGO_BUILD_HWLOC", "hwloc"), - from_variant("GINKGO_BUILD_DPCPP", "oneapi"), from_variant("GINKGO_DEVEL_TOOLS", "develtools"), # As we are not exposing benchmarks, examples, tests nor doc # as part of the installation, disable building them altogether. "-DGINKGO_BUILD_BENCHMARKS=OFF", "-DGINKGO_BUILD_DOC=OFF", "-DGINKGO_BUILD_EXAMPLES=OFF", + "-DGINKGO_WITH_CCACHE=OFF", self.define("GINKGO_BUILD_TESTS", self.run_tests), # Let spack handle the RPATH "-DGINKGO_INSTALL_RPATH=OFF", @@ -143,6 +152,7 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage): args.append("-DHIPBLAS_PATH={0}".format(spec["hipblas"].prefix)) args.append("-DHIPRAND_PATH={0}/hiprand".format(spec["rocrand"].prefix)) args.append("-DROCRAND_PATH={0}/rocrand".format(spec["rocrand"].prefix)) + args.append("-DROCPRIM_INCLUDE_DIRS={0}".format(spec["rocprim"].prefix.include)) archs = self.spec.variants["amdgpu_target"].value if archs != "none": arch_str = ";".join(archs) diff --git a/var/spack/repos/builtin/packages/rocrand/hiprand_prefer_samedir_rocrand.patch b/var/spack/repos/builtin/packages/rocrand/hiprand_prefer_samedir_rocrand.patch new file mode 100644 index 0000000000..60273c80e4 --- /dev/null +++ b/var/spack/repos/builtin/packages/rocrand/hiprand_prefer_samedir_rocrand.patch @@ -0,0 +1,13 @@ +diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt +index afbb6e2..478c63b 100644 +--- a/library/CMakeLists.txt ++++ b/library/CMakeLists.txt +@@ -62,7 +62,7 @@ set_target_properties(hiprand + PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/library" + DEBUG_POSTFIX "-d" +- INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/hiprand/lib" ++ INSTALL_RPATH "$ORIGIN;${CMAKE_INSTALL_PREFIX}/hiprand/lib" + ) + + rocm_install( diff --git a/var/spack/repos/builtin/packages/rocrand/package.py b/var/spack/repos/builtin/packages/rocrand/package.py index b6bf4e200a..ff88d55890 100644 --- a/var/spack/repos/builtin/packages/rocrand/package.py +++ b/var/spack/repos/builtin/packages/rocrand/package.py @@ -97,6 +97,12 @@ class Rocrand(CMakePackage): depends_on("googletest@1.10.0:", type="test") + # This patch ensures that libhiprand.so searches for librocrand.so in its + # own directory first thanks to the $ORIGIN RPATH setting. Otherwise, + # libhiprand.so cannot find dependency librocrand.so despite being in the + # same directory. + patch("hiprand_prefer_samedir_rocrand.patch", working_dir="hiprand", when="@5.2.0:") + resource( name="hipRAND", git="https://github.com/ROCmSoftwarePlatform/hipRAND.git", @@ -157,7 +163,7 @@ class Rocrand(CMakePackage): for lib in rocrand_libs: os.symlink(join_path(rocrand_lib_path, lib), join_path(self.prefix.lib, lib)) """Fix the rocRAND and hipRAND include path""" - # rocRAND installs irocrand*.h* and hiprand*.h* rocrand/include and + # rocRAND installs rocrand*.h* and hiprand*.h* rocrand/include and # hiprand/include, respectively. This confuses spack's RPATH management. We # fix it by adding a symlink to the header files. hiprand_include_path = join_path(self.prefix, "hiprand", "include") -- cgit v1.2.3-60-g2f50