Ginkgo 1.5.0 version, new MPI variant, related fixes (#33838)

* Ginkgo 1.5.0 release, new MPI variant

* Fix ROCTHRUST/ROCPRIM issues

* Fix deal.II issue with Ginkgo 1.5.0

* Also fix hipRAND+rocRAND RPATH settings

* Turn off CCACHE for spack builds.

Co-authored-by: Veselin Dobrev <dobrev@llnl.gov>

Co-authored-by: Veselin Dobrev <dobrev@llnl.gov>

4 files changed, 46 insertions, 9 deletions

diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py
index b4cade21df..5b6f16b0ef 100644
--- a/var/spack/repos/builtin/packages/dealii/package.py
+++ b/var/spack/repos/builtin/packages/dealii/package.py
@@ -265,6 +265,14 @@ class Dealii(CMakePackage, CudaPackage):
         when="@9.4.0 ^python",
     )
 
+    # Fix issues with the FIND_GINKGO module for the newer Ginkgo versions
+    # https://github.com/dealii/dealii/pull/14413
+    patch(
+        "https://github.com/dealii/dealii/commit/df6c5de8d6785fce701c10575982858f3aeb4cbd.patch?full_index=1",
+        sha256="c9884ebb0fe379c539012a225d8bcdcfe288edec8dc9d319fbfd64d8fbafba8e",
+        when="@:9.4.0+ginkgo ^ginkgo@1.5.0:",
+    )
+
     # Check for sufficiently modern versions
     conflicts("cxxstd=11", when="@9.3:")
 
diff --git a/var/spack/repos/builtin/packages/ginkgo/package.py b/var/spack/repos/builtin/packages/ginkgo/package.py
index e2ebbe1377..8700e7ea62 100644
--- a/var/spack/repos/builtin/packages/ginkgo/package.py
+++ b/var/spack/repos/builtin/packages/ginkgo/package.py
@@ -22,9 +22,9 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage):
 
     version("develop", branch="develop")
     version("master", branch="master")
-    version("1.5.0.glu", branch="glu")
+    version("1.5.0", commit="234594c92b58e2384dfb43c2d08e7f43e2b58e7a", preferred=True)  # v1.5.0
     version("1.5.0.glu_experimental", branch="glu_experimental")
-    version("1.4.0", commit="f811917c1def4d0fcd8db3fe5c948ce13409e28e", preferred=True)  # v1.4.0
+    version("1.4.0", commit="f811917c1def4d0fcd8db3fe5c948ce13409e28e")  # v1.4.0
     version("1.3.0", commit="4678668c66f634169def81620a85c9a20b7cec78")  # v1.3.0
     version("1.2.0", commit="b4be2be961fd5db45c3d02b5e004d73550722e31")  # v1.2.0
     version("1.1.1", commit="08d2c5200d3c78015ac8a4fd488bafe1e4240cf5")  # v1.1.1
@@ -37,6 +37,7 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage):
     variant("oneapi", default=False, description="Build with oneAPI support")
     variant("develtools", default=False, description="Compile with develtools enabled")
     variant("hwloc", default=False, description="Enable HWLOC support")
+    variant("mpi", default=False, description="Enable MPI support")
     variant(
         "build_type",
         default="Release",
@@ -46,12 +47,16 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage):
 
     depends_on("cmake@3.9:", type="build")
     depends_on("cuda@9:", when="+cuda")
+    depends_on("mpi", when="+mpi")
 
-    depends_on("rocthrust", type="build", when="+rocm")
-    depends_on("hipsparse", type="link", when="+rocm")
-    depends_on("hipblas", type="link", when="+rocm")
-    depends_on("rocrand", type="link", when="+rocm")
-    depends_on("hwloc@2.1:", type="link", when="+hwloc")
+    depends_on("rocthrust", when="+rocm")
+    depends_on("hipsparse", when="+rocm")
+    depends_on("hipblas", when="+rocm")
+    depends_on("rocrand", when="+rocm")
+    # ROCPRIM is not a direct dependency, but until we have reviewed our CMake
+    # setup for rocthrust, this needs to also be added here.
+    depends_on("rocprim", when="+rocm")
+    depends_on("hwloc@2.1:", when="+hwloc")
 
     depends_on("googletest", type="test")
     depends_on("numactl", type="test", when="+hwloc")
@@ -61,6 +66,7 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage):
 
     conflicts("%gcc@:5.2.9")
     conflicts("+rocm", when="@:1.1.1")
+    conflicts("+mpi", when="@:1.4.0")
     conflicts("+cuda", when="+rocm")
     conflicts("+openmp", when="+oneapi")
 
@@ -69,6 +75,7 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage):
     conflicts("^hipblas@4.1.0:", when="@:1.3.0")
     conflicts("^hipsparse@4.1.0:", when="@:1.3.0")
     conflicts("^rocthrust@4.1.0:", when="@:1.3.0")
+    conflicts("^rocprim@4.1.0:", when="@:1.3.0")
 
     # Skip smoke tests if compatible hardware isn't found
     patch("1.4.0_skip_invalid_smoke_tests.patch", when="@master")
@@ -110,17 +117,19 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage):
         args = [
             from_variant("GINKGO_BUILD_CUDA", "cuda"),
             from_variant("GINKGO_BUILD_HIP", "rocm"),
+            from_variant("GINKGO_BUILD_DPCPP", "oneapi"),
             from_variant("GINKGO_BUILD_OMP", "openmp"),
+            from_variant("GINKGO_BUILD_MPI", "mpi"),
             from_variant("BUILD_SHARED_LIBS", "shared"),
             from_variant("GINKGO_JACOBI_FULL_OPTIMIZATIONS", "full_optimizations"),
             from_variant("GINKGO_BUILD_HWLOC", "hwloc"),
-            from_variant("GINKGO_BUILD_DPCPP", "oneapi"),
             from_variant("GINKGO_DEVEL_TOOLS", "develtools"),
             # As we are not exposing benchmarks, examples, tests nor doc
             # as part of the installation, disable building them altogether.
             "-DGINKGO_BUILD_BENCHMARKS=OFF",
             "-DGINKGO_BUILD_DOC=OFF",
             "-DGINKGO_BUILD_EXAMPLES=OFF",
+            "-DGINKGO_WITH_CCACHE=OFF",
             self.define("GINKGO_BUILD_TESTS", self.run_tests),
             # Let spack handle the RPATH
             "-DGINKGO_INSTALL_RPATH=OFF",
@@ -143,6 +152,7 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage):
             args.append("-DHIPBLAS_PATH={0}".format(spec["hipblas"].prefix))
             args.append("-DHIPRAND_PATH={0}/hiprand".format(spec["rocrand"].prefix))
             args.append("-DROCRAND_PATH={0}/rocrand".format(spec["rocrand"].prefix))
+            args.append("-DROCPRIM_INCLUDE_DIRS={0}".format(spec["rocprim"].prefix.include))
             archs = self.spec.variants["amdgpu_target"].value
             if archs != "none":
                 arch_str = ";".join(archs)
diff --git a/var/spack/repos/builtin/packages/rocrand/hiprand_prefer_samedir_rocrand.patch b/var/spack/repos/builtin/packages/rocrand/hiprand_prefer_samedir_rocrand.patch
new file mode 100644
index 0000000000..60273c80e4
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rocrand/hiprand_prefer_samedir_rocrand.patch
@@ -0,0 +1,13 @@
+diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt
+index afbb6e2..478c63b 100644
+--- a/library/CMakeLists.txt
++++ b/library/CMakeLists.txt
+@@ -62,7 +62,7 @@ set_target_properties(hiprand
+     PROPERTIES
+         RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/library"
+         DEBUG_POSTFIX "-d"
+-        INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/hiprand/lib"
++        INSTALL_RPATH "$ORIGIN;${CMAKE_INSTALL_PREFIX}/hiprand/lib"
+ )
+ 
+ rocm_install(
diff --git a/var/spack/repos/builtin/packages/rocrand/package.py b/var/spack/repos/builtin/packages/rocrand/package.py
index b6bf4e200a..ff88d55890 100644
--- a/var/spack/repos/builtin/packages/rocrand/package.py
+++ b/var/spack/repos/builtin/packages/rocrand/package.py
@@ -97,6 +97,12 @@ class Rocrand(CMakePackage):
 
     depends_on("googletest@1.10.0:", type="test")
 
+    # This patch ensures that libhiprand.so searches for librocrand.so in its
+    # own directory first thanks to the $ORIGIN RPATH setting. Otherwise,
+    # libhiprand.so cannot find dependency librocrand.so despite being in the
+    # same directory.
+    patch("hiprand_prefer_samedir_rocrand.patch", working_dir="hiprand", when="@5.2.0:")
+
     resource(
         name="hipRAND",
         git="https://github.com/ROCmSoftwarePlatform/hipRAND.git",
@@ -157,7 +163,7 @@ class Rocrand(CMakePackage):
                 for lib in rocrand_libs:
                     os.symlink(join_path(rocrand_lib_path, lib), join_path(self.prefix.lib, lib))
             """Fix the rocRAND and hipRAND include path"""
-            # rocRAND installs irocrand*.h* and hiprand*.h* rocrand/include and
+            # rocRAND installs rocrand*.h* and hiprand*.h* rocrand/include and
             # hiprand/include, respectively. This confuses spack's RPATH management. We
             # fix it by adding a symlink to the header files.
             hiprand_include_path = join_path(self.prefix, "hiprand", "include")