Add hpx-kokkos package (#29766)

Co-authored-by: Mikael Simberg <mikael.simberg@iki.if>

3 files changed, 140 insertions, 0 deletions

diff --git a/var/spack/repos/builtin/packages/hpx-kokkos/package.py b/var/spack/repos/builtin/packages/hpx-kokkos/package.py
new file mode 100644
index 0000000000..5c3ed3aeea
--- /dev/null
+++ b/var/spack/repos/builtin/packages/hpx-kokkos/package.py
@@ -0,0 +1,49 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class HpxKokkos(CMakePackage, CudaPackage, ROCmPackage):
+    """HPXKokkos is an interoperability library for HPX and Kokkos"""
+
+    homepage = 'https://github.com/STEllAR-GROUP/hpx-kokkos'
+    url = "https://github.com/STEllAR-GROUP/hpx-kokkos/archive/0.0.0.tar.gz"
+    maintainers = ['G-071', 'msimberg']
+
+    version('master', git='https://github.com/STEllAR-GROUP/hpx-kokkos.git', branch='master')
+    version('0.3.0', sha256='83c1d11dab95552ad0abdae767c71f757811d7b51d82bd231653dc942e89a45d')
+    version('0.2.0', sha256='289b711cea26afe80be002fc521234c9194cd0e8f69863f3b08b654674dbe5d5')
+    version('0.1.0', sha256='24edb817d0969f4aea1b68eab4984c2ea9a58f4760a9b8395e20f85b178f0850')
+
+    cxxstds = ('14', '17', '20')
+    variant('cxxstd', default='14', values=cxxstds, description='Use the specified C++ standard when building.')
+
+    depends_on('cmake@3.19:', type='build')
+
+    depends_on('hpx')
+    depends_on('kokkos +hpx +hpx_async_dispatch')
+
+    depends_on('hpx@1.8:', when='@0.3:')
+    depends_on('kokkos@3.6:', when='@0.3:')
+
+    depends_on('hpx@1.7', when='@0.2')
+    depends_on('kokkos@3.6:', when='@0.2')
+
+    depends_on('hpx@1.6', when='@0.1')
+    depends_on('kokkos@3.2:3.5', when='@0.1')
+
+    for cxxstd in cxxstds:
+        depends_on('hpx cxxstd={0}'.format(cxxstd), when='cxxstd={0}'.format(cxxstd))
+        depends_on('kokkos std={0}'.format(cxxstd), when='cxxstd={0}'.format(cxxstd))
+
+    # HPXKokkos explicitly supports CUDA and ROCm. Other GPU backends can be
+    # used but without support in HPXKokkos. Other CPU backends, except Serial,
+    # can't be used together with the HPX backend.
+    depends_on('hpx +cuda', when='+cuda')
+    depends_on('kokkos +cuda +cuda_lambda +cuda_constexpr', when='+cuda')
+
+    depends_on('hpx +rocm', when='+rocm')
+    depends_on('kokkos +rocm', when='+rocm')
diff --git a/var/spack/repos/builtin/packages/kokkos/hpx_profiling_fences.patch b/var/spack/repos/builtin/packages/kokkos/hpx_profiling_fences.patch
new file mode 100644
index 0000000000..013fbd9f67
--- /dev/null
+++ b/var/spack/repos/builtin/packages/kokkos/hpx_profiling_fences.patch
@@ -0,0 +1,84 @@
+diff --git a/core/src/HPX/Kokkos_HPX_Task.hpp b/core/src/HPX/Kokkos_HPX_Task.hpp
+index 7bb3ca5d0..ff50fdc5f 100644
+--- a/core/src/HPX/Kokkos_HPX_Task.hpp
++++ b/core/src/HPX/Kokkos_HPX_Task.hpp
+@@ -216,7 +216,7 @@ class TaskQueueSpecializationConstrained<
+     task_queue.scheduler = &scheduler;
+     Kokkos::Impl::dispatch_execute_task(&task_queue,
+                                         Kokkos::Experimental::HPX());
+-    Kokkos::Experimental::HPX().fence()"Kokkos::Impl::TaskQueueSpecializationConstrained::execute: fence after task execution";
++    Kokkos::Experimental::HPX().fence("Kokkos::Impl::TaskQueueSpecializationConstrained::execute: fence after task execution");
+   }
+ 
+   // Must provide task queue execution function
+diff --git a/core/src/Kokkos_HPX.hpp b/core/src/Kokkos_HPX.hpp
+index 236211864..3e8522e94 100644
+--- a/core/src/Kokkos_HPX.hpp
++++ b/core/src/Kokkos_HPX.hpp
+@@ -282,11 +282,11 @@ class HPX {
+     m_mode                      = other.m_mode;
+     m_independent_instance_data = other.m_independent_instance_data;
+     m_buffer                    = m_mode == instance_mode::independent
+-                   ? m_independent_instance_data->m_buffer
+-                   : m_global_instance_data.m_buffer;
+-    m_future = m_mode == instance_mode::independent
+-                   ? m_independent_instance_data->m_future
+-                   : m_global_instance_data.m_future;
++                                      ? m_independent_instance_data->m_buffer
++                                      : m_global_instance_data.m_buffer;
++    m_future                    = m_mode == instance_mode::independent
++                                      ? m_independent_instance_data->m_future
++                                      : m_global_instance_data.m_future;
+     return *this;
+   }
+ #else
+@@ -322,25 +322,36 @@ class HPX {
+         "Fence");
+   }
+   void impl_fence_instance(const std::string &name) const {
+-    Kokkos::Tools::Experimental::Impl::profile_fence_event(name, *this, [&]() {
+-      if (hpx::threads::get_self_ptr() == nullptr) {
+-        hpx::threads::run_as_hpx_thread([this]() { impl_get_future().wait(); });
+-      } else {
+-        impl_get_future().wait();
+-      }
+-    });
++    Kokkos::Tools::Experimental::Impl::profile_fence_event<
++        Kokkos::Experimental::HPX>(
++        name,
++        Kokkos::Tools::Experimental::Impl::DirectFenceIDHandle{
++            impl_instance_id()},
++        [&]() {
++          if (hpx::threads::get_self_ptr() == nullptr) {
++            hpx::threads::run_as_hpx_thread(
++                [this]() { impl_get_future().wait(); });
++          } else {
++            impl_get_future().wait();
++          }
++        });
+   }
+ 
+   void impl_fence_all_instances() const {
+-    impl_fence_instance(
++    impl_fence_all_instances(
+         "Kokkos::Experimental::HPX::impl_fence_all_instances: Unnamed Global "
+         "HPX Fence");
+   }
+-  void impl_fence_all_instances(const std::string &namename) const {
+-    Kokkos::Tools::Experimental::Impl::profile_fence_event(name, *this, [&]() {
+-      hpx::util::yield_while(
+-          []() { return m_active_parallel_region_count.load() != 0; });
+-    });
++  void impl_fence_all_instances(const std::string &name) const {
++    Kokkos::Tools::Experimental::Impl::profile_fence_event<
++        Kokkos::Experimental::HPX>(
++        name,
++        Kokkos::Tools::Experimental::SpecialSynchronizationCases::
++            GlobalDeviceSynchronization,
++        [&]() {
++          hpx::util::yield_while(
++              []() { return m_active_parallel_region_count.load() != 0; });
++        });
+   }
+ #endif
+ 
diff --git a/var/spack/repos/builtin/packages/kokkos/package.py b/var/spack/repos/builtin/packages/kokkos/package.py
index aea3eca8b0..fe3e582006 100644
--- a/var/spack/repos/builtin/packages/kokkos/package.py
+++ b/var/spack/repos/builtin/packages/kokkos/package.py
@@ -197,6 +197,13 @@ class Kokkos(CMakePackage, CudaPackage, ROCmPackage):
     for std in stds:
         depends_on('hpx cxxstd={0}'.format(std), when='+hpx std={0}'.format(std))
 
+    # HPX version constraints
+    depends_on("hpx@:1.6", when="@:3.5 +hpx")
+    depends_on("hpx@1.7:", when="@3.6: +hpx")
+
+    # Patches
+    patch("hpx_profiling_fences.patch", when="@3.5.00 +hpx")
+
     variant('shared', default=True, description='Build shared libraries')
 
     @classmethod