From b80b08557541274b60a8f982c0a0870f587acc87 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Fri, 3 Dec 2021 02:13:25 +0100
Subject: Ginkgo package: add oneAPI support (#26868)
To use this, you can "spack install intel-oneapi-compilers" and then
"spack compiler add" the new compiler. You would need to install with
"spack install ginkgo+oneapi%dpcpp"
---
.../ginkgo/1.4.0_dpcpp_use_old_standard.patch | 70 ++++++++++++++++++++++
var/spack/repos/builtin/packages/ginkgo/package.py | 32 +++++++++-
2 files changed, 100 insertions(+), 2 deletions(-)
create mode 100644 var/spack/repos/builtin/packages/ginkgo/1.4.0_dpcpp_use_old_standard.patch
(limited to 'var')
diff --git a/var/spack/repos/builtin/packages/ginkgo/1.4.0_dpcpp_use_old_standard.patch b/var/spack/repos/builtin/packages/ginkgo/1.4.0_dpcpp_use_old_standard.patch
new file mode 100644
index 0000000000..54ed07c22d
--- /dev/null
+++ b/var/spack/repos/builtin/packages/ginkgo/1.4.0_dpcpp_use_old_standard.patch
@@ -0,0 +1,70 @@
+commit 83164570f0d3511d114114bcc2b02ad23b753ed0
+Author: Yuhsiang M. Tsai <yhmtsai@gmail.com>
+Date: Wed Oct 6 16:33:16 2021 +0200
+
+ syclstd 1.2.1 in new release to propagate subgroup
+ remove 64 subgroup in dense to avoid conj_trans issue on cpu temporarily
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index ceb269b1cb..b47388a596 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -67,7 +67,8 @@ if(MSVC)
+ elseif(GINKGO_BUILD_DPCPP OR CMAKE_CXX_COMPILER MATCHES "dpcpp")
+ # For now always use `-ffp-model=precise` with DPC++. This can be removed when
+ # the floating point issues are fixed.
+- set(GINKGO_COMPILER_FLAGS "-Wpedantic;-ffp-model=precise" CACHE STRING
++ # -sycl-std=1.2.1 (or -sycl-std=2017) is temporary workaround after 2021.4 to propagate subgroup setting correctly
++ set(GINKGO_COMPILER_FLAGS "-Wpedantic;-ffp-model=precise;-sycl-std=1.2.1" CACHE STRING
+ "Set the required CXX compiler flags, mainly used for warnings. Current default is `-Wpedantic;-ffp-model=precise`")
+ else()
+ set(GINKGO_COMPILER_FLAGS "-Wpedantic" CACHE STRING
+diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
+index 9d22406f9a..dcc452b293 100644
+--- a/cmake/create_test.cmake
++++ b/cmake/create_test.cmake
+@@ -40,6 +40,7 @@ function(ginkgo_create_dpcpp_test test_name)
+ add_executable(${test_target_name} ${test_name}.dp.cpp)
+ target_compile_features(${test_target_name} PUBLIC cxx_std_17)
+ target_compile_options(${test_target_name} PRIVATE "${GINKGO_DPCPP_FLAGS}")
++ target_compile_options(${test_target_name} PRIVATE "${GINKGO_COMPILER_FLAGS}")
+ target_link_options(${test_target_name} PRIVATE -fsycl-device-code-split=per_kernel)
+ ginkgo_set_test_target_properties(${test_name} ${test_target_name})
+ # Note: MKL_ENV is empty on linux. Maybe need to apply MKL_ENV to all test.
+diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
+index fee9ec3639..ce71fd5d3c 100644
+--- a/dpcpp/CMakeLists.txt
++++ b/dpcpp/CMakeLists.txt
+@@ -68,6 +68,7 @@ target_compile_definitions(ginkgo_dpcpp PRIVATE GKO_COMPILING_DPCPP)
+
+ set(GINKGO_DPCPP_FLAGS ${GINKGO_DPCPP_FLAGS} PARENT_SCOPE)
+ target_compile_options(ginkgo_dpcpp PRIVATE "${GINKGO_DPCPP_FLAGS}")
++target_compile_options(ginkgo_dpcpp PRIVATE "${GINKGO_COMPILER_FLAGS}")
+ # Note: add MKL as PRIVATE not PUBLIC (MKL example shows) to avoid propagating
+ # find_package(MKL) everywhere when linking ginkgo (see the MKL example
+ # https://software.intel.com/content/www/us/en/develop/documentation/onemkl-windows-developer-guide/top/getting-started/cmake-config-for-onemkl.html)
+diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
+index 0c89530d1d..9a86ab9cd1 100644
+--- a/dpcpp/matrix/dense_kernels.dp.cpp
++++ b/dpcpp/matrix/dense_kernels.dp.cpp
+@@ -69,14 +69,14 @@ namespace dpcpp {
+ namespace dense {
+
+
++// Disable the 64 subgroup. CPU supports 64 now, but conj_transpose will
++// lead CL_OUT_OF_RESOURCES. TODO: investigate this issue.
+ using KCFG_1D = ConfigSet<11, 7>;
+ constexpr auto kcfg_1d_list =
+- syn::value_list<std::uint32_t, KCFG_1D::encode(512, 64),
+- KCFG_1D::encode(512, 32), KCFG_1D::encode(512, 16),
+- KCFG_1D::encode(256, 32), KCFG_1D::encode(256, 16),
+- KCFG_1D::encode(256, 8)>();
+-constexpr auto subgroup_list =
+- syn::value_list<std::uint32_t, 64, 32, 16, 8, 4>();
++ syn::value_list<std::uint32_t, KCFG_1D::encode(512, 32),
++ KCFG_1D::encode(512, 16), KCFG_1D::encode(256, 32),
++ KCFG_1D::encode(256, 16), KCFG_1D::encode(256, 8)>();
++constexpr auto subgroup_list = syn::value_list<std::uint32_t, 32, 16, 8, 4>();
+ constexpr auto kcfg_1d_array = syn::as_array(kcfg_1d_list);
+ constexpr int default_block_size = 256;
+
diff --git a/var/spack/repos/builtin/packages/ginkgo/package.py b/var/spack/repos/builtin/packages/ginkgo/package.py
index 4518f3cce1..694ae2702e 100644
--- a/var/spack/repos/builtin/packages/ginkgo/package.py
+++ b/var/spack/repos/builtin/packages/ginkgo/package.py
@@ -3,6 +3,7 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os
import sys
from spack import *
@@ -31,6 +32,7 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage):
variant('shared', default=True, description='Build shared libraries')
variant('full_optimizations', default=False, description='Compile with all optimizations')
variant('openmp', default=sys.platform != 'darwin', description='Build with OpenMP')
+ variant('oneapi', default=False, description='Build with oneAPI support')
variant('develtools', default=False, description='Compile with develtools enabled')
variant('hwloc', default=False, description='Enable HWLOC support')
variant('build_type', default='Release',
@@ -49,9 +51,13 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage):
depends_on('googletest', type="test")
depends_on('numactl', type="test", when="+hwloc")
+ depends_on('intel-oneapi-mkl', when="+oneapi")
+ depends_on('intel-oneapi-dpl', when="+oneapi")
+
conflicts('%gcc@:5.2.9')
conflicts("+rocm", when="@:1.1.1")
conflicts("+cuda", when="+rocm")
+ conflicts("+openmp", when="+oneapi")
# ROCm 4.1.0 breaks platform settings which breaks Ginkgo's HIP support.
conflicts("^hip@4.1.0:", when="@:1.3.0")
@@ -63,18 +69,39 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage):
patch('1.4.0_skip_invalid_smoke_tests.patch', when='@master')
patch('1.4.0_skip_invalid_smoke_tests.patch', when='@1.4.0')
+ # Newer DPC++ compilers use the updated SYCL 2020 standard which change
+ # kernel attribute propagation rules. This doesn't work well with the
+ # initial Ginkgo oneAPI support.
+ patch('1.4.0_dpcpp_use_old_standard.patch', when='+oneapi @master')
+ patch('1.4.0_dpcpp_use_old_standard.patch', when='+oneapi @1.4.0')
+
+ def setup_build_environment(self, env):
+ spec = self.spec
+ if '+oneapi' in spec:
+ env.set('MKLROOT',
+ join_path(spec['intel-oneapi-mkl'].prefix,
+ 'mkl', 'latest'))
+ env.set('DPL_ROOT',
+ join_path(spec['intel-oneapi-dpl'].prefix,
+ 'dpl', 'latest'))
+
def cmake_args(self):
# Check that the have the correct C++ standard is available
if self.spec.satisfies('@:1.2.0'):
try:
self.compiler.cxx11_flag
except UnsupportedCompilerFlag:
- InstallError('Ginkgo requires a C++11-compliant C++ compiler')
+ raise InstallError('Ginkgo requires a C++11-compliant C++ compiler')
else:
try:
self.compiler.cxx14_flag
except UnsupportedCompilerFlag:
- InstallError('Ginkgo requires a C++14-compliant C++ compiler')
+ raise InstallError('Ginkgo requires a C++14-compliant C++ compiler')
+
+ cxx_is_dpcpp = os.path.basename(self.compiler.cxx) == "dpcpp"
+ if self.spec.satisfies('+oneapi') and not cxx_is_dpcpp:
+ raise InstallError("Ginkgo's oneAPI backend requires the" +
+ "DPC++ compiler as main CXX compiler.")
spec = self.spec
from_variant = self.define_from_variant
@@ -85,6 +112,7 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage):
from_variant('BUILD_SHARED_LIBS', 'shared'),
from_variant('GINKGO_JACOBI_FULL_OPTIMIZATIONS', 'full_optimizations'),
from_variant('GINKGO_BUILD_HWLOC', 'hwloc'),
+ from_variant('GINKGO_BUILD_DPCPP', 'oneapi'),
from_variant('GINKGO_DEVEL_TOOLS', 'develtools'),
# As we are not exposing benchmarks, examples, tests nor doc
# as part of the installation, disable building them altogether.
--
cgit v1.2.3-70-g09d2