1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
|
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import llnl.util.tty as tty
import spack.hooks.sbang as sbang
from spack.package import *
class Phist(CMakePackage):
"""The Pipelined, Hybrid-parallel Iterative Solver Toolkit provides
implementations of and interfaces to block iterative solvers for sparse
linear and eigenvalue problems. In contrast to other libraries we support
multiple backends (e.g. Trilinos, PETSc and our own optimized kernels),
and interfaces in multiple languages such as C, C++, Fortran 2003 and
Python. PHIST has a clear focus on portability and hardware performance:
in particular support row-major storage of block vectors and using GPUs
(via the ghost library or Trilinos/Tpetra).
"""
homepage = "https://bitbucket.org/essex/phist/"
url = "https://bitbucket.org/essex/phist/get/phist-1.9.6.tar.gz"
git = "https://bitbucket.org/essex/phist.git"
maintainers = ["jthies"]
tags = ["e4s"]
# phist is a required part of spack GitLab CI pipelines. In them, mpich is requested
# to provide 'mpi' like this: spack install phist ^mpich %gcc@7.5.0
# Failure of this command to succeed breaks spack's gitlab CI pipelines!
version("develop", branch="devel")
version("master", branch="master")
# updated the Trilinos interface to work with trilinos@13:
# without using deprecated interfaces in tpetra
version("1.10.0", sha256="3ec660c85d37818ee219edc80e977140dfb062bdca1f38623c94a45d13634bd1")
# phist-1.9.6 updated from the older "use mpi" to the newer "use mpi_f08" (MPI3.1):
# The motivation was fixing it on Cray: https://github.com/spack/spack/issues/26002
# Documentation: https://www.mpi-forum.org/docs/mpi-3.1/mpi31-report/node408.htm
# mpich does not provide mpi_f08.mod with gfortran-[789], it needs gfortran>=10:
# https://stackoverflow.com/questions/65750862
version("1.9.6", sha256="98ed5ccb22bb98d5b6bf9de0c9960105473e5244978853070b9a3c44138db662")
# As spack GitLab CI pipelines use ^mpich %gcc@7.5.0, @1.9.6 with it can't be used:
# A conflict would be possible, but when %gcc@10: or another compiler is available,
# clingo can select the other compiler despite a request for %gcc@7.5.0 in place,
# at least when the version is requested: spack solve phist@1.9.6 ^mpich %gcc@7.5.0
# With conflicts('mpich', when='@1.9.6:%gcc@:9'), this is the result:
# spack solve phist@1.9.6 ^mpich %gcc@7.5.0|grep -e phist -e mpich|sed 's/+.*//'
# phist@1.9.6%gcc@11.2.0
# ^mpich@3.4.2%gcc@7.5.0~argobots
# A mismatch of gfortan between gcc@7.5.0(for mpich) and @10:(phist) would not work,
# and also does not solve the build problem.
# Instead, a check with a helpful error message is added to the build (see below).
# and we use preferred=True to select 1.9.5 by default:
version(
"1.9.5",
sha256="24faa3373003f185c82a658c510e36cba9acc4110eb60cbfded9de370ae9ea32",
preferred=True,
)
version("1.9.4", sha256="9dde3ca0480358fa0877ec8424aaee4011c5defc929219a5930388a7cdb4c8a6")
version("1.9.3", sha256="3ab7157e9f535a4c8537846cb11b516271ef13f82d0f8ebb7f96626fb9ab86cf")
version("1.9.2", sha256="289678fa7172708f5d32d6bd924c8fdfe72b413bba5bbb8ce6373c85c5ec5ae5")
version("1.9.1", sha256="6e6411115ec48afe605b4f2179e9bc45d60f15459428f474f3f32b80d2830f1f")
version("1.9.0", sha256="990d3308fc0083ed0f9f565d00c649ee70c3df74d44cbe5f19dfe05263d06559")
version("1.8.0", sha256="ee42946bce187e126452053b5f5c200b57b6e40ee3f5bcf0751f3ced585adeb0")
version("1.7.5", sha256="f11fe27f2aa13d69eb285cc0f32c33c1603fa1286b84e54c81856c6f2bdef500")
version("1.7.4", sha256="c5324f639b8c95b07cd29c3cd8dd7dd576c84a0b9228dbb88f8b87605424419e")
version("1.7.3", sha256="ab2d853c9ba13bcd3069fcc61c359cb412466a2e4b22ebbd2f5263cffa685126")
version("1.7.2", sha256="29b504d78b5efd57b87d2ca6e20bc8a32b1ba55b40f5a5b7189cc0d28e43bcc0")
version("1.6.1", sha256="4ed4869f24f920a494aeae0f7d1d94fe9efce55ebe0d298a5948c9603e07994d")
version("1.6.0", sha256="667a967b37d248242c275226c96efc447ef73a2b15f241c6a588d570d7fac07b")
version("1.4.3", sha256="9cc1c7ba7f7a04e94f4497da14199e4631a0d02d0e4187f3e16f4c242dc777c1")
variant(
name="kernel_lib",
default="builtin",
description="select the kernel library (backend) for phist",
values=["builtin", "epetra", "tpetra", "petsc", "eigen", "ghost"],
)
variant(name="int64", default=True, description="Use 64-bit global indices.")
variant(
name="outlev",
default="2",
values=["0", "1", "2", "3", "4", "5"],
description="verbosity. 0: errors 1: +warnings 2: +info "
"3: +verbose 4: +extreme 5: +debug",
)
variant(
"host",
default=True,
description="allow PHIST to use compiler flags that lead to host-"
"specific code. Set this to False when cross-compiling.",
)
variant("shared", default=True, description="Enables the build of shared libraries")
variant(
"mpi",
default=True,
description="enable/disable MPI (note that the kernel library may "
"not support this choice)",
)
variant("openmp", default=True, description="enable/disable OpenMP")
variant(
"parmetis",
default=False,
description="enable/disable ParMETIS partitioning (only actually "
"used with kernel_lib=builtin)",
)
variant(
"scamac",
default=True,
description='enable/disable building the "SCAlable MAtrix '
'Collection" matrix generators.',
)
variant(
"trilinos",
default=False,
description="enable/disable Trilinos third-party libraries. "
"For all kernel_libs, we can use Belos and Anasazi "
"iterative solvers. For the Trilinos backends "
"(kernel_lib=epetra|tpetra) we can use preconditioner "
"packages such as Ifpack, Ifpack2 and ML.",
)
variant(
"fortran",
default=True,
description="generate Fortran 2003 bindings (requires Python3 and " "a Fortran compiler)",
)
# in older versions, it is not possible to completely turn off OpenMP
conflicts("~openmp", when="@:1.7.3")
# in older versions, it is not possible to turn off the use of host-
# specific compiler flags in Release mode.
conflicts("~host", when="@:1.7.3")
# builtin always uses 64-bit indices
conflicts("~int64", when="kernel_lib=builtin")
conflicts("+int64", when="kernel_lib=eigen")
# ###################### Patches ##########################
# Only applies to 1.9.4: While SSE instructions are handled correctly,
# build fails on ppc64le unless -DNO_WARN_X86_INTRINSICS is defined.
patch("ppc64_sse.patch", when="@1.9.4")
patch("update_tpetra_gotypes.patch", when="@1.6:1.8")
patch("sbang.patch", when="+fortran")
# ###################### Dependencies ##########################
depends_on("cmake@3.8:", type="build")
depends_on("blas")
depends_on("lapack")
# Python 3 or later is required for generating the Fortran 2003 bindings
# since version 1.7, you can get rid of the dependency by switching off
# the feature (e.g. use the '~fortran' variant)
depends_on("python@3:", when="@1.7: +fortran", type="build")
depends_on("mpi", when="+mpi")
depends_on("trilinos@12:+tpetra gotype=long_long", when="kernel_lib=tpetra +int64")
depends_on("trilinos@12:+tpetra gotype=int", when="kernel_lib=tpetra ~int64")
# Epetra backend also works with older Trilinos versions
depends_on("trilinos+epetra", when="kernel_lib=epetra")
depends_on("petsc +int64", when="kernel_lib=petsc +int64")
depends_on("petsc ~int64", when="kernel_lib=petsc ~int64")
depends_on("eigen", when="kernel_lib=eigen")
depends_on("ghost", when="kernel_lib=ghost")
depends_on("trilinos+anasazi+belos", when="+trilinos")
depends_on("parmetis+int64", when="+parmetis+int64")
depends_on("parmetis~int64", when="+parmetis~int64")
depends_on("py-pytest", type="test")
depends_on("py-numpy", type="test", when="+mpi")
# The test_install compiles the examples and needs pkgconfig for it
depends_on("pkgconfig", type="test")
# in 1.10 we removed some use of deprecated Trilinos interfaces
# (some functions in tpetra were renamed)
conflicts("^trilinos@13.4:", when="@:1.9 kernel_lib=tpetra")
conflicts("^trilinos@:13.2", when="@1.10: kernel_lib=tpetra")
# Fortran 2003 bindings were included in version 1.7, previously they
# required a separate package
conflicts("+fortran", when="@:1.6")
# older gcc's may produce incorrect SIMD code and fail
# to compile some OpenMP statements
conflicts("%gcc@:4.9.1")
# gcc@10: Error: Rank mismatch between actual argument at (1)
# and actual argument at (2) (scalar and rank-1)
conflicts("%gcc@10:", when="@:1.9.0")
# the phist repo came with it's own FindMPI.cmake before, which may cause some other
# MPI installation to be used than the one spack wants.
def patch(self):
if self.spec.satisfies("@1.9.6"):
filter_file("USE mpi", "use mpi_f08", "src/kernels/builtin/crsmat_module.F90")
# filter_file('use mpi', 'use mpi_f08', -> Needs more fixes
# 'fortran_bindings/phist_testing.F90')
# These are not needed for the build but as a reminder to be consistent:
filter_file("use mpi", "use mpi_f08", "fortran_bindings/test/core.F90")
filter_file("use mpi", "use mpi_f08", "fortran_bindings/test/jada.F90")
filter_file("use mpi", "use mpi_f08", "fortran_bindings/test/kernels.F90")
if "^mpich" in self.spec and self.spec.satisfies("%gcc@:9"):
raise InstallError("PR 26773: gcc<10 can't build phist>1.9.5 w/ ^mpich")
if self.spec.satisfies("@:1.9.5"):
# Tag '1.9.5' has moved to an older commit later without fixing the version:
filter_file("VERSION_PATCH 4", "VERSION_PATCH 5", "CMakeLists.txt")
force_remove("cmake/FindMPI.cmake")
# mpiexec -n12 puts a lot of stress on a pod and gets stuck in a loop very often
test = FileFilter("CMakeLists.txt")
test.filter("1 2 3 12", "1 2 3")
test.filter("12/", "6/")
test.filter("TEST_DRIVERS_NUM_THREADS 6", "TEST_DRIVERS_NUM_THREADS 3")
def setup_build_environment(self, env):
env.set("SPACK_SBANG", sbang.sbang_install_path())
def cmake_args(self):
spec = self.spec
define = CMakePackage.define
if spec.satisfies("kernel_lib=builtin") and spec.satisfies("~mpi"):
raise InstallError("~mpi not possible with kernel_lib=builtin!")
kernel_lib = spec.variants["kernel_lib"].value
outlev = spec.variants["outlev"].value
lapacke_libs = (
spec["lapack:c"].libs + spec["blas:c"].libs + find_system_libraries(["libm"])
).joined(";")
lapacke_include_dir = spec["lapack:c"].headers.directories[0]
args = [
"-DPHIST_USE_CCACHE=OFF",
"-DPHIST_KERNEL_LIB=%s" % kernel_lib,
"-DPHIST_OUTLEV=%s" % outlev,
"-DTPL_LAPACKE_LIBRARIES=%s" % lapacke_libs,
"-DTPL_LAPACKE_INCLUDE_DIRS=%s" % lapacke_include_dir,
self.define_from_variant("PHIST_ENABLE_MPI", "mpi"),
self.define_from_variant("PHIST_ENABLE_OPENMP", "openmp"),
self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
self.define_from_variant("PHIST_ENABLE_SCAMAC", "scamac"),
"-DPHIST_USE_TRILINOS_TPLS:BOOL=%s" % ("ON" if "^trilinos" in spec else "OFF"),
"-DPHIST_USE_SOLVER_TPLS:BOOL=%s"
% ("ON" if "^trilinos+belos+anasazi" in spec else "OFF"),
"-DPHIST_USE_PRECON_TPLS:BOOL=%s" % ("ON" if "^trilinos" in spec else "OFF"),
self.define_from_variant("XSDK_ENABLE_Fortran", "fortran"),
"-DXSDK_INDEX_SIZE=%s" % ("64" if "+int64" in spec else "32"),
self.define_from_variant("PHIST_HOST_OPTIMIZE", "host"),
]
# Force phist to use the MPI wrappers instead of raw compilers
# (see issue #26002 and the comment in the trilinos package.py)
if "+mpi" in spec:
args.extend(
[
define("CMAKE_C_COMPILER", spec["mpi"].mpicc),
define("CMAKE_CXX_COMPILER", spec["mpi"].mpicxx),
define("CMAKE_Fortran_COMPILER", spec["mpi"].mpifc),
define("MPI_HOME", spec["mpi"].prefix),
]
)
# Workaround for non-compliant and possibly broken code needed for gcc@10:
# MPI_Allreduce(localRes, res, 1, MPI_LOGICAL, MPI_LAND, map1%comm, ierr);
# MPI_Allreduce(my_ierr,global_ierr,1,MPI_INTEGER,MPI_MAX,map1%comm,mpi_ierr)
# Error: Type mismatch between actual argument at (1)
# and actual argument at (2) (LOGICAL(4)/INTEGER(4)).
if spec.satisfies("%gcc@10:"):
args.append(define("CMAKE_Fortran_FLAGS", "-fallow-argument-mismatch"))
return args
def check(self):
with working_dir(self.build_directory):
# This affects all versions of phist with ^mpich with all gcc versions:
if "^mpich" in self.spec:
hint = "Expect tests to timeout with mpich. Should work with: ^openmpi."
tty.warn("========================== %s =======================" % hint)
try:
make("check")
except spack.util.executable.ProcessError:
raise InstallError("run-test of phist ^mpich: Hint: " + hint)
else:
make("check")
@run_after("install")
@on_package_attributes(run_tests=True)
def test_install(self):
# The build script of test_install expects the sources to be copied here:
install_tree(
join_path(self.stage.source_path, "exampleProjects"),
join_path(self.stage.path, "exampleProjects"),
)
with working_dir(self.build_directory):
make("test_install")
@property
def parallel(self):
return self.spec.satisfies("@1.8:")
|