1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
|
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""
This module contains the traversal logic and models that can be used to generate
depfiles from an environment.
"""
import os
import re
from enum import Enum
from typing import List, Optional
import spack.environment.environment as ev
import spack.spec
import spack.traverse as traverse
class UseBuildCache(Enum):
ONLY = 1
NEVER = 2
AUTO = 3
@staticmethod
def from_string(s: str) -> "UseBuildCache":
if s == "only":
return UseBuildCache.ONLY
elif s == "never":
return UseBuildCache.NEVER
elif s == "auto":
return UseBuildCache.AUTO
raise ValueError(f"invalid value for UseBuildCache: {s}")
def _deptypes(use_buildcache: UseBuildCache):
"""What edges should we follow for a given node? If it's a cache-only
node, then we can drop build type deps."""
return ("link", "run") if use_buildcache == UseBuildCache.ONLY else ("build", "link", "run")
class DepfileNode:
"""Contains a spec, a subset of its dependencies, and a flag whether it should be
buildcache only/never/auto."""
def __init__(
self, target: spack.spec.Spec, prereqs: List[spack.spec.Spec], buildcache: UseBuildCache
):
self.target = MakefileSpec(target)
self.prereqs = list(MakefileSpec(x) for x in prereqs)
if buildcache == UseBuildCache.ONLY:
self.buildcache_flag = "--use-buildcache=only"
elif buildcache == UseBuildCache.NEVER:
self.buildcache_flag = "--use-buildcache=never"
else:
self.buildcache_flag = ""
class DepfileSpecVisitor:
"""This visitor produces an adjacency list of a (reduced) DAG, which
is used to generate depfile targets with their prerequisites. Currently
it only drops build deps when using buildcache only mode.
Note that the DAG could be reduced even more by dropping build edges of specs
installed at the moment the depfile is generated, but that would produce
stateful depfiles that would not fail when the database is wiped later."""
def __init__(self, pkg_buildcache: UseBuildCache, deps_buildcache: UseBuildCache):
self.adjacency_list: List[DepfileNode] = []
self.pkg_buildcache = pkg_buildcache
self.deps_buildcache = deps_buildcache
self.deptypes_root = _deptypes(pkg_buildcache)
self.deptypes_deps = _deptypes(deps_buildcache)
def neighbors(self, node):
"""Produce a list of spec to follow from node"""
deptypes = self.deptypes_root if node.depth == 0 else self.deptypes_deps
return traverse.sort_edges(node.edge.spec.edges_to_dependencies(deptype=deptypes))
def accept(self, node):
self.adjacency_list.append(
DepfileNode(
target=node.edge.spec,
prereqs=[edge.spec for edge in self.neighbors(node)],
buildcache=self.pkg_buildcache if node.depth == 0 else self.deps_buildcache,
)
)
# We already accepted this
return True
class MakefileSpec(object):
"""Limited interface to spec to help generate targets etc. without
introducing unwanted special characters.
"""
_pattern = None
def __init__(self, spec):
self.spec = spec
def safe_name(self):
return self.safe_format("{name}-{version}-{hash}")
def spec_hash(self):
return self.spec.dag_hash()
def safe_format(self, format_str):
unsafe_result = self.spec.format(format_str)
if not MakefileSpec._pattern:
MakefileSpec._pattern = re.compile(r"[^A-Za-z0-9_.-]")
return MakefileSpec._pattern.sub("_", unsafe_result)
def unsafe_format(self, format_str):
return self.spec.format(format_str)
class MakefileModel:
"""This class produces all data to render a makefile for specs of an environment."""
def __init__(
self,
env: ev.Environment,
roots: List[spack.spec.Spec],
adjacency_list: List[DepfileNode],
make_prefix: Optional[str],
jobserver: bool,
):
"""
Args:
env: environment to generate the makefile for
roots: specs that get built in the default target
adjacency_list: list of DepfileNode, mapping specs to their dependencies
make_prefix: prefix for makefile targets
jobserver: when enabled, make will invoke Spack with jobserver support. For
dry-run this should be disabled.
"""
# Currently we can only use depfile with an environment since Spack needs to
# find the concrete specs somewhere.
self.env_path = env.path
# These specs are built in the default target.
self.roots = list(MakefileSpec(x) for x in roots)
# The SPACK_PACKAGE_IDS variable is "exported", which can be used when including
# generated makefiles to add post-install hooks, like pushing to a buildcache,
# running tests, etc.
if make_prefix is None:
self.make_prefix = os.path.join(env.env_subdir_path, "makedeps")
self.pkg_identifier_variable = "SPACK_PACKAGE_IDS"
else:
# NOTE: GNU Make allows directory separators in variable names, so for consistency
# we can namespace this variable with the same prefix as targets.
self.make_prefix = make_prefix
self.pkg_identifier_variable = os.path.join(make_prefix, "SPACK_PACKAGE_IDS")
# And here we collect a tuple of (target, prereqs, dag_hash, nice_name, buildcache_flag)
self.make_adjacency_list = [
(
item.target.safe_name(),
" ".join(self._install_target(s.safe_name()) for s in item.prereqs),
item.target.spec_hash(),
item.target.unsafe_format(
"{name}{@version}{%compiler}{variants}{arch=architecture}"
),
item.buildcache_flag,
)
for item in adjacency_list
]
# Root specs without deps are the prereqs for the environment target
self.root_install_targets = [self._install_target(s.safe_name()) for s in self.roots]
self.jobserver_support = "+" if jobserver else ""
# All package identifiers, used to generate the SPACK_PACKAGE_IDS variable
self.all_pkg_identifiers: List[str] = []
# All install and install-deps targets
self.all_install_related_targets: List[str] = []
# Convenience shortcuts: ensure that `make install/pkg-version-hash` triggers
# <absolute path to env>/.spack-env/makedeps/install/pkg-version-hash in case
# we don't have a custom make target prefix.
self.phony_convenience_targets: List[str] = []
for node in adjacency_list:
tgt = node.target.safe_name()
self.all_pkg_identifiers.append(tgt)
self.all_install_related_targets.append(self._install_target(tgt))
self.all_install_related_targets.append(self._install_deps_target(tgt))
if make_prefix is None:
self.phony_convenience_targets.append(os.path.join("install", tgt))
self.phony_convenience_targets.append(os.path.join("install-deps", tgt))
def _target(self, name: str) -> str:
# The `all` and `clean` targets are phony. It doesn't make sense to
# have /abs/path/to/env/metadir/{all,clean} targets. But it *does* make
# sense to have a prefix like `env/all`, `env/clean` when they are
# supposed to be included
if name in ("all", "clean") and os.path.isabs(self.make_prefix):
return name
else:
return os.path.join(self.make_prefix, name)
def _install_target(self, name: str) -> str:
return os.path.join(self.make_prefix, "install", name)
def _install_deps_target(self, name: str) -> str:
return os.path.join(self.make_prefix, "install-deps", name)
def to_dict(self):
return {
"all_target": self._target("all"),
"env_target": self._target("env"),
"clean_target": self._target("clean"),
"all_install_related_targets": " ".join(self.all_install_related_targets),
"root_install_targets": " ".join(self.root_install_targets),
"dirs_target": self._target("dirs"),
"environment": self.env_path,
"install_target": self._target("install"),
"install_deps_target": self._target("install-deps"),
"any_hash_target": self._target("%"),
"jobserver_support": self.jobserver_support,
"adjacency_list": self.make_adjacency_list,
"phony_convenience_targets": " ".join(self.phony_convenience_targets),
"pkg_ids_variable": self.pkg_identifier_variable,
"pkg_ids": " ".join(self.all_pkg_identifiers),
}
@staticmethod
def from_env(
env: ev.Environment,
*,
filter_specs: Optional[List[spack.spec.Spec]] = None,
pkg_buildcache: UseBuildCache = UseBuildCache.AUTO,
dep_buildcache: UseBuildCache = UseBuildCache.AUTO,
make_prefix: Optional[str] = None,
jobserver: bool = True,
) -> "MakefileModel":
"""Produces a MakefileModel from an environment and a list of specs.
Args:
env: the environment to use
filter_specs: if provided, only these specs will be built from the environment,
otherwise the environment roots are used.
pkg_buildcache: whether to only use the buildcache for top-level specs.
dep_buildcache: whether to only use the buildcache for non-top-level specs.
make_prefix: the prefix for the makefile targets
jobserver: when enabled, make will invoke Spack with jobserver support. For
dry-run this should be disabled.
"""
# If no specs are provided as a filter, build all the specs in the environment.
if filter_specs:
entrypoints = [env.matching_spec(s) for s in filter_specs]
else:
entrypoints = [s for _, s in env.concretized_specs()]
visitor = DepfileSpecVisitor(pkg_buildcache, dep_buildcache)
traverse.traverse_breadth_first_with_visitor(
entrypoints, traverse.CoverNodesVisitor(visitor, key=lambda s: s.dag_hash())
)
return MakefileModel(env, entrypoints, visitor.adjacency_list, make_prefix, jobserver)
|