1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
|
# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""Utilities for managing paths in Spack.
TODO: this is really part of spack.config. Consolidate it.
"""
import contextlib
import getpass
import os
import re
import subprocess
import tempfile
import llnl.util.tty as tty
from llnl.util.lang import memoized
import spack.paths
import spack.util.spack_yaml as syaml
__all__ = [
'substitute_config_variables',
'substitute_path_variables',
'canonicalize_path']
# Substitutions to perform
replacements = {
'spack': spack.paths.prefix,
'user': getpass.getuser(),
'tempdir': tempfile.gettempdir(),
'user_config_path': spack.paths.user_config_path,
'user_cache_path': spack.paths.user_cache_path,
}
# This is intended to be longer than the part of the install path
# spack generates from the root path we give it. Included in the
# estimate:
#
# os-arch -> 30
# compiler -> 30
# package name -> 50 (longest is currently 47 characters)
# version -> 20
# hash -> 32
# buffer -> 138
# ---------------------
# total -> 300
SPACK_MAX_INSTALL_PATH_LENGTH = 300
#: Padded paths comprise directories with this name (or some prefix of it). :
#: It starts with two underscores to make it unlikely that prefix matches would
#: include some other component of the intallation path.
SPACK_PATH_PADDING_CHARS = '__spack_path_placeholder__'
@memoized
def get_system_path_max():
# Choose a conservative default
sys_max_path_length = 256
try:
path_max_proc = subprocess.Popen(['getconf', 'PATH_MAX', '/'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
proc_output = str(path_max_proc.communicate()[0].decode())
sys_max_path_length = int(proc_output)
except (ValueError, subprocess.CalledProcessError, OSError):
tty.msg('Unable to find system max path length, using: {0}'.format(
sys_max_path_length))
return sys_max_path_length
def substitute_config_variables(path):
"""Substitute placeholders into paths.
Spack allows paths in configs to have some placeholders, as follows:
- $env The active Spack environment.
- $spack The Spack instance's prefix
- $tempdir Default temporary directory returned by tempfile.gettempdir()
- $user The current user's username
- $user_config_path The user configuration directory (~/.spack, unless overridden)
- $user_cache_path The user cache directory (~/.spack, unless overridden)
These are substituted case-insensitively into the path, and users can
use either ``$var`` or ``${var}`` syntax for the variables. $env is only
replaced if there is an active environment, and should only be used in
environment yaml files.
"""
import spack.environment as ev # break circular
env = ev.active_environment()
if env:
replacements.update({'env': env.path})
else:
# If a previous invocation added env, remove it
replacements.pop('env', None)
# Look up replacements
def repl(match):
m = match.group(0).strip('${}')
return replacements.get(m.lower(), match.group(0))
# Replace $var or ${var}.
return re.sub(r'(\$\w+\b|\$\{\w+\})', repl, path)
def substitute_path_variables(path):
"""Substitute config vars, expand environment vars, expand user home."""
path = substitute_config_variables(path)
path = os.path.expandvars(path)
path = os.path.expanduser(path)
return path
def _get_padding_string(length):
spack_path_padding_size = len(SPACK_PATH_PADDING_CHARS)
num_reps = int(length / (spack_path_padding_size + 1))
extra_chars = length % (spack_path_padding_size + 1)
reps_list = [SPACK_PATH_PADDING_CHARS for i in range(num_reps)]
reps_list.append(SPACK_PATH_PADDING_CHARS[:extra_chars])
return os.path.sep.join(reps_list)
def add_padding(path, length):
"""Add padding subdirectories to path until total is length characters
Returns the padded path. If path is length - 1 or more characters long,
returns path. If path is length - 1 characters, warns that it is not
padding to length
Assumes path does not have a trailing path separator"""
padding_length = length - len(path)
if padding_length == 1:
# The only 1 character addition we can make to a path is `/`
# Spack internally runs normpath, so `foo/` will be reduced to `foo`
# Even if we removed this behavior from Spack, the user could normalize
# the path, removing the additional `/`.
# Because we can't expect one character of padding to show up in the
# resulting binaries, we warn the user and do not pad by a single char
tty.warn("Cannot pad path by exactly one character.")
if padding_length <= 0:
return path
# we subtract 1 from the padding_length to account for the path separator
# coming from os.path.join below
padding = _get_padding_string(padding_length - 1)
return os.path.join(path, padding)
def canonicalize_path(path):
"""Same as substitute_path_variables, but also take absolute path."""
# Get file in which path was written in case we need to make it absolute
# relative to that path.
filename = None
if isinstance(path, syaml.syaml_str):
filename = os.path.dirname(path._start_mark.name)
assert path._start_mark.name == path._end_mark.name
path = substitute_path_variables(path)
if not os.path.isabs(path):
if filename:
path = os.path.join(filename, path)
else:
path = os.path.abspath(path)
tty.debug("Using current working directory as base for abspath")
return os.path.normpath(path)
def longest_prefix_re(string, capture=True):
"""Return a regular expression that matches a the longest possible prefix of string.
i.e., if the input string is ``the_quick_brown_fox``, then::
m = re.compile(longest_prefix('the_quick_brown_fox'))
m.match('the_').group(1) == 'the_'
m.match('the_quick').group(1) == 'the_quick'
m.match('the_quick_brown_fox').group(1) == 'the_quick_brown_fox'
m.match('the_xquick_brown_fox').group(1) == 'the_'
m.match('the_quickx_brown_fox').group(1) == 'the_quick'
"""
if len(string) < 2:
return string
return "(%s%s%s?)" % (
"" if capture else "?:",
string[0],
longest_prefix_re(string[1:], capture=False)
)
#: regex cache for padding_filter function
_filter_re = None
def padding_filter(string):
"""Filter used to reduce output from path padding in log output.
This turns paths like this:
/foo/bar/__spack_path_placeholder__/__spack_path_placeholder__/...
Into paths like this:
/foo/bar/[padded-to-512-chars]/...
Where ``padded-to-512-chars`` indicates that the prefix was padded with
placeholders until it hit 512 characters. The actual value of this number
depends on what the `install_tree``'s ``padded_length`` is configured to.
For a path to match and be filtered, the placeholder must appear in its
entirety at least one time. e.g., "/spack/" would not be filtered, but
"/__spack_path_placeholder__/spack/" would be.
"""
global _filter_re
pad = spack.util.path.SPACK_PATH_PADDING_CHARS
if not _filter_re:
longest_prefix = longest_prefix_re(pad)
regex = (
r"((?:/[^/\s]*)*?)" # zero or more leading non-whitespace path components
r"(/{pad})+" # the padding string repeated one or more times
r"(/{longest_prefix})?(?=/)" # trailing prefix of padding as path component
)
regex = regex.replace("/", os.sep)
regex = regex.format(pad=pad, longest_prefix=longest_prefix)
_filter_re = re.compile(regex)
def replacer(match):
return "%s%s[padded-to-%d-chars]" % (
match.group(1),
os.sep,
len(match.group(0))
)
return _filter_re.sub(replacer, string)
@contextlib.contextmanager
def filter_padding():
"""Context manager to safely disable path padding in all Spack output.
This is needed because Spack's debug output gets extremely long when we use a
long padded installation path.
"""
padding = spack.config.get("config:install_tree:padded_length", None)
if padding:
# filter out all padding from the intsall command output
with tty.output_filter(padding_filter):
yield
else:
yield # no-op: don't filter unless padding is actually enabled
|
