summaryrefslogtreecommitdiff
path: root/lib/spack/llnl
diff options
context:
space:
mode:
authorTodd Gamblin <tgamblin@llnl.gov>2022-07-30 15:19:18 -0700
committerTodd Gamblin <tgamblin@llnl.gov>2022-07-31 13:29:20 -0700
commitf52f6e99dbf1131886a80112b8c79dfc414afb7c (patch)
tree05cb7d64b2395922f2f24683da49f472075be12c /lib/spack/llnl
parent549ba1ed32372c67fc57271cde3797d58b7dec6e (diff)
downloadspack-f52f6e99dbf1131886a80112b8c79dfc414afb7c.tar.gz
spack-f52f6e99dbf1131886a80112b8c79dfc414afb7c.tar.bz2
spack-f52f6e99dbf1131886a80112b8c79dfc414afb7c.tar.xz
spack-f52f6e99dbf1131886a80112b8c79dfc414afb7c.zip
black: reformat entire repository with black
Diffstat (limited to 'lib/spack/llnl')
-rw-r--r--lib/spack/llnl/util/argparsewriter.py86
-rw-r--r--lib/spack/llnl/util/compat.py28
-rw-r--r--lib/spack/llnl/util/filesystem.py444
-rw-r--r--lib/spack/llnl/util/lang.py198
-rw-r--r--lib/spack/llnl/util/link_tree.py136
-rw-r--r--lib/spack/llnl/util/lock.py164
-rw-r--r--lib/spack/llnl/util/multiproc.py4
-rw-r--r--lib/spack/llnl/util/symlink.py16
-rw-r--r--lib/spack/llnl/util/tty/__init__.py126
-rw-r--r--lib/spack/llnl/util/tty/colify.py74
-rw-r--r--lib/spack/llnl/util/tty/color.py90
-rw-r--r--lib/spack/llnl/util/tty/log.py145
-rw-r--r--lib/spack/llnl/util/tty/pty.py54
13 files changed, 801 insertions, 764 deletions
diff --git a/lib/spack/llnl/util/argparsewriter.py b/lib/spack/llnl/util/argparsewriter.py
index eb35d26aa8..a8db508c2f 100644
--- a/lib/spack/llnl/util/argparsewriter.py
+++ b/lib/spack/llnl/util/argparsewriter.py
@@ -29,8 +29,8 @@ class Command(object):
- optionals: list of optional arguments (list)
- subcommands: list of subcommand parsers (list)
"""
- def __init__(self, prog, description, usage,
- positionals, optionals, subcommands):
+
+ def __init__(self, prog, description, usage, positionals, optionals, subcommands):
self.prog = prog
self.description = description
self.usage = usage
@@ -71,15 +71,15 @@ class ArgparseWriter(argparse.HelpFormatter):
"""
self.parser = parser
- split_prog = parser.prog.split(' ')
+ split_prog = parser.prog.split(" ")
split_prog[-1] = prog
- prog = ' '.join(split_prog)
+ prog = " ".join(split_prog)
description = parser.description
fmt = parser._get_formatter()
actions = parser._actions
groups = parser._mutually_exclusive_groups
- usage = fmt._format_usage(None, actions, groups, '').strip()
+ usage = fmt._format_usage(None, actions, groups, "").strip()
# Go through actions and split them into optionals, positionals,
# and subcommands
@@ -90,8 +90,8 @@ class ArgparseWriter(argparse.HelpFormatter):
if action.option_strings:
flags = action.option_strings
dest_flags = fmt._format_action_invocation(action)
- help = self._expand_help(action) if action.help else ''
- help = help.replace('\n', ' ')
+ help = self._expand_help(action) if action.help else ""
+ help = help.replace("\n", " ")
optionals.append((flags, dest_flags, help))
elif isinstance(action, argparse._SubParsersAction):
for subaction in action._choices_actions:
@@ -100,20 +100,19 @@ class ArgparseWriter(argparse.HelpFormatter):
# Look for aliases of the form 'name (alias, ...)'
if self.aliases:
- match = re.match(r'(.*) \((.*)\)', subaction.metavar)
+ match = re.match(r"(.*) \((.*)\)", subaction.metavar)
if match:
- aliases = match.group(2).split(', ')
+ aliases = match.group(2).split(", ")
for alias in aliases:
subparser = action._name_parser_map[alias]
subcommands.append((subparser, alias))
else:
args = fmt._format_action_invocation(action)
- help = self._expand_help(action) if action.help else ''
- help = help.replace('\n', ' ')
+ help = self._expand_help(action) if action.help else ""
+ help = help.replace("\n", " ")
positionals.append((args, help))
- return Command(
- prog, description, usage, positionals, optionals, subcommands)
+ return Command(prog, description, usage, positionals, optionals, subcommands)
def format(self, cmd):
"""Returns the string representation of a single node in the
@@ -161,14 +160,13 @@ class ArgparseWriter(argparse.HelpFormatter):
raise
-_rst_levels = ['=', '-', '^', '~', ':', '`']
+_rst_levels = ["=", "-", "^", "~", ":", "`"]
class ArgparseRstWriter(ArgparseWriter):
"""Write argparse output as rst sections."""
- def __init__(self, prog, out=None, aliases=False,
- rst_levels=_rst_levels):
+ def __init__(self, prog, out=None, aliases=False, rst_levels=_rst_levels):
"""Create a new ArgparseRstWriter.
Parameters:
@@ -217,11 +215,12 @@ class ArgparseRstWriter(ArgparseWriter):
{1}
{2}
-""".format(prog.replace(' ', '-'), prog,
- self.rst_levels[self.level] * len(prog))
+""".format(
+ prog.replace(" ", "-"), prog, self.rst_levels[self.level] * len(prog)
+ )
def description(self, description):
- return description + '\n\n'
+ return description + "\n\n"
def usage(self, usage):
return """\
@@ -229,33 +228,39 @@ class ArgparseRstWriter(ArgparseWriter):
{0}
-""".format(usage)
+""".format(
+ usage
+ )
def begin_positionals(self):
- return '\n**Positional arguments**\n\n'
+ return "\n**Positional arguments**\n\n"
def positional(self, name, help):
return """\
{0}
{1}
-""".format(name, help)
+""".format(
+ name, help
+ )
def end_positionals(self):
- return ''
+ return ""
def begin_optionals(self):
- return '\n**Optional arguments**\n\n'
+ return "\n**Optional arguments**\n\n"
def optional(self, opts, help):
return """\
``{0}``
{1}
-""".format(opts, help)
+""".format(
+ opts, help
+ )
def end_optionals(self):
- return ''
+ return ""
def begin_subcommands(self, subcommands):
string = """
@@ -267,11 +272,10 @@ class ArgparseRstWriter(ArgparseWriter):
"""
for cmd, _ in subcommands:
- prog = re.sub(r'^[^ ]* ', '', cmd.prog)
- string += ' * :ref:`{0} <{1}>`\n'.format(
- prog, cmd.prog.replace(' ', '-'))
+ prog = re.sub(r"^[^ ]* ", "", cmd.prog)
+ string += " * :ref:`{0} <{1}>`\n".format(prog, cmd.prog.replace(" ", "-"))
- return string + '\n'
+ return string + "\n"
class ArgparseCompletionWriter(ArgparseWriter):
@@ -306,9 +310,11 @@ class ArgparseCompletionWriter(ArgparseWriter):
# Flatten lists of lists
optionals = [x for xx in optionals for x in xx]
- return (self.start_function(cmd.prog) +
- self.body(positionals, optionals, subcommands) +
- self.end_function(cmd.prog))
+ return (
+ self.start_function(cmd.prog)
+ + self.body(positionals, optionals, subcommands)
+ + self.end_function(cmd.prog)
+ )
def start_function(self, prog):
"""Returns the syntax needed to begin a function definition.
@@ -319,8 +325,8 @@ class ArgparseCompletionWriter(ArgparseWriter):
Returns:
str: the function definition beginning
"""
- name = prog.replace('-', '_').replace(' ', '_')
- return '\n_{0}() {{'.format(name)
+ name = prog.replace("-", "_").replace(" ", "_")
+ return "\n_{0}() {{".format(name)
def end_function(self, prog=None):
"""Returns the syntax needed to end a function definition.
@@ -331,7 +337,7 @@ class ArgparseCompletionWriter(ArgparseWriter):
Returns:
str: the function definition ending
"""
- return '}\n'
+ return "}\n"
def body(self, positionals, optionals, subcommands):
"""Returns the body of the function.
@@ -344,7 +350,7 @@ class ArgparseCompletionWriter(ArgparseWriter):
Returns:
str: the function body
"""
- return ''
+ return ""
def positionals(self, positionals):
"""Returns the syntax for reporting positional arguments.
@@ -355,7 +361,7 @@ class ArgparseCompletionWriter(ArgparseWriter):
Returns:
str: the syntax for positional arguments
"""
- return ''
+ return ""
def optionals(self, optionals):
"""Returns the syntax for reporting optional flags.
@@ -366,7 +372,7 @@ class ArgparseCompletionWriter(ArgparseWriter):
Returns:
str: the syntax for optional flags
"""
- return ''
+ return ""
def subcommands(self, subcommands):
"""Returns the syntax for reporting subcommands.
@@ -377,4 +383,4 @@ class ArgparseCompletionWriter(ArgparseWriter):
Returns:
str: the syntax for subcommand parsers
"""
- return ''
+ return ""
diff --git a/lib/spack/llnl/util/compat.py b/lib/spack/llnl/util/compat.py
index ca914d0fb6..ebe509f3a7 100644
--- a/lib/spack/llnl/util/compat.py
+++ b/lib/spack/llnl/util/compat.py
@@ -18,22 +18,22 @@ else:
map = map
zip = zip
from itertools import zip_longest as zip_longest # novm # noqa: F401
- from urllib.parse import urlencode as urlencode # novm # noqa: F401
- from urllib.request import urlopen as urlopen # novm # noqa: F401
+ from urllib.parse import urlencode as urlencode # novm # noqa: F401
+ from urllib.request import urlopen as urlopen # novm # noqa: F401
if sys.version_info >= (3, 3):
- from collections.abc import Hashable as Hashable # novm
- from collections.abc import Iterable as Iterable # novm
- from collections.abc import Mapping as Mapping # novm
- from collections.abc import MutableMapping as MutableMapping # novm
+ from collections.abc import Hashable as Hashable # novm
+ from collections.abc import Iterable as Iterable # novm
+ from collections.abc import Mapping as Mapping # novm
+ from collections.abc import MutableMapping as MutableMapping # novm
from collections.abc import MutableSequence as MutableSequence # novm
- from collections.abc import MutableSet as MutableSet # novm
- from collections.abc import Sequence as Sequence # novm
+ from collections.abc import MutableSet as MutableSet # novm
+ from collections.abc import Sequence as Sequence # novm
else:
- from collections import Hashable as Hashable # noqa: F401
- from collections import Iterable as Iterable # noqa: F401
- from collections import Mapping as Mapping # noqa: F401
- from collections import MutableMapping as MutableMapping # noqa: F401
+ from collections import Hashable as Hashable # noqa: F401
+ from collections import Iterable as Iterable # noqa: F401
+ from collections import Mapping as Mapping # noqa: F401
+ from collections import MutableMapping as MutableMapping # noqa: F401
from collections import MutableSequence as MutableSequence # noqa: F401
- from collections import MutableSet as MutableSet # noqa: F401
- from collections import Sequence as Sequence # noqa: F401
+ from collections import MutableSet as MutableSet # noqa: F401
+ from collections import Sequence as Sequence # noqa: F401
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index 185f45d131..1740fb71c0 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -27,7 +27,7 @@ from llnl.util.symlink import symlink
from spack.util.executable import Executable
from spack.util.path import path_to_os_path, system_path_filter
-is_windows = _platform == 'win32'
+is_windows = _platform == "win32"
if not is_windows:
import grp
@@ -37,56 +37,57 @@ else:
__all__ = [
- 'FileFilter',
- 'FileList',
- 'HeaderList',
- 'LibraryList',
- 'ancestor',
- 'can_access',
- 'change_sed_delimiter',
- 'copy_mode',
- 'filter_file',
- 'find',
- 'find_headers',
- 'find_all_headers',
- 'find_libraries',
- 'find_system_libraries',
- 'fix_darwin_install_name',
- 'force_remove',
- 'force_symlink',
- 'getuid',
- 'chgrp',
- 'chmod_x',
- 'copy',
- 'install',
- 'copy_tree',
- 'install_tree',
- 'is_exe',
- 'join_path',
- 'last_modification_time_recursive',
- 'library_extensions',
- 'mkdirp',
- 'partition_path',
- 'prefixes',
- 'remove_dead_links',
- 'remove_directory_contents',
- 'remove_if_dead_link',
- 'remove_linked_tree',
- 'rename',
- 'set_executable',
- 'set_install_permissions',
- 'touch',
- 'touchp',
- 'traverse_tree',
- 'unset_executable_mode',
- 'working_dir',
- 'keep_modification_time'
+ "FileFilter",
+ "FileList",
+ "HeaderList",
+ "LibraryList",
+ "ancestor",
+ "can_access",
+ "change_sed_delimiter",
+ "copy_mode",
+ "filter_file",
+ "find",
+ "find_headers",
+ "find_all_headers",
+ "find_libraries",
+ "find_system_libraries",
+ "fix_darwin_install_name",
+ "force_remove",
+ "force_symlink",
+ "getuid",
+ "chgrp",
+ "chmod_x",
+ "copy",
+ "install",
+ "copy_tree",
+ "install_tree",
+ "is_exe",
+ "join_path",
+ "last_modification_time_recursive",
+ "library_extensions",
+ "mkdirp",
+ "partition_path",
+ "prefixes",
+ "remove_dead_links",
+ "remove_directory_contents",
+ "remove_if_dead_link",
+ "remove_linked_tree",
+ "rename",
+ "set_executable",
+ "set_install_permissions",
+ "touch",
+ "touchp",
+ "traverse_tree",
+ "unset_executable_mode",
+ "working_dir",
+ "keep_modification_time",
]
def getuid():
if is_windows:
import ctypes
+
if ctypes.windll.shell32.IsUserAnAdmin() == 0:
return 1
return 0
@@ -111,7 +112,7 @@ def path_contains_subdirectory(path, root):
#: This generates the library filenames that may appear on any OS.
-library_extensions = ['a', 'la', 'so', 'tbd', 'dylib']
+library_extensions = ["a", "la", "so", "tbd", "dylib"]
def possible_library_filenames(library_names):
@@ -120,8 +121,9 @@ def possible_library_filenames(library_names):
"""
lib_extensions = library_extensions
return set(
- '.'.join((lib, extension)) for lib, extension in
- itertools.product(library_names, lib_extensions))
+ ".".join((lib, extension))
+ for lib, extension in itertools.product(library_names, lib_extensions)
+ )
def paths_containing_libs(paths, library_names):
@@ -174,19 +176,21 @@ def filter_file(regex, repl, *filenames, **kwargs):
file is copied verbatim. Default is to filter until the end of the
file.
"""
- string = kwargs.get('string', False)
- backup = kwargs.get('backup', False)
- ignore_absent = kwargs.get('ignore_absent', False)
- stop_at = kwargs.get('stop_at', None)
+ string = kwargs.get("string", False)
+ backup = kwargs.get("backup", False)
+ ignore_absent = kwargs.get("ignore_absent", False)
+ stop_at = kwargs.get("stop_at", None)
# Allow strings to use \1, \2, etc. for replacement, like sed
if not callable(repl):
- unescaped = repl.replace(r'\\', '\\')
+ unescaped = repl.replace(r"\\", "\\")
def replace_groups_with_groupid(m):
def groupid_to_group(x):
return m.group(int(x.group(1)))
- return re.sub(r'\\([1-9])', groupid_to_group, unescaped)
+
+ return re.sub(r"\\([1-9])", groupid_to_group, unescaped)
+
repl = replace_groups_with_groupid
if string:
@@ -217,16 +221,16 @@ def filter_file(regex, repl, *filenames, **kwargs):
try:
extra_kwargs = {}
if sys.version_info > (3, 0):
- extra_kwargs = {'errors': 'surrogateescape'}
+ extra_kwargs = {"errors": "surrogateescape"}
# Open as a text file and filter until the end of the file is
# reached or we found a marker in the line if it was specified
- with open(tmp_filename, mode='r', **extra_kwargs) as input_file:
- with open(filename, mode='w', **extra_kwargs) as output_file:
+ with open(tmp_filename, mode="r", **extra_kwargs) as input_file:
+ with open(filename, mode="w", **extra_kwargs) as output_file:
# Using iter and readline is a workaround needed not to
# disable input_file.tell(), which will happen if we call
# input_file.next() implicitly via the for loop
- for line in iter(input_file.readline, ''):
+ for line in iter(input_file.readline, ""):
if stop_at is not None:
current_position = input_file.tell()
if stop_at == line.strip():
@@ -240,9 +244,9 @@ def filter_file(regex, repl, *filenames, **kwargs):
# If we stopped filtering at some point, reopen the file in
# binary mode and copy verbatim the remaining part
if current_position and stop_at:
- with open(tmp_filename, mode='rb') as input_file:
+ with open(tmp_filename, mode="rb") as input_file:
input_file.seek(current_position)
- with open(filename, mode='ab') as output_file:
+ with open(filename, mode="ab") as output_file:
output_file.writelines(input_file.readlines())
except BaseException:
@@ -281,26 +285,26 @@ def change_sed_delimiter(old_delim, new_delim, *filenames):
new_delim (str): The delimiter to replace with
*filenames: One or more files to search and replace
"""
- assert(len(old_delim) == 1)
- assert(len(new_delim) == 1)
+ assert len(old_delim) == 1
+ assert len(new_delim) == 1
# TODO: handle these cases one day?
- assert(old_delim != '"')
- assert(old_delim != "'")
- assert(new_delim != '"')
- assert(new_delim != "'")
+ assert old_delim != '"'
+ assert old_delim != "'"
+ assert new_delim != '"'
+ assert new_delim != "'"
whole_lines = "^s@([^@]*)@(.*)@[gIp]$"
- whole_lines = whole_lines.replace('@', old_delim)
+ whole_lines = whole_lines.replace("@", old_delim)
single_quoted = r"'s@((?:\\'|[^@'])*)@((?:\\'|[^'])*)@[gIp]?'"
- single_quoted = single_quoted.replace('@', old_delim)
+ single_quoted = single_quoted.replace("@", old_delim)
double_quoted = r'"s@((?:\\"|[^@"])*)@((?:\\"|[^"])*)@[gIp]?"'
- double_quoted = double_quoted.replace('@', old_delim)
+ double_quoted = double_quoted.replace("@", old_delim)
- repl = r's@\1@\2@g'
- repl = repl.replace('@', new_delim)
+ repl = r"s@\1@\2@g"
+ repl = repl.replace("@", new_delim)
filenames = path_to_os_path(*filenames)
for f in filenames:
filter_file(whole_lines, repl, f)
@@ -324,8 +328,7 @@ def exploding_archive_catch(stage):
# Expand all tarballs in their own directory to contain
# exploding tarballs.
- tarball_container = os.path.join(stage.path,
- "spack-expanded-archive")
+ tarball_container = os.path.join(stage.path, "spack-expanded-archive")
mkdirp(tarball_container)
orig_dir = os.getcwd()
os.chdir(tarball_container)
@@ -349,7 +352,7 @@ def exploding_archive_handler(tarball_container, stage):
where archive is being expanded
"""
files = os.listdir(tarball_container)
- non_hidden = [f for f in files if not f.startswith('.')]
+ non_hidden = [f for f in files if not f.startswith(".")]
if len(non_hidden) == 1:
src = os.path.join(tarball_container, non_hidden[0])
if os.path.isdir(src):
@@ -377,11 +380,13 @@ def get_owner_uid(path, err_msg=None):
p_stat = os.stat(path)
if p_stat.st_mode & stat.S_IRWXU != stat.S_IRWXU:
- tty.error("Expected {0} to support mode {1}, but it is {2}"
- .format(path, stat.S_IRWXU, p_stat.st_mode))
+ tty.error(
+ "Expected {0} to support mode {1}, but it is {2}".format(
+ path, stat.S_IRWXU, p_stat.st_mode
+ )
+ )
- raise OSError(errno.EACCES,
- err_msg.format(path, path) if err_msg else "")
+ raise OSError(errno.EACCES, err_msg.format(path, path) if err_msg else "")
else:
p_stat = os.stat(path)
@@ -389,8 +394,8 @@ def get_owner_uid(path, err_msg=None):
owner_uid = p_stat.st_uid
else:
sid = win32security.GetFileSecurity(
- path, win32security.OWNER_SECURITY_INFORMATION) \
- .GetSecurityDescriptorOwner()
+ path, win32security.OWNER_SECURITY_INFORMATION
+ ).GetSecurityDescriptorOwner()
owner_uid = win32security.LookupAccountSid(None, sid)[0]
return owner_uid
@@ -460,8 +465,7 @@ def chmod_x(entry, perms):
@system_path_filter
def copy_mode(src, dest):
- """Set the mode of dest to that of src unless it is a link.
- """
+ """Set the mode of dest to that of src unless it is a link."""
if os.path.islink(dest):
return
src_mode = os.stat(src).st_mode
@@ -504,17 +508,17 @@ def copy(src, dest, _permissions=False):
not a directory
"""
if _permissions:
- tty.debug('Installing {0} to {1}'.format(src, dest))
+ tty.debug("Installing {0} to {1}".format(src, dest))
else:
- tty.debug('Copying {0} to {1}'.format(src, dest))
+ tty.debug("Copying {0} to {1}".format(src, dest))
files = glob.glob(src)
if not files:
raise IOError("No such file or directory: '{0}'".format(src))
if len(files) > 1 and not os.path.isdir(dest):
raise ValueError(
- "'{0}' matches multiple files but '{1}' is not a directory".format(
- src, dest))
+ "'{0}' matches multiple files but '{1}' is not a directory".format(src, dest)
+ )
for src in files:
# Expand dest to its eventual full path if it is a directory.
@@ -592,9 +596,9 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
ValueError: if *src* is a parent directory of *dest*
"""
if _permissions:
- tty.debug('Installing {0} to {1}'.format(src, dest))
+ tty.debug("Installing {0} to {1}".format(src, dest))
else:
- tty.debug('Copying {0} to {1}'.format(src, dest))
+ tty.debug("Copying {0} to {1}".format(src, dest))
abs_dest = os.path.abspath(dest)
if not abs_dest.endswith(os.path.sep):
@@ -612,15 +616,20 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
# Stop early to avoid unnecessary recursion if being asked to copy
# from a parent directory.
if abs_dest.startswith(abs_src):
- raise ValueError('Cannot copy ancestor directory {0} into {1}'.
- format(abs_src, abs_dest))
+ raise ValueError(
+ "Cannot copy ancestor directory {0} into {1}".format(abs_src, abs_dest)
+ )
mkdirp(abs_dest)
- for s, d in traverse_tree(abs_src, abs_dest, order='pre',
- follow_symlinks=not symlinks,
- ignore=ignore,
- follow_nonexisting=True):
+ for s, d in traverse_tree(
+ abs_src,
+ abs_dest,
+ order="pre",
+ follow_symlinks=not symlinks,
+ ignore=ignore,
+ follow_nonexisting=True,
+ ):
if os.path.islink(s):
link_target = resolve_link_target_relative_to_the_link(s)
if symlinks:
@@ -628,8 +637,7 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
if os.path.isabs(target):
new_target = re.sub(abs_src, abs_dest, target)
if new_target != target:
- tty.debug("Redirecting link {0} to {1}"
- .format(target, new_target))
+ tty.debug("Redirecting link {0} to {1}".format(target, new_target))
target = new_target
symlink(target, d)
@@ -679,10 +687,9 @@ def get_filetype(path_name):
"""
Return the output of file path_name as a string to identify file type.
"""
- file = Executable('file')
- file.add_default_env('LC_ALL', 'C')
- output = file('-b', '-h', '%s' % path_name,
- output=str, error=str)
+ file = Executable("file")
+ file.add_default_env("LC_ALL", "C")
+ output = file("-b", "-h", "%s" % path_name, output=str, error=str)
return output.strip()
@@ -703,8 +710,8 @@ def is_nonsymlink_exe_with_shebang(path):
return False
# Should start with a shebang
- with open(path, 'rb') as f:
- return f.read(2) == b'#!'
+ with open(path, "rb") as f:
+ return f.read(2) == b"#!"
except (IOError, OSError):
return False
@@ -736,16 +743,16 @@ def mkdirp(*paths, **kwargs):
intermediate get the same permissions specified in the arguments to
mkdirp -- default value is 'args'
"""
- mode = kwargs.get('mode', None)
- group = kwargs.get('group', None)
- default_perms = kwargs.get('default_perms', 'args')
+ mode = kwargs.get("mode", None)
+ group = kwargs.get("group", None)
+ default_perms = kwargs.get("default_perms", "args")
paths = path_to_os_path(*paths)
for path in paths:
if not os.path.exists(path):
try:
# detect missing intermediate folders
intermediate_folders = []
- last_parent = ''
+ last_parent = ""
intermediate_path = os.path.dirname(path)
@@ -772,10 +779,10 @@ def mkdirp(*paths, **kwargs):
# ones and if mode_intermediate has been specified, otherwise
# intermediate folders list is not populated at all and default
# OS mode will be used
- if default_perms == 'args':
+ if default_perms == "args":
intermediate_mode = mode
intermediate_group = group
- elif default_perms == 'parents':
+ elif default_perms == "parents":
stat_info = os.stat(last_parent)
intermediate_mode = stat_info.st_mode
intermediate_group = stat_info.st_gid
@@ -788,10 +795,8 @@ def mkdirp(*paths, **kwargs):
if intermediate_mode is not None:
os.chmod(intermediate_path, intermediate_mode)
if intermediate_group is not None:
- chgrp_if_not_world_writable(intermediate_path,
- intermediate_group)
- os.chmod(intermediate_path,
- intermediate_mode) # reset sticky bit after
+ chgrp_if_not_world_writable(intermediate_path, intermediate_group)
+ os.chmod(intermediate_path, intermediate_mode) # reset sticky bit after
except OSError as e:
if e.errno != errno.EEXIST or not os.path.isdir(path):
@@ -803,7 +808,7 @@ def mkdirp(*paths, **kwargs):
@system_path_filter
def force_remove(*paths):
"""Remove files without printing errors. Like ``rm -f``, does NOT
- remove directories."""
+ remove directories."""
for path in paths:
try:
os.remove(path)
@@ -814,7 +819,7 @@ def force_remove(*paths):
@contextmanager
@system_path_filter
def working_dir(dirname, **kwargs):
- if kwargs.get('create', False):
+ if kwargs.get("create", False):
mkdirp(dirname)
orig_dir = os.getcwd()
@@ -847,19 +852,17 @@ def replace_directory_transaction(directory_name):
# Check the input is indeed a directory with absolute path.
# Raise before anything is done to avoid moving the wrong directory
directory_name = os.path.abspath(directory_name)
- assert os.path.isdir(directory_name), 'Not a directory: ' + directory_name
+ assert os.path.isdir(directory_name), "Not a directory: " + directory_name
# Note: directory_name is normalized here, meaning the trailing slash is dropped,
# so dirname is the directory's parent not the directory itself.
- tmpdir = tempfile.mkdtemp(
- dir=os.path.dirname(directory_name),
- prefix='.backup')
+ tmpdir = tempfile.mkdtemp(dir=os.path.dirname(directory_name), prefix=".backup")
# We have to jump through hoops to support Windows, since
# os.rename(directory_name, tmpdir) errors there.
- backup_dir = os.path.join(tmpdir, 'backup')
+ backup_dir = os.path.join(tmpdir, "backup")
os.rename(directory_name, backup_dir)
- tty.debug('Directory moved [src={0}, dest={1}]'.format(directory_name, backup_dir))
+ tty.debug("Directory moved [src={0}, dest={1}]".format(directory_name, backup_dir))
try:
yield backup_dir
@@ -874,12 +877,12 @@ def replace_directory_transaction(directory_name):
except Exception as outer_exception:
raise CouldNotRestoreDirectoryBackup(inner_exception, outer_exception)
- tty.debug('Directory recovered [{0}]'.format(directory_name))
+ tty.debug("Directory recovered [{0}]".format(directory_name))
raise
else:
# Otherwise delete the temporary directory
shutil.rmtree(tmpdir, ignore_errors=True)
- tty.debug('Temporary directory deleted [{0}]'.format(tmpdir))
+ tty.debug("Temporary directory deleted [{0}]".format(tmpdir))
@system_path_filter
@@ -904,7 +907,7 @@ def hash_directory(directory, ignore=[]):
# TODO: if caching big files becomes an issue, convert this to
# TODO: read in chunks. Currently it's used only for testing
# TODO: purposes.
- with open(filename, 'rb') as f:
+ with open(filename, "rb") as f:
md5_hash.update(f.read())
return md5_hash.hexdigest()
@@ -916,15 +919,15 @@ def write_tmp_and_move(filename):
"""Write to a temporary file, then move into place."""
dirname = os.path.dirname(filename)
basename = os.path.basename(filename)
- tmp = os.path.join(dirname, '.%s.tmp' % basename)
- with open(tmp, 'w') as f:
+ tmp = os.path.join(dirname, ".%s.tmp" % basename)
+ with open(tmp, "w") as f:
yield f
shutil.move(tmp, filename)
@contextmanager
@system_path_filter
-def open_if_filename(str_or_file, mode='r'):
+def open_if_filename(str_or_file, mode="r"):
"""Takes either a path or a file object, and opens it if it is a path.
If it's a file object, just yields the file object.
@@ -940,9 +943,9 @@ def open_if_filename(str_or_file, mode='r'):
def touch(path):
"""Creates an empty file at the specified path."""
if is_windows:
- perms = (os.O_WRONLY | os.O_CREAT)
+ perms = os.O_WRONLY | os.O_CREAT
else:
- perms = (os.O_WRONLY | os.O_CREAT | os.O_NONBLOCK | os.O_NOCTTY)
+ perms = os.O_WRONLY | os.O_CREAT | os.O_NONBLOCK | os.O_NOCTTY
fd = None
try:
fd = os.open(path, perms)
@@ -954,8 +957,7 @@ def touch(path):
@system_path_filter
def touchp(path):
- """Like ``touch``, but creates any parent directories needed for the file.
- """
+ """Like ``touch``, but creates any parent directories needed for the file."""
mkdirp(os.path.dirname(path))
touch(path)
@@ -990,8 +992,7 @@ def ancestor(dir, n=1):
def get_single_file(directory):
fnames = os.listdir(directory)
if len(fnames) != 1:
- raise ValueError("Expected exactly 1 file, got {0}"
- .format(str(len(fnames))))
+ raise ValueError("Expected exactly 1 file, got {0}".format(str(len(fnames))))
return fnames[0]
@@ -1025,7 +1026,7 @@ def can_access(file_name):
@system_path_filter
-def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
+def traverse_tree(source_root, dest_root, rel_path="", **kwargs):
"""Traverse two filesystem trees simultaneously.
Walks the LinkTree directory in pre or post order. Yields each
@@ -1057,16 +1058,16 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
``src`` that do not exit in ``dest``. Default is True
follow_links (bool): Whether to descend into symlinks in ``src``
"""
- follow_nonexisting = kwargs.get('follow_nonexisting', True)
- follow_links = kwargs.get('follow_link', False)
+ follow_nonexisting = kwargs.get("follow_nonexisting", True)
+ follow_links = kwargs.get("follow_link", False)
# Yield in pre or post order?
- order = kwargs.get('order', 'pre')
- if order not in ('pre', 'post'):
+ order = kwargs.get("order", "pre")
+ if order not in ("pre", "post"):
raise ValueError("Order must be 'pre' or 'post'.")
# List of relative paths to ignore under the src root.
- ignore = kwargs.get('ignore', None) or (lambda filename: False)
+ ignore = kwargs.get("ignore", None) or (lambda filename: False)
# Don't descend into ignored directories
if ignore(rel_path):
@@ -1076,7 +1077,7 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
dest_path = os.path.join(dest_root, rel_path)
# preorder yields directories before children
- if order == 'pre':
+ if order == "pre":
yield (source_path, dest_path)
for f in os.listdir(source_path):
@@ -1088,14 +1089,12 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
# TODO: for symlinks, os.path.isdir looks for the link target. If the
# target is relative to the link, then that may not resolve properly
# relative to our cwd - see resolve_link_target_relative_to_the_link
- if os.path.isdir(source_child) and (
- follow_links or not os.path.islink(source_child)):
+ if os.path.isdir(source_child) and (follow_links or not os.path.islink(source_child)):
# When follow_nonexisting isn't set, don't descend into dirs
# in source that do not exist in dest
if follow_nonexisting or os.path.exists(dest_child):
- tuples = traverse_tree(
- source_root, dest_root, rel_child, **kwargs)
+ tuples = traverse_tree(source_root, dest_root, rel_child, **kwargs)
for t in tuples:
yield t
@@ -1103,7 +1102,7 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
elif not ignore(os.path.join(rel_path, f)):
yield (source_child, dest_child)
- if order == 'post':
+ if order == "post":
yield (source_path, dest_path)
@@ -1134,7 +1133,7 @@ def lexists_islink_isdir(path):
return True, is_link, is_dir
-def visit_directory_tree(root, visitor, rel_path='', depth=0):
+def visit_directory_tree(root, visitor, rel_path="", depth=0):
"""
Recurses the directory root depth-first through a visitor pattern
@@ -1172,8 +1171,7 @@ def visit_directory_tree(root, visitor, rel_path='', depth=0):
try:
isdir = f.is_dir()
except OSError as e:
- if is_windows and hasattr(e, 'winerror')\
- and e.winerror == 5 and islink:
+ if is_windows and hasattr(e, "winerror") and e.winerror == 5 and islink:
# if path is a symlink, determine destination and
# evaluate file vs directory
link_target = resolve_link_target_relative_to_the_link(f)
@@ -1221,9 +1219,11 @@ def set_executable(path):
def last_modification_time_recursive(path):
path = os.path.abspath(path)
times = [os.stat(path).st_mtime]
- times.extend(os.stat(os.path.join(root, name)).st_mtime
- for root, dirs, files in os.walk(path)
- for name in dirs + files)
+ times.extend(
+ os.stat(os.path.join(root, name)).st_mtime
+ for root, dirs, files in os.walk(path)
+ for name in dirs + files
+ )
return max(times)
@@ -1282,18 +1282,23 @@ def readonly_file_handler(ignore_errors=False):
and will raise a separate error if it is ever invoked (by accident)
on a non-Windows system.
"""
+
def error_remove_readonly(func, path, exc):
if not is_windows:
raise RuntimeError("This method should only be invoked on Windows")
excvalue = exc[1]
- if is_windows and func in (os.rmdir, os.remove, os.unlink) and\
- excvalue.errno == errno.EACCES:
+ if (
+ is_windows
+ and func in (os.rmdir, os.remove, os.unlink)
+ and excvalue.errno == errno.EACCES
+ ):
# change the file to be readable,writable,executable: 0777
os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
# retry
func(path)
elif not ignore_errors:
raise
+
return error_remove_readonly
@@ -1309,13 +1314,13 @@ def remove_linked_tree(path):
Parameters:
path (str): Directory to be removed
"""
- kwargs = {'ignore_errors': True}
+ kwargs = {"ignore_errors": True}
# Windows readonly files cannot be removed by Python
# directly.
if is_windows:
- kwargs['ignore_errors'] = False
- kwargs['onerror'] = readonly_file_handler(ignore_errors=True)
+ kwargs["ignore_errors"] = False
+ kwargs["onerror"] = readonly_file_handler(ignore_errors=True)
if os.path.exists(path):
if os.path.islink(path):
@@ -1344,9 +1349,7 @@ def safe_remove(*files_or_dirs):
# Sort them so that shorter paths like "/foo/bar" come before
# nested paths like "/foo/bar/baz.yaml". This simplifies the
# handling of temporary copies below
- sorted_matches = sorted([
- os.path.abspath(x) for x in itertools.chain(*glob_matches)
- ], key=len)
+ sorted_matches = sorted([os.path.abspath(x) for x in itertools.chain(*glob_matches)], key=len)
# Copy files and directories in a temporary location
removed, dst_root = {}, tempfile.mkdtemp()
@@ -1361,7 +1364,7 @@ def safe_remove(*files_or_dirs):
continue
# The monotonic ID is a simple way to make the filename
# or directory name unique in the temporary folder
- basename = os.path.basename(file_or_dir) + '-{0}'.format(id)
+ basename = os.path.basename(file_or_dir) + "-{0}".format(id)
temporary_path = os.path.join(dst_root, basename)
shutil.move(file_or_dir, temporary_path)
removed[file_or_dir] = temporary_path
@@ -1390,11 +1393,11 @@ def fix_darwin_install_name(path):
libs = glob.glob(join_path(path, "*.dylib"))
for lib in libs:
# fix install name first:
- install_name_tool = Executable('install_name_tool')
- install_name_tool('-id', lib, lib)
- otool = Executable('otool')
- long_deps = otool('-L', lib, output=str).split('\n')
- deps = [dep.partition(' ')[0][1::] for dep in long_deps[2:-1]]
+ install_name_tool = Executable("install_name_tool")
+ install_name_tool("-id", lib, lib)
+ otool = Executable("otool")
+ long_deps = otool("-L", lib, output=str).split("\n")
+ deps = [dep.partition(" ")[0][1::] for dep in long_deps[2:-1]]
# fix all dependencies:
for dep in deps:
for loc in libs:
@@ -1404,7 +1407,7 @@ def fix_darwin_install_name(path):
# but we don't know builddir (nor how symbolic links look
# in builddir). We thus only compare the basenames.
if os.path.basename(dep) == os.path.basename(loc):
- install_name_tool('-change', dep, loc, lib)
+ install_name_tool("-change", dep, loc, lib)
break
@@ -1534,9 +1537,7 @@ class FileList(Sequence):
Returns:
list: A list of directories
"""
- return list(dedupe(
- os.path.dirname(x) for x in self.files if os.path.dirname(x)
- ))
+ return list(dedupe(os.path.dirname(x) for x in self.files if os.path.dirname(x)))
@property
def basenames(self):
@@ -1572,11 +1573,11 @@ class FileList(Sequence):
def __len__(self):
return len(self.files)
- def joined(self, separator=' '):
+ def joined(self, separator=" "):
return separator.join(self.files)
def __repr__(self):
- return self.__class__.__name__ + '(' + repr(self.files) + ')'
+ return self.__class__.__name__ + "(" + repr(self.files) + ")"
def __str__(self):
return self.joined()
@@ -1593,7 +1594,7 @@ class HeaderList(FileList):
# as "xinclude" will cause false matches.
# Avoid matching paths such as <prefix>/include/something/detail/include,
# e.g. in the CUDA Toolkit which ships internal libc++ headers.
- include_regex = re.compile(r'(.*?)(\binclude\b)(.*)')
+ include_regex = re.compile(r"(.*?)(\binclude\b)(.*)")
def __init__(self, files):
super(HeaderList, self).__init__(files)
@@ -1658,7 +1659,7 @@ class HeaderList(FileList):
name = x
# Valid extensions include: ['.cuh', '.hpp', '.hh', '.h']
- for ext in ['.cuh', '.hpp', '.hh', '.h']:
+ for ext in [".cuh", ".hpp", ".hh", ".h"]:
i = name.rfind(ext)
if i != -1:
names.append(name[:i])
@@ -1680,7 +1681,7 @@ class HeaderList(FileList):
Returns:
str: A joined list of include flags
"""
- return ' '.join(['-I' + x for x in self.directories])
+ return " ".join(["-I" + x for x in self.directories])
@property
def macro_definitions(self):
@@ -1695,7 +1696,7 @@ class HeaderList(FileList):
Returns:
str: A joined list of macro definitions
"""
- return ' '.join(self._macro_definitions)
+ return " ".join(self._macro_definitions)
@property
def cpp_flags(self):
@@ -1713,7 +1714,7 @@ class HeaderList(FileList):
"""
cpp_flags = self.include_flags
if self.macro_definitions:
- cpp_flags += ' ' + self.macro_definitions
+ cpp_flags += " " + self.macro_definitions
return cpp_flags
def add_macro(self, macro):
@@ -1752,24 +1753,30 @@ def find_headers(headers, root, recursive=False):
if isinstance(headers, six.string_types):
headers = [headers]
elif not isinstance(headers, Sequence):
- message = '{0} expects a string or sequence of strings as the '
- message += 'first argument [got {1} instead]'
+ message = "{0} expects a string or sequence of strings as the "
+ message += "first argument [got {1} instead]"
message = message.format(find_headers.__name__, type(headers))
raise TypeError(message)
# Construct the right suffix for the headers
suffixes = [
# C
- 'h',
+ "h",
# C++
- 'hpp', 'hxx', 'hh', 'H', 'txx', 'tcc', 'icc',
+ "hpp",
+ "hxx",
+ "hh",
+ "H",
+ "txx",
+ "tcc",
+ "icc",
# Fortran
- 'mod', 'inc',
+ "mod",
+ "inc",
]
# List of headers we are searching with suffixes
- headers = ['{0}.{1}'.format(header, suffix) for header in headers
- for suffix in suffixes]
+ headers = ["{0}.{1}".format(header, suffix) for header in headers for suffix in suffixes]
return HeaderList(find(root, headers, recursive))
@@ -1785,7 +1792,7 @@ def find_all_headers(root):
Returns:
List of all headers found in ``root`` and subdirectories.
"""
- return find_headers('*', root=root, recursive=True)
+ return find_headers("*", root=root, recursive=True)
class LibraryList(FileList):
@@ -1819,11 +1826,11 @@ class LibraryList(FileList):
for x in self.basenames:
name = x
- if x.startswith('lib'):
+ if x.startswith("lib"):
name = x[3:]
# Valid extensions include: ['.dylib', '.so', '.a']
- for ext in ['.dylib', '.so', '.a']:
+ for ext in [".dylib", ".so", ".a"]:
i = name.rfind(ext)
if i != -1:
names.append(name[:i])
@@ -1845,7 +1852,7 @@ class LibraryList(FileList):
Returns:
str: A joined list of search flags
"""
- return ' '.join(['-L' + x for x in self.directories])
+ return " ".join(["-L" + x for x in self.directories])
@property
def link_flags(self):
@@ -1858,7 +1865,7 @@ class LibraryList(FileList):
Returns:
str: A joined list of link flags
"""
- return ' '.join(['-l' + name for name in self.names])
+ return " ".join(["-l" + name for name in self.names])
@property
def ld_flags(self):
@@ -1871,7 +1878,7 @@ class LibraryList(FileList):
Returns:
str: A joined list of search flags and link flags
"""
- return self.search_flags + ' ' + self.link_flags
+ return self.search_flags + " " + self.link_flags
def find_system_libraries(libraries, shared=True):
@@ -1908,20 +1915,19 @@ def find_system_libraries(libraries, shared=True):
if isinstance(libraries, six.string_types):
libraries = [libraries]
elif not isinstance(libraries, Sequence):
- message = '{0} expects a string or sequence of strings as the '
- message += 'first argument [got {1} instead]'
- message = message.format(find_system_libraries.__name__,
- type(libraries))
+ message = "{0} expects a string or sequence of strings as the "
+ message += "first argument [got {1} instead]"
+ message = message.format(find_system_libraries.__name__, type(libraries))
raise TypeError(message)
libraries_found = []
search_locations = [
- '/lib64',
- '/lib',
- '/usr/lib64',
- '/usr/lib',
- '/usr/local/lib64',
- '/usr/local/lib',
+ "/lib64",
+ "/lib",
+ "/usr/lib64",
+ "/usr/lib",
+ "/usr/local/lib64",
+ "/usr/local/lib",
]
for library in libraries:
@@ -1962,24 +1968,23 @@ def find_libraries(libraries, root, shared=True, recursive=False):
if isinstance(libraries, six.string_types):
libraries = [libraries]
elif not isinstance(libraries, Sequence):
- message = '{0} expects a string or sequence of strings as the '
- message += 'first argument [got {1} instead]'
+ message = "{0} expects a string or sequence of strings as the "
+ message += "first argument [got {1} instead]"
message = message.format(find_libraries.__name__, type(libraries))
raise TypeError(message)
# Construct the right suffix for the library
if shared:
# Used on both Linux and macOS
- suffixes = ['so']
- if sys.platform == 'darwin':
+ suffixes = ["so"]
+ if sys.platform == "darwin":
# Only used on macOS
- suffixes.append('dylib')
+ suffixes.append("dylib")
else:
- suffixes = ['a']
+ suffixes = ["a"]
# List of libraries we are searching with suffixes
- libraries = ['{0}.{1}'.format(lib, suffix) for lib in libraries
- for suffix in suffixes]
+ libraries = ["{0}.{1}".format(lib, suffix) for lib in libraries for suffix in suffixes]
if not recursive:
# If not recursive, look for the libraries directly in root
@@ -1989,7 +1994,7 @@ def find_libraries(libraries, root, shared=True, recursive=False):
# perform first non-recursive search in root/lib then in root/lib64 and
# finally search all of root recursively. The search stops when the first
# match is found.
- for subdir in ('lib', 'lib64'):
+ for subdir in ("lib", "lib64"):
dirname = join_path(root, subdir)
if not os.path.isdir(dirname):
continue
@@ -2045,10 +2050,11 @@ def files_in(*search_paths):
"""
files = []
for d in filter(can_access_dir, search_paths):
- files.extend(filter(
- lambda x: os.path.isfile(x[1]),
- [(f, os.path.join(d, f)) for f in os.listdir(d)]
- ))
+ files.extend(
+ filter(
+ lambda x: os.path.isfile(x[1]), [(f, os.path.join(d, f)) for f in os.listdir(d)]
+ )
+ )
return files
@@ -2078,7 +2084,7 @@ def search_paths_for_executables(*path_hints):
path = os.path.abspath(path)
executable_paths.append(path)
- bin_dir = os.path.join(path, 'bin')
+ bin_dir = os.path.join(path, "bin")
if os.path.isdir(bin_dir):
executable_paths.append(bin_dir)
@@ -2106,11 +2112,11 @@ def search_paths_for_libraries(*path_hints):
path = os.path.abspath(path)
library_paths.append(path)
- lib_dir = os.path.join(path, 'lib')
+ lib_dir = os.path.join(path, "lib")
if os.path.isdir(lib_dir):
library_paths.append(lib_dir)
- lib64_dir = os.path.join(path, 'lib64')
+ lib64_dir = os.path.join(path, "lib64")
if os.path.isdir(lib64_dir):
library_paths.append(lib64_dir)
@@ -2140,13 +2146,13 @@ def partition_path(path, entry=None):
# Handle drive letters e.g. C:/ on Windows
entries[0] = entries[0] + sep
i = entries.index(entry)
- if '' in entries:
+ if "" in entries:
i -= 1
- return paths[:i], paths[i], paths[i + 1:]
+ return paths[:i], paths[i], paths[i + 1 :]
except ValueError:
pass
- return paths, '', []
+ return paths, "", []
@system_path_filter
@@ -2181,7 +2187,7 @@ def prefixes(path):
elif parts[0].endswith(":"):
# Handle drive letters e.g. C:/ on Windows
parts[0] = parts[0] + sep
- paths = [os.path.join(*parts[:i + 1]) for i in range(len(parts))]
+ paths = [os.path.join(*parts[: i + 1]) for i in range(len(parts))]
try:
paths.remove(sep)
@@ -2189,7 +2195,7 @@ def prefixes(path):
pass
try:
- paths.remove('.')
+ paths.remove(".")
except ValueError:
pass
diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py
index 463310b7a2..314566e97a 100644
--- a/lib/spack/llnl/util/lang.py
+++ b/lib/spack/llnl/util/lang.py
@@ -21,7 +21,7 @@ from six import string_types
from llnl.util.compat import MutableMapping, MutableSequence, zip_longest
# Ignore emacs backups when listing modules
-ignore_modules = [r'^\.#', '~$']
+ignore_modules = [r"^\.#", "~$"]
def index_by(objects, *funcs):
@@ -91,9 +91,9 @@ def index_by(objects, *funcs):
def caller_locals():
"""This will return the locals of the *parent* of the caller.
- This allows a function to insert variables into its caller's
- scope. Yes, this is some black magic, and yes it's useful
- for implementing things like depends_on and provides.
+ This allows a function to insert variables into its caller's
+ scope. Yes, this is some black magic, and yes it's useful
+ for implementing things like depends_on and provides.
"""
# Passing zero here skips line context for speed.
stack = inspect.stack(0)
@@ -105,7 +105,7 @@ def caller_locals():
def get_calling_module_name():
"""Make sure that the caller is a class definition, and return the
- enclosing module's name.
+ enclosing module's name.
"""
# Passing zero here skips line context for speed.
stack = inspect.stack(0)
@@ -115,12 +115,13 @@ def get_calling_module_name():
finally:
del stack
- if '__module__' not in caller_locals:
- raise RuntimeError("Must invoke get_calling_module_name() "
- "from inside a class definition!")
+ if "__module__" not in caller_locals:
+ raise RuntimeError(
+ "Must invoke get_calling_module_name() " "from inside a class definition!"
+ )
- module_name = caller_locals['__module__']
- base_name = module_name.split('.')[-1]
+ module_name = caller_locals["__module__"]
+ base_name = module_name.split(".")[-1]
return base_name
@@ -128,8 +129,8 @@ def attr_required(obj, attr_name):
"""Ensure that a class has a required attribute."""
if not hasattr(obj, attr_name):
raise RequiredAttributeError(
- "No required attribute '%s' in class '%s'"
- % (attr_name, obj.__class__.__name__))
+ "No required attribute '%s' in class '%s'" % (attr_name, obj.__class__.__name__)
+ )
def attr_setdefault(obj, name, value):
@@ -201,33 +202,35 @@ def memoized(func):
# TypeError is raised when indexing into a dict if the key is unhashable.
raise six.raise_from(
UnhashableArguments(
- "args + kwargs '{}' was not hashable for function '{}'"
- .format(key, func.__name__),
+ "args + kwargs '{}' was not hashable for function '{}'".format(
+ key, func.__name__
+ ),
),
- e)
+ e,
+ )
return _memoized_function
def list_modules(directory, **kwargs):
"""Lists all of the modules, excluding ``__init__.py``, in a
- particular directory. Listed packages have no particular
- order."""
- list_directories = kwargs.setdefault('directories', True)
+ particular directory. Listed packages have no particular
+ order."""
+ list_directories = kwargs.setdefault("directories", True)
for name in os.listdir(directory):
- if name == '__init__.py':
+ if name == "__init__.py":
continue
path = os.path.join(directory, name)
if list_directories and os.path.isdir(path):
- init_py = os.path.join(path, '__init__.py')
+ init_py = os.path.join(path, "__init__.py")
if os.path.isfile(init_py):
yield name
- elif name.endswith('.py'):
+ elif name.endswith(".py"):
if not any(re.search(pattern, name) for pattern in ignore_modules):
- yield re.sub('.py$', '', name)
+ yield re.sub(".py$", "", name)
def decorator_with_or_without_args(decorator):
@@ -257,41 +260,34 @@ def decorator_with_or_without_args(decorator):
def key_ordering(cls):
"""Decorates a class with extra methods that implement rich comparison
- operations and ``__hash__``. The decorator assumes that the class
- implements a function called ``_cmp_key()``. The rich comparison
- operations will compare objects using this key, and the ``__hash__``
- function will return the hash of this key.
+ operations and ``__hash__``. The decorator assumes that the class
+ implements a function called ``_cmp_key()``. The rich comparison
+ operations will compare objects using this key, and the ``__hash__``
+ function will return the hash of this key.
- If a class already has ``__eq__``, ``__ne__``, ``__lt__``, ``__le__``,
- ``__gt__``, or ``__ge__`` defined, this decorator will overwrite them.
+ If a class already has ``__eq__``, ``__ne__``, ``__lt__``, ``__le__``,
+ ``__gt__``, or ``__ge__`` defined, this decorator will overwrite them.
- Raises:
- TypeError: If the class does not have a ``_cmp_key`` method
+ Raises:
+ TypeError: If the class does not have a ``_cmp_key`` method
"""
+
def setter(name, value):
value.__name__ = name
setattr(cls, name, value)
- if not has_method(cls, '_cmp_key'):
+ if not has_method(cls, "_cmp_key"):
raise TypeError("'%s' doesn't define _cmp_key()." % cls.__name__)
- setter('__eq__',
- lambda s, o:
- (s is o) or (o is not None and s._cmp_key() == o._cmp_key()))
- setter('__lt__',
- lambda s, o: o is not None and s._cmp_key() < o._cmp_key())
- setter('__le__',
- lambda s, o: o is not None and s._cmp_key() <= o._cmp_key())
+ setter("__eq__", lambda s, o: (s is o) or (o is not None and s._cmp_key() == o._cmp_key()))
+ setter("__lt__", lambda s, o: o is not None and s._cmp_key() < o._cmp_key())
+ setter("__le__", lambda s, o: o is not None and s._cmp_key() <= o._cmp_key())
- setter('__ne__',
- lambda s, o:
- (s is not o) and (o is None or s._cmp_key() != o._cmp_key()))
- setter('__gt__',
- lambda s, o: o is None or s._cmp_key() > o._cmp_key())
- setter('__ge__',
- lambda s, o: o is None or s._cmp_key() >= o._cmp_key())
+ setter("__ne__", lambda s, o: (s is not o) and (o is None or s._cmp_key() != o._cmp_key()))
+ setter("__gt__", lambda s, o: o is None or s._cmp_key() > o._cmp_key())
+ setter("__ge__", lambda s, o: o is None or s._cmp_key() >= o._cmp_key())
- setter('__hash__', lambda self: hash(self._cmp_key()))
+ setter("__hash__", lambda self: hash(self._cmp_key()))
return cls
@@ -458,8 +454,7 @@ def lazy_lexicographic_ordering(cls, set_hash=True):
def le(self, other):
if self is other:
return True
- return (other is not None) and not lazy_lt(other._cmp_iter,
- self._cmp_iter)
+ return (other is not None) and not lazy_lt(other._cmp_iter, self._cmp_iter)
def ge(self, other):
if self is other:
@@ -489,7 +484,7 @@ def lazy_lexicographic_ordering(cls, set_hash=True):
@lazy_lexicographic_ordering
class HashableMap(MutableMapping):
"""This is a hashable, comparable dictionary. Hash is performed on
- a tuple of the values in the dictionary."""
+ a tuple of the values in the dictionary."""
def __init__(self):
self.dict = {}
@@ -527,7 +522,7 @@ class HashableMap(MutableMapping):
def in_function(function_name):
"""True if the caller was called from some function with
- the supplied Name, False otherwise."""
+ the supplied Name, False otherwise."""
stack = inspect.stack()
try:
for elt in stack[2:]:
@@ -540,24 +535,25 @@ def in_function(function_name):
def check_kwargs(kwargs, fun):
"""Helper for making functions with kwargs. Checks whether the kwargs
- are empty after all of them have been popped off. If they're
- not, raises an error describing which kwargs are invalid.
+ are empty after all of them have been popped off. If they're
+ not, raises an error describing which kwargs are invalid.
- Example::
+ Example::
- def foo(self, **kwargs):
- x = kwargs.pop('x', None)
- y = kwargs.pop('y', None)
- z = kwargs.pop('z', None)
- check_kwargs(kwargs, self.foo)
+ def foo(self, **kwargs):
+ x = kwargs.pop('x', None)
+ y = kwargs.pop('y', None)
+ z = kwargs.pop('z', None)
+ check_kwargs(kwargs, self.foo)
- # This raises a TypeError:
- foo(w='bad kwarg')
+ # This raises a TypeError:
+ foo(w='bad kwarg')
"""
if kwargs:
raise TypeError(
"'%s' is an invalid keyword argument for function %s()."
- % (next(iter(kwargs)), fun.__name__))
+ % (next(iter(kwargs)), fun.__name__)
+ )
def match_predicate(*args):
@@ -573,6 +569,7 @@ def match_predicate(*args):
* any regex in a list or tuple of regexes matches.
* any predicate in args matches.
"""
+
def match(string):
for arg in args:
if isinstance(arg, string_types):
@@ -585,9 +582,11 @@ def match_predicate(*args):
if arg(string):
return True
else:
- raise ValueError("args to match_predicate must be regex, "
- "list of regexes, or callable.")
+ raise ValueError(
+ "args to match_predicate must be regex, " "list of regexes, or callable."
+ )
return False
+
return match
@@ -647,7 +646,7 @@ def pretty_date(time, now=None):
day_diff = diff.days
if day_diff < 0:
- return ''
+ return ""
if day_diff == 0:
if second_diff < 10:
@@ -705,43 +704,40 @@ def pretty_string_to_date(date_str, now=None):
now = now or datetime.now()
# datetime formats
- pattern[re.compile(r'^\d{4}$')] = lambda x: datetime.strptime(x, '%Y')
- pattern[re.compile(r'^\d{4}-\d{2}$')] = lambda x: datetime.strptime(
- x, '%Y-%m'
+ pattern[re.compile(r"^\d{4}$")] = lambda x: datetime.strptime(x, "%Y")
+ pattern[re.compile(r"^\d{4}-\d{2}$")] = lambda x: datetime.strptime(x, "%Y-%m")
+ pattern[re.compile(r"^\d{4}-\d{2}-\d{2}$")] = lambda x: datetime.strptime(x, "%Y-%m-%d")
+ pattern[re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}$")] = lambda x: datetime.strptime(
+ x, "%Y-%m-%d %H:%M"
)
- pattern[re.compile(r'^\d{4}-\d{2}-\d{2}$')] = lambda x: datetime.strptime(
- x, '%Y-%m-%d'
+ pattern[re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$")] = lambda x: datetime.strptime(
+ x, "%Y-%m-%d %H:%M:%S"
)
- pattern[re.compile(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}$')] = \
- lambda x: datetime.strptime(x, '%Y-%m-%d %H:%M')
- pattern[re.compile(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$')] = \
- lambda x: datetime.strptime(x, '%Y-%m-%d %H:%M:%S')
- pretty_regex = re.compile(
- r'(a|\d+)\s*(year|month|week|day|hour|minute|second)s?\s*ago')
+ pretty_regex = re.compile(r"(a|\d+)\s*(year|month|week|day|hour|minute|second)s?\s*ago")
def _n_xxx_ago(x):
how_many, time_period = pretty_regex.search(x).groups()
- how_many = 1 if how_many == 'a' else int(how_many)
+ how_many = 1 if how_many == "a" else int(how_many)
# timedelta natively supports time periods up to 'weeks'.
# To apply month or year we convert to 30 and 365 days
- if time_period == 'month':
+ if time_period == "month":
how_many *= 30
- time_period = 'day'
- elif time_period == 'year':
+ time_period = "day"
+ elif time_period == "year":
how_many *= 365
- time_period = 'day'
+ time_period = "day"
- kwargs = {(time_period + 's'): how_many}
+ kwargs = {(time_period + "s"): how_many}
return now - timedelta(**kwargs)
pattern[pretty_regex] = _n_xxx_ago
# yesterday
callback = lambda x: now - timedelta(days=1)
- pattern[re.compile('^yesterday$')] = callback
+ pattern[re.compile("^yesterday$")] = callback
for regexp, parser in pattern.items():
if bool(regexp.match(date_str)):
@@ -752,7 +748,6 @@ def pretty_string_to_date(date_str, now=None):
class RequiredAttributeError(ValueError):
-
def __init__(self, message):
super(RequiredAttributeError, self).__init__(message)
@@ -764,6 +759,7 @@ class ObjectWrapper(object):
This class is modeled after the stackoverflow answer:
* http://stackoverflow.com/a/1445289/771663
"""
+
def __init__(self, wrapped_object):
wrapped_cls = type(wrapped_object)
wrapped_name = wrapped_cls.__name__
@@ -807,7 +803,7 @@ class Singleton(object):
# requested but not yet set. The final 'getattr' line here requires
# 'instance'/'_instance' to be defined or it will enter an infinite
# loop, so protect against that here.
- if name in ['_instance', 'instance']:
+ if name in ["_instance", "instance"]:
raise AttributeError()
return getattr(self.instance, name)
@@ -837,7 +833,7 @@ class LazyReference(object):
self.ref_function = ref_function
def __getattr__(self, name):
- if name == 'ref_function':
+ if name == "ref_function":
raise AttributeError()
return getattr(self.ref_function(), name)
@@ -875,8 +871,8 @@ def load_module_from_file(module_name, module_path):
# This recipe is adapted from https://stackoverflow.com/a/67692/771663
if sys.version_info[0] == 3 and sys.version_info[1] >= 5:
import importlib.util
- spec = importlib.util.spec_from_file_location( # novm
- module_name, module_path)
+
+ spec = importlib.util.spec_from_file_location(module_name, module_path) # novm
module = importlib.util.module_from_spec(spec) # novm
# The module object needs to exist in sys.modules before the
# loader executes the module code.
@@ -893,6 +889,7 @@ def load_module_from_file(module_name, module_path):
raise
elif sys.version_info[0] == 2:
import imp
+
module = imp.load_source(module_name, module_path)
return module
@@ -924,8 +921,10 @@ def uniq(sequence):
def star(func):
"""Unpacks arguments for use with Multiprocessing mapping functions"""
+
def _wrapper(args):
return func(*args)
+
return _wrapper
@@ -934,22 +933,23 @@ class Devnull(object):
See https://stackoverflow.com/a/2929954.
"""
+
def write(self, *_):
pass
def elide_list(line_list, max_num=10):
"""Takes a long list and limits it to a smaller number of elements,
- replacing intervening elements with '...'. For example::
+ replacing intervening elements with '...'. For example::
- elide_list([1,2,3,4,5,6], 4)
+ elide_list([1,2,3,4,5,6], 4)
- gives::
+ gives::
- [1, 2, 3, '...', 6]
+ [1, 2, 3, '...', 6]
"""
if len(line_list) > max_num:
- return line_list[:max_num - 1] + ['...'] + line_list[-1:]
+ return line_list[: max_num - 1] + ["..."] + line_list[-1:]
else:
return line_list
@@ -972,7 +972,7 @@ def enum(**kwargs):
Args:
**kwargs: explicit dictionary of enums
"""
- return type('Enum', (object,), kwargs)
+ return type("Enum", (object,), kwargs)
class TypedMutableSequence(MutableSequence):
@@ -988,6 +988,7 @@ class TypedMutableSequence(MutableSequence):
if isinstance(l, Foo):
# do something
"""
+
def __init__(self, iterable):
self.data = list(iterable)
@@ -1017,7 +1018,7 @@ class GroupedExceptionHandler(object):
"""A generic mechanism to coalesce multiple exceptions and preserve tracebacks."""
def __init__(self):
- self.exceptions = [] # type: List[Tuple[str, Exception, List[str]]]
+ self.exceptions = [] # type: List[Tuple[str, Exception, List[str]]]
def __bool__(self):
"""Whether any exceptions were handled."""
@@ -1036,17 +1037,15 @@ class GroupedExceptionHandler(object):
# type: (bool) -> str
"""Print out an error message coalescing all the forwarded errors."""
each_exception_message = [
- '{0} raised {1}: {2}{3}'.format(
+ "{0} raised {1}: {2}{3}".format(
context,
exc.__class__.__name__,
exc,
- '\n{0}'.format(''.join(tb)) if with_tracebacks else '',
+ "\n{0}".format("".join(tb)) if with_tracebacks else "",
)
for context, exc, tb in self.exceptions
]
- return 'due to the following failures:\n{0}'.format(
- '\n'.join(each_exception_message)
- )
+ return "due to the following failures:\n{0}".format("\n".join(each_exception_message))
class GroupedExceptionForwarder(object):
@@ -1079,6 +1078,7 @@ class classproperty(object):
the evaluation is injected at creation time and take an instance (could be None) and
an owner (i.e. the class that originated the instance)
"""
+
def __init__(self, callback):
self.callback = callback
diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py
index 34cce1247c..947ca9c541 100644
--- a/lib/spack/llnl/util/link_tree.py
+++ b/lib/spack/llnl/util/link_tree.py
@@ -16,9 +16,9 @@ import llnl.util.tty as tty
from llnl.util.filesystem import mkdirp, touch, traverse_tree
from llnl.util.symlink import islink, symlink
-__all__ = ['LinkTree']
+__all__ = ["LinkTree"]
-empty_file_name = '.spack-empty'
+empty_file_name = ".spack-empty"
def remove_link(src, dest):
@@ -38,6 +38,7 @@ class MergeConflict:
project(src_a) == project(src_b) == dst
"""
+
def __init__(self, dst, src_a=None, src_b=None):
self.dst = dst
self.src_a = src_a
@@ -51,13 +52,14 @@ class SourceMergeVisitor(object):
- A list of files to link in dst
- A list of merge conflicts in dst/
"""
+
def __init__(self, ignore=None):
self.ignore = ignore if ignore is not None else lambda f: False
# When mapping <src root> to <dst root>/<projection>, we need
# to prepend the <projection> bit to the relative path in the
# destination dir.
- self.projection = ''
+ self.projection = ""
# When a file blocks another file, the conflict can sometimes
# be resolved / ignored (e.g. <prefix>/LICENSE or
@@ -88,10 +90,13 @@ class SourceMergeVisitor(object):
elif proj_rel_path in self.files:
# Can't create a dir where a file is.
src_a_root, src_a_relpath = self.files[proj_rel_path]
- self.fatal_conflicts.append(MergeConflict(
- dst=proj_rel_path,
- src_a=os.path.join(src_a_root, src_a_relpath),
- src_b=os.path.join(root, rel_path)))
+ self.fatal_conflicts.append(
+ MergeConflict(
+ dst=proj_rel_path,
+ src_a=os.path.join(src_a_root, src_a_relpath),
+ src_b=os.path.join(root, rel_path),
+ )
+ )
return False
elif proj_rel_path in self.directories:
# No new directory, carry on.
@@ -147,17 +152,23 @@ class SourceMergeVisitor(object):
elif proj_rel_path in self.directories:
# Can't create a file where a dir is; fatal error
src_a_root, src_a_relpath = self.directories[proj_rel_path]
- self.fatal_conflicts.append(MergeConflict(
- dst=proj_rel_path,
- src_a=os.path.join(src_a_root, src_a_relpath),
- src_b=os.path.join(root, rel_path)))
+ self.fatal_conflicts.append(
+ MergeConflict(
+ dst=proj_rel_path,
+ src_a=os.path.join(src_a_root, src_a_relpath),
+ src_b=os.path.join(root, rel_path),
+ )
+ )
elif proj_rel_path in self.files:
# In some cases we can resolve file-file conflicts
src_a_root, src_a_relpath = self.files[proj_rel_path]
- self.file_conflicts.append(MergeConflict(
- dst=proj_rel_path,
- src_a=os.path.join(src_a_root, src_a_relpath),
- src_b=os.path.join(root, rel_path)))
+ self.file_conflicts.append(
+ MergeConflict(
+ dst=proj_rel_path,
+ src_a=os.path.join(src_a_root, src_a_relpath),
+ src_b=os.path.join(root, rel_path),
+ )
+ )
else:
# Otherwise register this file to be linked.
self.files[proj_rel_path] = (root, rel_path)
@@ -166,24 +177,27 @@ class SourceMergeVisitor(object):
self.projection = os.path.normpath(projection)
# Todo, is this how to check in general for empty projection?
- if self.projection == '.':
- self.projection = ''
+ if self.projection == ".":
+ self.projection = ""
return
# If there is a projection, we'll also create the directories
# it consists of, and check whether that's causing conflicts.
- path = ''
+ path = ""
for part in self.projection.split(os.sep):
path = os.path.join(path, part)
if path not in self.files:
- self.directories[path] = ('<projection>', path)
+ self.directories[path] = ("<projection>", path)
else:
# Can't create a dir where a file is.
src_a_root, src_a_relpath = self.files[path]
- self.fatal_conflicts.append(MergeConflict(
- dst=path,
- src_a=os.path.join(src_a_root, src_a_relpath),
- src_b=os.path.join('<projection>', path)))
+ self.fatal_conflicts.append(
+ MergeConflict(
+ dst=path,
+ src_a=os.path.join(src_a_root, src_a_relpath),
+ src_b=os.path.join("<projection>", path),
+ )
+ )
class DestinationMergeVisitor(object):
@@ -200,6 +214,7 @@ class DestinationMergeVisitor(object):
in the target prefix will never be merged with
directories in the sources directories.
"""
+
def __init__(self, source_merge_visitor):
self.src = source_merge_visitor
@@ -208,10 +223,11 @@ class DestinationMergeVisitor(object):
# and don't traverse deeper
if rel_path in self.src.files:
src_a_root, src_a_relpath = self.src.files[rel_path]
- self.src.fatal_conflicts.append(MergeConflict(
- rel_path,
- os.path.join(src_a_root, src_a_relpath),
- os.path.join(root, rel_path)))
+ self.src.fatal_conflicts.append(
+ MergeConflict(
+ rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path)
+ )
+ )
return False
# If destination dir was also a src dir, remove the mkdir
@@ -236,17 +252,19 @@ class DestinationMergeVisitor(object):
# Always conflict
if rel_path in self.src.directories:
src_a_root, src_a_relpath = self.src.directories[rel_path]
- self.src.fatal_conflicts.append(MergeConflict(
- rel_path,
- os.path.join(src_a_root, src_a_relpath),
- os.path.join(root, rel_path)))
+ self.src.fatal_conflicts.append(
+ MergeConflict(
+ rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path)
+ )
+ )
if rel_path in self.src.files:
src_a_root, src_a_relpath = self.src.files[rel_path]
- self.src.fatal_conflicts.append(MergeConflict(
- rel_path,
- os.path.join(src_a_root, src_a_relpath),
- os.path.join(root, rel_path)))
+ self.src.fatal_conflicts.append(
+ MergeConflict(
+ rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path)
+ )
+ )
# Never descend into symlinked target dirs.
return False
@@ -258,17 +276,19 @@ class DestinationMergeVisitor(object):
# Can't merge a file if target already exists
if rel_path in self.src.directories:
src_a_root, src_a_relpath = self.src.directories[rel_path]
- self.src.fatal_conflicts.append(MergeConflict(
- rel_path,
- os.path.join(src_a_root, src_a_relpath),
- os.path.join(root, rel_path)))
+ self.src.fatal_conflicts.append(
+ MergeConflict(
+ rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path)
+ )
+ )
elif rel_path in self.src.files:
src_a_root, src_a_relpath = self.src.files[rel_path]
- self.src.fatal_conflicts.append(MergeConflict(
- rel_path,
- os.path.join(src_a_root, src_a_relpath),
- os.path.join(root, rel_path)))
+ self.src.fatal_conflicts.append(
+ MergeConflict(
+ rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path)
+ )
+ )
class LinkTree(object):
@@ -281,30 +301,31 @@ class LinkTree(object):
symlinked to, to prevent the source directory from ever being
modified.
"""
+
def __init__(self, source_root):
if not os.path.exists(source_root):
raise IOError("No such file or directory: '%s'", source_root)
self._root = source_root
- def find_conflict(self, dest_root, ignore=None,
- ignore_file_conflicts=False):
+ def find_conflict(self, dest_root, ignore=None, ignore_file_conflicts=False):
"""Returns the first file in dest that conflicts with src"""
ignore = ignore or (lambda x: False)
conflicts = self.find_dir_conflicts(dest_root, ignore)
if not ignore_file_conflicts:
conflicts.extend(
- dst for src, dst
- in self.get_file_map(dest_root, ignore).items()
- if os.path.exists(dst))
+ dst
+ for src, dst in self.get_file_map(dest_root, ignore).items()
+ if os.path.exists(dst)
+ )
if conflicts:
return conflicts[0]
def find_dir_conflicts(self, dest_root, ignore):
conflicts = []
- kwargs = {'follow_nonexisting': False, 'ignore': ignore}
+ kwargs = {"follow_nonexisting": False, "ignore": ignore}
for src, dest in traverse_tree(self._root, dest_root, **kwargs):
if os.path.isdir(src):
if os.path.exists(dest) and not os.path.isdir(dest):
@@ -315,7 +336,7 @@ class LinkTree(object):
def get_file_map(self, dest_root, ignore):
merge_map = {}
- kwargs = {'follow_nonexisting': True, 'ignore': ignore}
+ kwargs = {"follow_nonexisting": True, "ignore": ignore}
for src, dest in traverse_tree(self._root, dest_root, **kwargs):
if not os.path.isdir(src):
merge_map[src] = dest
@@ -337,8 +358,7 @@ class LinkTree(object):
touch(marker)
def unmerge_directories(self, dest_root, ignore):
- for src, dest in traverse_tree(
- self._root, dest_root, ignore=ignore, order='post'):
+ for src, dest in traverse_tree(self._root, dest_root, ignore=ignore, order="post"):
if os.path.isdir(src):
if not os.path.exists(dest):
continue
@@ -354,8 +374,7 @@ class LinkTree(object):
if os.path.exists(marker):
os.remove(marker)
- def merge(self, dest_root, ignore_conflicts=False, ignore=None,
- link=symlink, relative=False):
+ def merge(self, dest_root, ignore_conflicts=False, ignore=None, link=symlink, relative=False):
"""Link all files in src into dest, creating directories
if necessary.
@@ -377,7 +396,8 @@ class LinkTree(object):
ignore = lambda x: False
conflict = self.find_conflict(
- dest_root, ignore=ignore, ignore_file_conflicts=ignore_conflicts)
+ dest_root, ignore=ignore, ignore_file_conflicts=ignore_conflicts
+ )
if conflict:
raise SingleMergeConflictError(conflict)
@@ -416,8 +436,7 @@ class MergeConflictError(Exception):
class SingleMergeConflictError(MergeConflictError):
def __init__(self, path):
- super(MergeConflictError, self).__init__(
- "Package merge blocked by file: %s" % path)
+ super(MergeConflictError, self).__init__("Package merge blocked by file: %s" % path)
class MergeConflictSummary(MergeConflictError):
@@ -430,5 +449,6 @@ class MergeConflictSummary(MergeConflictError):
# show the first 3 merge conflicts.
for conflict in conflicts[:3]:
msg += "\n `{0}` and `{1}` both project to `{2}`".format(
- conflict.src_a, conflict.src_b, conflict.dst)
+ conflict.src_a, conflict.src_b, conflict.dst
+ )
super(MergeConflictSummary, self).__init__(msg)
diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py
index 1ff7ceec64..0682ce059a 100644
--- a/lib/spack/llnl/util/lock.py
+++ b/lib/spack/llnl/util/lock.py
@@ -15,22 +15,22 @@ import llnl.util.tty as tty
import spack.util.string
-if sys.platform != 'win32':
+if sys.platform != "win32":
import fcntl
__all__ = [
- 'Lock',
- 'LockDowngradeError',
- 'LockUpgradeError',
- 'LockTransaction',
- 'WriteTransaction',
- 'ReadTransaction',
- 'LockError',
- 'LockTimeoutError',
- 'LockPermissionError',
- 'LockROFileError',
- 'CantCreateLockError'
+ "Lock",
+ "LockDowngradeError",
+ "LockUpgradeError",
+ "LockTransaction",
+ "WriteTransaction",
+ "ReadTransaction",
+ "LockError",
+ "LockTimeoutError",
+ "LockPermissionError",
+ "LockROFileError",
+ "CantCreateLockError",
]
@@ -47,6 +47,7 @@ class OpenFile(object):
the file descriptor from the file handle if needed -- or we could make this track
file descriptors as well in the future.
"""
+
def __init__(self, fh):
self.fh = fh
self.refs = 0
@@ -92,11 +93,11 @@ class OpenFileTracker(object):
path (str): path to lock file we want a filehandle for
"""
# Open writable files as 'r+' so we can upgrade to write later
- os_mode, fh_mode = (os.O_RDWR | os.O_CREAT), 'r+'
+ os_mode, fh_mode = (os.O_RDWR | os.O_CREAT), "r+"
pid = os.getpid()
open_file = None # OpenFile object, if there is one
- stat = None # stat result for the lockfile, if it exists
+ stat = None # stat result for the lockfile, if it exists
try:
# see whether we've seen this inode/pid before
@@ -109,7 +110,7 @@ class OpenFileTracker(object):
raise
# path does not exist -- fail if we won't be able to create it
- parent = os.path.dirname(path) or '.'
+ parent = os.path.dirname(path) or "."
if not os.access(parent, os.W_OK):
raise CantCreateLockError(path)
@@ -119,7 +120,7 @@ class OpenFileTracker(object):
# we know path exists but not if it's writable. If it's read-only,
# only open the file for reading (and fail if we're trying to get
# an exclusive (write) lock on it)
- os_mode, fh_mode = os.O_RDONLY, 'r'
+ os_mode, fh_mode = os.O_RDONLY, "r"
fd = os.open(path, os_mode)
fh = os.fdopen(fd, fh_mode)
@@ -162,10 +163,10 @@ file_tracker = OpenFileTracker()
def _attempts_str(wait_time, nattempts):
# Don't print anything if we succeeded on the first try
if nattempts <= 1:
- return ''
+ return ""
- attempts = spack.util.string.plural(nattempts, 'attempt')
- return ' after {0:0.2f}s and {1}'.format(wait_time, attempts)
+ attempts = spack.util.string.plural(nattempts, "attempt")
+ return " after {0:0.2f}s and {1}".format(wait_time, attempts)
class LockType(object):
@@ -188,8 +189,7 @@ class LockType(object):
@staticmethod
def is_valid(op):
- return op == LockType.READ \
- or op == LockType.WRITE
+ return op == LockType.READ or op == LockType.WRITE
class Lock(object):
@@ -207,8 +207,7 @@ class Lock(object):
overlapping byte ranges in the same file).
"""
- def __init__(self, path, start=0, length=0, default_timeout=None,
- debug=False, desc=''):
+ def __init__(self, path, start=0, length=0, default_timeout=None, debug=False, desc=""):
"""Construct a new lock on the file at ``path``.
By default, the lock applies to the whole file. Optionally,
@@ -243,7 +242,7 @@ class Lock(object):
self.debug = debug
# optional debug description
- self.desc = ' ({0})'.format(desc) if desc else ''
+ self.desc = " ({0})".format(desc) if desc else ""
# If the user doesn't set a default timeout, or if they choose
# None, 0, etc. then lock attempts will not time out (unless the
@@ -280,17 +279,17 @@ class Lock(object):
def __repr__(self):
"""Formal representation of the lock."""
- rep = '{0}('.format(self.__class__.__name__)
+ rep = "{0}(".format(self.__class__.__name__)
for attr, value in self.__dict__.items():
- rep += '{0}={1}, '.format(attr, value.__repr__())
- return '{0})'.format(rep.strip(', '))
+ rep += "{0}={1}, ".format(attr, value.__repr__())
+ return "{0})".format(rep.strip(", "))
def __str__(self):
"""Readable string (with key fields) of the lock."""
- location = '{0}[{1}:{2}]'.format(self.path, self._start, self._length)
- timeout = 'timeout={0}'.format(self.default_timeout)
- activity = '#reads={0}, #writes={1}'.format(self._reads, self._writes)
- return '({0}, {1}, {2})'.format(location, timeout, activity)
+ location = "{0}[{1}:{2}]".format(self.path, self._start, self._length)
+ timeout = "timeout={0}".format(self.default_timeout)
+ activity = "#reads={0}, #writes={1}".format(self._reads, self._writes)
+ return "({0}, {1}, {2})".format(location, timeout, activity)
def _lock(self, op, timeout=None):
"""This takes a lock using POSIX locks (``fcntl.lockf``).
@@ -305,7 +304,7 @@ class Lock(object):
assert LockType.is_valid(op)
op_str = LockType.to_str(op)
- self._log_acquiring('{0} LOCK'.format(op_str))
+ self._log_acquiring("{0} LOCK".format(op_str))
timeout = timeout or self.default_timeout
# Create file and parent directories if they don't exist.
@@ -313,14 +312,16 @@ class Lock(object):
self._ensure_parent_directory()
self._file = file_tracker.get_fh(self.path)
- if LockType.to_module(op) == fcntl.LOCK_EX and self._file.mode == 'r':
+ if LockType.to_module(op) == fcntl.LOCK_EX and self._file.mode == "r":
# Attempt to upgrade to write lock w/a read-only file.
# If the file were writable, we'd have opened it 'r+'
raise LockROFileError(self.path)
- self._log_debug("{0} locking [{1}:{2}]: timeout {3} sec"
- .format(op_str.lower(), self._start, self._length,
- timeout))
+ self._log_debug(
+ "{0} locking [{1}:{2}]: timeout {3} sec".format(
+ op_str.lower(), self._start, self._length, timeout
+ )
+ )
poll_intervals = iter(Lock._poll_interval_generator())
start_time = time.time()
@@ -339,8 +340,7 @@ class Lock(object):
total_wait_time = time.time() - start_time
return total_wait_time, num_attempts
- raise LockTimeoutError("Timed out waiting for a {0} lock."
- .format(op_str.lower()))
+ raise LockTimeoutError("Timed out waiting for a {0} lock.".format(op_str.lower()))
def _poll_lock(self, op):
"""Attempt to acquire the lock in a non-blocking manner. Return whether
@@ -349,16 +349,19 @@ class Lock(object):
module_op = LockType.to_module(op)
try:
# Try to get the lock (will raise if not available.)
- fcntl.lockf(self._file, module_op | fcntl.LOCK_NB,
- self._length, self._start, os.SEEK_SET)
+ fcntl.lockf(
+ self._file, module_op | fcntl.LOCK_NB, self._length, self._start, os.SEEK_SET
+ )
# help for debugging distributed locking
if self.debug:
# All locks read the owner PID and host
self._read_log_debug_data()
- self._log_debug('{0} locked {1} [{2}:{3}] (owner={4})'
- .format(LockType.to_str(op), self.path,
- self._start, self._length, self.pid))
+ self._log_debug(
+ "{0} locked {1} [{2}:{3}] (owner={4})".format(
+ LockType.to_str(op), self.path, self._start, self._length, self.pid
+ )
+ )
# Exclusive locks write their PID/host
if module_op == fcntl.LOCK_EX:
@@ -378,14 +381,13 @@ class Lock(object):
# relative paths to lockfiles in the current directory have no parent
if not parent:
- return '.'
+ return "."
try:
os.makedirs(parent)
except OSError as e:
# makedirs can fail when diretory already exists.
- if not (e.errno == errno.EEXIST and os.path.isdir(parent) or
- e.errno == errno.EISDIR):
+ if not (e.errno == errno.EEXIST and os.path.isdir(parent) or e.errno == errno.EISDIR):
raise
return parent
@@ -396,9 +398,9 @@ class Lock(object):
line = self._file.read()
if line:
- pid, host = line.strip().split(',')
- _, _, self.pid = pid.rpartition('=')
- _, _, self.host = host.rpartition('=')
+ pid, host = line.strip().split(",")
+ _, _, self.pid = pid.rpartition("=")
+ _, _, self.host = host.rpartition("=")
self.pid = int(self.pid)
def _write_log_debug_data(self):
@@ -423,8 +425,7 @@ class Lock(object):
be masquerading as write locks, but this removes either.
"""
- fcntl.lockf(self._file, fcntl.LOCK_UN,
- self._length, self._start, os.SEEK_SET)
+ fcntl.lockf(self._file, fcntl.LOCK_UN, self._length, self._start, os.SEEK_SET)
file_tracker.release_fh(self.path)
self._file = None
@@ -449,7 +450,7 @@ class Lock(object):
wait_time, nattempts = self._lock(LockType.READ, timeout=timeout)
self._reads += 1
# Log if acquired, which includes counts when verbose
- self._log_acquired('READ LOCK', wait_time, nattempts)
+ self._log_acquired("READ LOCK", wait_time, nattempts)
return True
else:
# Increment the read count for nested lock tracking
@@ -474,7 +475,7 @@ class Lock(object):
wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout)
self._writes += 1
# Log if acquired, which includes counts when verbose
- self._log_acquired('WRITE LOCK', wait_time, nattempts)
+ self._log_acquired("WRITE LOCK", wait_time, nattempts)
# return True only if we weren't nested in a read lock.
# TODO: we may need to return two values: whether we got
@@ -561,7 +562,7 @@ class Lock(object):
"""
assert self._reads > 0
- locktype = 'READ LOCK'
+ locktype = "READ LOCK"
if self._reads == 1 and self._writes == 0:
self._log_releasing(locktype)
@@ -569,7 +570,7 @@ class Lock(object):
release_fn = release_fn or true_fn
result = release_fn()
- self._unlock() # can raise LockError.
+ self._unlock() # can raise LockError.
self._reads = 0
self._log_released(locktype)
return result
@@ -597,14 +598,14 @@ class Lock(object):
assert self._writes > 0
release_fn = release_fn or true_fn
- locktype = 'WRITE LOCK'
+ locktype = "WRITE LOCK"
if self._writes == 1 and self._reads == 0:
self._log_releasing(locktype)
# we need to call release_fn before releasing the lock
result = release_fn()
- self._unlock() # can raise LockError.
+ self._unlock() # can raise LockError.
self._writes = 0
self._log_released(locktype)
return result
@@ -625,56 +626,55 @@ class Lock(object):
raise LockError("Attempting to cleanup active lock.")
def _get_counts_desc(self):
- return '(reads {0}, writes {1})'.format(self._reads, self._writes) \
- if tty.is_verbose() else ''
+ return (
+ "(reads {0}, writes {1})".format(self._reads, self._writes) if tty.is_verbose() else ""
+ )
def _log_acquired(self, locktype, wait_time, nattempts):
attempts_part = _attempts_str(wait_time, nattempts)
now = datetime.now()
- desc = 'Acquired at %s' % now.strftime("%H:%M:%S.%f")
- self._log_debug(self._status_msg(locktype, '{0}{1}'
- .format(desc, attempts_part)))
+ desc = "Acquired at %s" % now.strftime("%H:%M:%S.%f")
+ self._log_debug(self._status_msg(locktype, "{0}{1}".format(desc, attempts_part)))
def _log_acquiring(self, locktype):
- self._log_debug(self._status_msg(locktype, 'Acquiring'), level=3)
+ self._log_debug(self._status_msg(locktype, "Acquiring"), level=3)
def _log_debug(self, *args, **kwargs):
"""Output lock debug messages."""
- kwargs['level'] = kwargs.get('level', 2)
+ kwargs["level"] = kwargs.get("level", 2)
tty.debug(*args, **kwargs)
def _log_downgraded(self, wait_time, nattempts):
attempts_part = _attempts_str(wait_time, nattempts)
now = datetime.now()
- desc = 'Downgraded at %s' % now.strftime("%H:%M:%S.%f")
- self._log_debug(self._status_msg('READ LOCK', '{0}{1}'
- .format(desc, attempts_part)))
+ desc = "Downgraded at %s" % now.strftime("%H:%M:%S.%f")
+ self._log_debug(self._status_msg("READ LOCK", "{0}{1}".format(desc, attempts_part)))
def _log_downgrading(self):
- self._log_debug(self._status_msg('WRITE LOCK', 'Downgrading'), level=3)
+ self._log_debug(self._status_msg("WRITE LOCK", "Downgrading"), level=3)
def _log_released(self, locktype):
now = datetime.now()
- desc = 'Released at %s' % now.strftime("%H:%M:%S.%f")
+ desc = "Released at %s" % now.strftime("%H:%M:%S.%f")
self._log_debug(self._status_msg(locktype, desc))
def _log_releasing(self, locktype):
- self._log_debug(self._status_msg(locktype, 'Releasing'), level=3)
+ self._log_debug(self._status_msg(locktype, "Releasing"), level=3)
def _log_upgraded(self, wait_time, nattempts):
attempts_part = _attempts_str(wait_time, nattempts)
now = datetime.now()
- desc = 'Upgraded at %s' % now.strftime("%H:%M:%S.%f")
- self._log_debug(self._status_msg('WRITE LOCK', '{0}{1}'.
- format(desc, attempts_part)))
+ desc = "Upgraded at %s" % now.strftime("%H:%M:%S.%f")
+ self._log_debug(self._status_msg("WRITE LOCK", "{0}{1}".format(desc, attempts_part)))
def _log_upgrading(self):
- self._log_debug(self._status_msg('READ LOCK', 'Upgrading'), level=3)
+ self._log_debug(self._status_msg("READ LOCK", "Upgrading"), level=3)
def _status_msg(self, locktype, status):
- status_desc = '[{0}] {1}'.format(status, self._get_counts_desc())
- return '{0}{1.desc}: {1.path}[{1._start}:{1._length}] {2}'.format(
- locktype, self, status_desc)
+ status_desc = "[{0}] {1}".format(status, self._get_counts_desc())
+ return "{0}{1.desc}: {1.path}[{1._start}:{1._length}] {2}".format(
+ locktype, self, status_desc
+ )
class LockTransaction(object):
@@ -715,7 +715,7 @@ class LockTransaction(object):
def __enter__(self):
if self._enter() and self._acquire_fn:
self._as = self._acquire_fn()
- if hasattr(self._as, '__enter__'):
+ if hasattr(self._as, "__enter__"):
return self._as.__enter__()
else:
return self._as
@@ -727,7 +727,7 @@ class LockTransaction(object):
if self._release_fn is not None:
return self._release_fn(type, value, traceback)
- if self._as and hasattr(self._as, '__exit__'):
+ if self._as and hasattr(self._as, "__exit__"):
if self._as.__exit__(type, value, traceback):
suppress = True
@@ -739,6 +739,7 @@ class LockTransaction(object):
class ReadTransaction(LockTransaction):
"""LockTransaction context manager that does a read and releases it."""
+
def _enter(self):
return self._lock.acquire_read(self._timeout)
@@ -748,6 +749,7 @@ class ReadTransaction(LockTransaction):
class WriteTransaction(LockTransaction):
"""LockTransaction context manager that does a write and releases it."""
+
def _enter(self):
return self._lock.acquire_write(self._timeout)
@@ -761,6 +763,7 @@ class LockError(Exception):
class LockDowngradeError(LockError):
"""Raised when unable to downgrade from a write to a read lock."""
+
def __init__(self, path):
msg = "Cannot downgrade lock from write to read on file: %s" % path
super(LockDowngradeError, self).__init__(msg)
@@ -776,6 +779,7 @@ class LockTimeoutError(LockError):
class LockUpgradeError(LockError):
"""Raised when unable to upgrade from a read to a write lock."""
+
def __init__(self, path):
msg = "Cannot upgrade lock from read to write on file: %s" % path
super(LockUpgradeError, self).__init__(msg)
@@ -787,6 +791,7 @@ class LockPermissionError(LockError):
class LockROFileError(LockPermissionError):
"""Tried to take an exclusive lock on a read-only file."""
+
def __init__(self, path):
msg = "Can't take write lock on read-only file: %s" % path
super(LockROFileError, self).__init__(msg)
@@ -794,6 +799,7 @@ class LockROFileError(LockPermissionError):
class CantCreateLockError(LockPermissionError):
"""Attempt to create a lock in an unwritable location."""
+
def __init__(self, path):
msg = "cannot create lock '%s': " % path
msg += "file does not exist and location is not writable"
diff --git a/lib/spack/llnl/util/multiproc.py b/lib/spack/llnl/util/multiproc.py
index 86b9e81bcc..e6a0091191 100644
--- a/lib/spack/llnl/util/multiproc.py
+++ b/lib/spack/llnl/util/multiproc.py
@@ -10,7 +10,7 @@ to pickle functions if they're passed indirectly as parameters.
"""
from multiprocessing import Semaphore, Value
-__all__ = ['Barrier']
+__all__ = ["Barrier"]
class Barrier:
@@ -24,7 +24,7 @@ class Barrier:
def __init__(self, n, timeout=None):
self.n = n
self.to = timeout
- self.count = Value('i', 0)
+ self.count = Value("i", 0)
self.mutex = Semaphore(1)
self.turnstile1 = Semaphore(0)
self.turnstile2 = Semaphore(1)
diff --git a/lib/spack/llnl/util/symlink.py b/lib/spack/llnl/util/symlink.py
index 6c55d74f66..103c5b4c38 100644
--- a/lib/spack/llnl/util/symlink.py
+++ b/lib/spack/llnl/util/symlink.py
@@ -11,7 +11,7 @@ from sys import platform as _platform
from llnl.util import lang
-is_windows = _platform == 'win32'
+is_windows = _platform == "win32"
if is_windows:
from win32file import CreateHardLink
@@ -47,7 +47,7 @@ def _win32_junction(path, link):
# os.symlink will fail if link exists, emulate the behavior here
if exists(link):
- raise OSError(errno.EEXIST, 'File exists: %s -> %s' % (link, path))
+ raise OSError(errno.EEXIST, "File exists: %s -> %s" % (link, path))
if not os.path.isabs(path):
parent = os.path.join(link, os.pardir)
@@ -61,13 +61,14 @@ def _win32_junction(path, link):
def _win32_can_symlink():
tempdir = tempfile.mkdtemp()
- dpath = join(tempdir, 'dpath')
- fpath = join(tempdir, 'fpath.txt')
+ dpath = join(tempdir, "dpath")
+ fpath = join(tempdir, "fpath.txt")
- dlink = join(tempdir, 'dlink')
- flink = join(tempdir, 'flink.txt')
+ dlink = join(tempdir, "dlink")
+ flink = join(tempdir, "flink.txt")
import llnl.util.filesystem as fs
+
fs.touchp(fpath)
try:
@@ -106,7 +107,6 @@ def _win32_is_junction(path):
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
res = GetFileAttributes(path)
- return res != INVALID_FILE_ATTRIBUTES and \
- bool(res & FILE_ATTRIBUTE_REPARSE_POINT)
+ return res != INVALID_FILE_ATTRIBUTES and bool(res & FILE_ATTRIBUTE_REPARSE_POINT)
return False
diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py
index de49f3f77f..ed847298ef 100644
--- a/lib/spack/llnl/util/tty/__init__.py
+++ b/lib/spack/llnl/util/tty/__init__.py
@@ -54,7 +54,7 @@ def is_stacktrace():
def set_debug(level=0):
global _debug
- assert level >= 0, 'Debug level must be a positive value'
+ assert level >= 0, "Debug level must be a positive value"
_debug = level
@@ -110,10 +110,7 @@ def output_filter(filter_fn):
class SuppressOutput:
"""Class for disabling output in a scope using 'with' keyword"""
- def __init__(self,
- msg_enabled=True,
- warn_enabled=True,
- error_enabled=True):
+ def __init__(self, msg_enabled=True, warn_enabled=True, error_enabled=True):
self._msg_enabled_initial = _msg_enabled
self._warn_enabled_initial = _warn_enabled
@@ -164,11 +161,10 @@ def get_timestamp(force=False):
"""Get a string timestamp"""
if _debug or _timestamp or force:
# Note inclusion of the PID is useful for parallel builds.
- pid = ', {0}'.format(os.getpid()) if show_pid() else ''
- return '[{0}{1}] '.format(
- datetime.now().strftime("%Y-%m-%d-%H:%M:%S.%f"), pid)
+ pid = ", {0}".format(os.getpid()) if show_pid() else ""
+ return "[{0}{1}] ".format(datetime.now().strftime("%Y-%m-%d-%H:%M:%S.%f"), pid)
else:
- return ''
+ return ""
def msg(message, *args, **kwargs):
@@ -178,26 +174,14 @@ def msg(message, *args, **kwargs):
if isinstance(message, Exception):
message = "%s: %s" % (message.__class__.__name__, str(message))
- newline = kwargs.get('newline', True)
+ newline = kwargs.get("newline", True)
st_text = ""
if _stacktrace:
st_text = process_stacktrace(2)
if newline:
- cprint(
- "@*b{%s==>} %s%s" % (
- st_text,
- get_timestamp(),
- cescape(_output_filter(message))
- )
- )
+ cprint("@*b{%s==>} %s%s" % (st_text, get_timestamp(), cescape(_output_filter(message))))
else:
- cwrite(
- "@*b{%s==>} %s%s" % (
- st_text,
- get_timestamp(),
- cescape(_output_filter(message))
- )
- )
+ cwrite("@*b{%s==>} %s%s" % (st_text, get_timestamp(), cescape(_output_filter(message))))
for arg in args:
print(indent + _output_filter(six.text_type(arg)))
@@ -206,23 +190,19 @@ def info(message, *args, **kwargs):
if isinstance(message, Exception):
message = "%s: %s" % (message.__class__.__name__, str(message))
- format = kwargs.get('format', '*b')
- stream = kwargs.get('stream', sys.stdout)
- wrap = kwargs.get('wrap', False)
- break_long_words = kwargs.get('break_long_words', False)
- st_countback = kwargs.get('countback', 3)
+ format = kwargs.get("format", "*b")
+ stream = kwargs.get("stream", sys.stdout)
+ wrap = kwargs.get("wrap", False)
+ break_long_words = kwargs.get("break_long_words", False)
+ st_countback = kwargs.get("countback", 3)
st_text = ""
if _stacktrace:
st_text = process_stacktrace(st_countback)
cprint(
- "@%s{%s==>} %s%s" % (
- format,
- st_text,
- get_timestamp(),
- cescape(_output_filter(six.text_type(message)))
- ),
- stream=stream
+ "@%s{%s==>} %s%s"
+ % (format, st_text, get_timestamp(), cescape(_output_filter(six.text_type(message)))),
+ stream=stream,
)
for arg in args:
if wrap:
@@ -230,27 +210,25 @@ def info(message, *args, **kwargs):
_output_filter(six.text_type(arg)),
initial_indent=indent,
subsequent_indent=indent,
- break_long_words=break_long_words
+ break_long_words=break_long_words,
)
for line in lines:
- stream.write(line + '\n')
+ stream.write(line + "\n")
else:
- stream.write(
- indent + _output_filter(six.text_type(arg)) + '\n'
- )
+ stream.write(indent + _output_filter(six.text_type(arg)) + "\n")
def verbose(message, *args, **kwargs):
if _verbose:
- kwargs.setdefault('format', 'c')
+ kwargs.setdefault("format", "c")
info(message, *args, **kwargs)
def debug(message, *args, **kwargs):
- level = kwargs.get('level', 1)
+ level = kwargs.get("level", 1)
if is_debug(level):
- kwargs.setdefault('format', 'g')
- kwargs.setdefault('stream', sys.stderr)
+ kwargs.setdefault("format", "g")
+ kwargs.setdefault("stream", sys.stderr)
info(message, *args, **kwargs)
@@ -258,8 +236,8 @@ def error(message, *args, **kwargs):
if not error_enabled():
return
- kwargs.setdefault('format', '*r')
- kwargs.setdefault('stream', sys.stderr)
+ kwargs.setdefault("format", "*r")
+ kwargs.setdefault("stream", sys.stderr)
info("Error: " + six.text_type(message), *args, **kwargs)
@@ -267,27 +245,27 @@ def warn(message, *args, **kwargs):
if not warn_enabled():
return
- kwargs.setdefault('format', '*Y')
- kwargs.setdefault('stream', sys.stderr)
+ kwargs.setdefault("format", "*Y")
+ kwargs.setdefault("stream", sys.stderr)
info("Warning: " + six.text_type(message), *args, **kwargs)
def die(message, *args, **kwargs):
- kwargs.setdefault('countback', 4)
+ kwargs.setdefault("countback", 4)
error(message, *args, **kwargs)
sys.exit(1)
def get_number(prompt, **kwargs):
- default = kwargs.get('default', None)
- abort = kwargs.get('abort', None)
+ default = kwargs.get("default", None)
+ abort = kwargs.get("abort", None)
if default is not None and abort is not None:
- prompt += ' (default is %s, %s to abort) ' % (default, abort)
+ prompt += " (default is %s, %s to abort) " % (default, abort)
elif default is not None:
- prompt += ' (default is %s) ' % default
+ prompt += " (default is %s) " % default
elif abort is not None:
- prompt += ' (%s to abort) ' % abort
+ prompt += " (%s to abort) " % abort
number = None
while number is None:
@@ -310,17 +288,16 @@ def get_number(prompt, **kwargs):
def get_yes_or_no(prompt, **kwargs):
- default_value = kwargs.get('default', None)
+ default_value = kwargs.get("default", None)
if default_value is None:
- prompt += ' [y/n] '
+ prompt += " [y/n] "
elif default_value is True:
- prompt += ' [Y/n] '
+ prompt += " [Y/n] "
elif default_value is False:
- prompt += ' [y/N] '
+ prompt += " [y/N] "
else:
- raise ValueError(
- "default for get_yes_no() must be True, False, or None.")
+ raise ValueError("default for get_yes_no() must be True, False, or None.")
result = None
while result is None:
@@ -331,9 +308,9 @@ def get_yes_or_no(prompt, **kwargs):
if result is None:
print("Please enter yes or no.")
else:
- if ans == 'y' or ans == 'yes':
+ if ans == "y" or ans == "yes":
result = True
- elif ans == 'n' or ans == 'no':
+ elif ans == "n" or ans == "no":
result = False
return result
@@ -345,12 +322,12 @@ def hline(label=None, **kwargs):
char (str): Char to draw the line with. Default '-'
max_width (int): Maximum width of the line. Default is 64 chars.
"""
- char = kwargs.pop('char', '-')
- max_width = kwargs.pop('max_width', 64)
+ char = kwargs.pop("char", "-")
+ max_width = kwargs.pop("max_width", 64)
if kwargs:
raise TypeError(
- "'%s' is an invalid keyword argument for this function."
- % next(kwargs.iterkeys()))
+ "'%s' is an invalid keyword argument for this function." % next(kwargs.iterkeys())
+ )
rows, cols = terminal_size()
if not cols:
@@ -374,13 +351,14 @@ def hline(label=None, **kwargs):
def terminal_size():
"""Gets the dimensions of the console: (rows, cols)."""
if _platform != "win32":
+
def ioctl_gwinsz(fd):
try:
- rc = struct.unpack('hh', fcntl.ioctl(
- fd, termios.TIOCGWINSZ, '1234'))
+ rc = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
except BaseException:
return
return rc
+
rc = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2)
if not rc:
try:
@@ -390,12 +368,14 @@ def terminal_size():
except BaseException:
pass
if not rc:
- rc = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80))
+ rc = (os.environ.get("LINES", 25), os.environ.get("COLUMNS", 80))
return int(rc[0]), int(rc[1])
else:
if sys.version_info[0] < 3:
- raise RuntimeError("Terminal size not obtainable on Windows with a\
-Python version older than 3")
- rc = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80))
+ raise RuntimeError(
+ "Terminal size not obtainable on Windows with a\
+Python version older than 3"
+ )
+ rc = (os.environ.get("LINES", 25), os.environ.get("COLUMNS", 80))
return int(rc[0]), int(rc[1])
diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py
index 65e56b5473..73c1daf0d5 100644
--- a/lib/spack/llnl/util/tty/colify.py
+++ b/lib/spack/llnl/util/tty/colify.py
@@ -18,29 +18,27 @@ from llnl.util.tty.color import cextra, clen
class ColumnConfig:
-
def __init__(self, cols):
self.cols = cols
self.line_length = 0
self.valid = True
- self.widths = [0] * cols # does not include ansi colors
+ self.widths = [0] * cols # does not include ansi colors
def __repr__(self):
- attrs = [(a, getattr(self, a))
- for a in dir(self) if not a.startswith("__")]
+ attrs = [(a, getattr(self, a)) for a in dir(self) if not a.startswith("__")]
return "<Config: %s>" % ", ".join("%s: %r" % a for a in attrs)
def config_variable_cols(elts, console_width, padding, cols=0):
"""Variable-width column fitting algorithm.
- This function determines the most columns that can fit in the
- screen width. Unlike uniform fitting, where all columns take
- the width of the longest element in the list, each column takes
- the width of its own longest element. This packs elements more
- efficiently on screen.
+ This function determines the most columns that can fit in the
+ screen width. Unlike uniform fitting, where all columns take
+ the width of the longest element in the list, each column takes
+ the width of its own longest element. This packs elements more
+ efficiently on screen.
- If cols is nonzero, force
+ If cols is nonzero, force
"""
if cols < 0:
raise ValueError("cols must be non-negative.")
@@ -64,8 +62,8 @@ def config_variable_cols(elts, console_width, padding, cols=0):
if conf.widths[col] < (length + p):
conf.line_length += length + p - conf.widths[col]
- conf.widths[col] = length + p
- conf.valid = (conf.line_length < console_width)
+ conf.widths[col] = length + p
+ conf.valid = conf.line_length < console_width
try:
config = next(conf for conf in reversed(configs) if conf.valid)
@@ -81,9 +79,9 @@ def config_variable_cols(elts, console_width, padding, cols=0):
def config_uniform_cols(elts, console_width, padding, cols=0):
"""Uniform-width column fitting algorithm.
- Determines the longest element in the list, and determines how
- many columns of that width will fit on screen. Returns a
- corresponding column config.
+ Determines the longest element in the list, and determines how
+ many columns of that width will fit on screen. Returns a
+ corresponding column config.
"""
if cols < 0:
raise ValueError("cols must be non-negative.")
@@ -122,18 +120,18 @@ def colify(elts, **options):
and fit less data on the screen
"""
# Get keyword arguments or set defaults
- cols = options.pop("cols", 0)
- output = options.pop("output", sys.stdout)
- indent = options.pop("indent", 0)
- padding = options.pop("padding", 2)
- tty = options.pop('tty', None)
- method = options.pop("method", "variable")
+ cols = options.pop("cols", 0)
+ output = options.pop("output", sys.stdout)
+ indent = options.pop("indent", 0)
+ padding = options.pop("padding", 2)
+ tty = options.pop("tty", None)
+ method = options.pop("method", "variable")
console_cols = options.pop("width", None)
if options:
raise TypeError(
- "'%s' is an invalid keyword argument for this function."
- % next(options.iterkeys()))
+ "'%s' is an invalid keyword argument for this function." % next(options.iterkeys())
+ )
# elts needs to be an array of strings so we can count the elements
elts = [text_type(elt) for elt in elts]
@@ -141,10 +139,10 @@ def colify(elts, **options):
return (0, ())
# environment size is of the form "<rows>x<cols>"
- env_size = os.environ.get('COLIFY_SIZE')
+ env_size = os.environ.get("COLIFY_SIZE")
if env_size:
try:
- r, c = env_size.split('x')
+ r, c = env_size.split("x")
console_rows, console_cols = int(r), int(c)
tty = True
except BaseException:
@@ -180,7 +178,7 @@ def colify(elts, **options):
elt = col * rows + row
width = config.widths[col] + cextra(elts[elt])
if col < cols - 1:
- fmt = '%%-%ds' % width
+ fmt = "%%-%ds" % width
output.write(fmt % elts[elt])
else:
# Don't pad the rightmost column (sapces can wrap on
@@ -198,15 +196,15 @@ def colify(elts, **options):
def colify_table(table, **options):
"""Version of ``colify()`` for data expressed in rows, (list of lists).
- Same as regular colify but:
+ Same as regular colify but:
- 1. This takes a list of lists, where each sub-list must be the
- same length, and each is interpreted as a row in a table.
- Regular colify displays a sequential list of values in columns.
+ 1. This takes a list of lists, where each sub-list must be the
+ same length, and each is interpreted as a row in a table.
+ Regular colify displays a sequential list of values in columns.
- 2. Regular colify will always print with 1 column when the output
- is not a tty. This will always print with same dimensions of
- the table argument.
+ 2. Regular colify will always print with 1 column when the output
+ is not a tty. This will always print with same dimensions of
+ the table argument.
"""
if table is None:
@@ -221,20 +219,20 @@ def colify_table(table, **options):
for row in table:
yield row[i]
- if 'cols' in options:
+ if "cols" in options:
raise ValueError("Cannot override columsn in colify_table.")
- options['cols'] = columns
+ options["cols"] = columns
# don't reduce to 1 column for non-tty
- options['tty'] = True
+ options["tty"] = True
colify(transpose(), **options)
def colified(elts, **options):
"""Invokes the ``colify()`` function but returns the result as a string
- instead of writing it to an output string."""
+ instead of writing it to an output string."""
sio = StringIO()
- options['output'] = sio
+ options["output"] = sio
colify(elts, **options)
return sio.getvalue()
diff --git a/lib/spack/llnl/util/tty/color.py b/lib/spack/llnl/util/tty/color.py
index 99c0a5c7ac..1bc80331d4 100644
--- a/lib/spack/llnl/util/tty/color.py
+++ b/lib/spack/llnl/util/tty/color.py
@@ -76,29 +76,33 @@ class ColorParseError(Exception):
# Text styles for ansi codes
-styles = {'*': '1', # bold
- '_': '4', # underline
- None: '0'} # plain
+styles = {"*": "1", "_": "4", None: "0"} # bold # underline # plain
# Dim and bright ansi colors
-colors = {'k': 30, 'K': 90, # black
- 'r': 31, 'R': 91, # red
- 'g': 32, 'G': 92, # green
- 'y': 33, 'Y': 93, # yellow
- 'b': 34, 'B': 94, # blue
- 'm': 35, 'M': 95, # magenta
- 'c': 36, 'C': 96, # cyan
- 'w': 37, 'W': 97} # white
+colors = {
+ "k": 30,
+ "K": 90, # black
+ "r": 31,
+ "R": 91, # red
+ "g": 32,
+ "G": 92, # green
+ "y": 33,
+ "Y": 93, # yellow
+ "b": 34,
+ "B": 94, # blue
+ "m": 35,
+ "M": 95, # magenta
+ "c": 36,
+ "C": 96, # cyan
+ "w": 37,
+ "W": 97,
+} # white
# Regex to be used for color formatting
-color_re = r'@(?:@|\.|([*_])?([a-zA-Z])?(?:{((?:[^}]|}})*)})?)'
+color_re = r"@(?:@|\.|([*_])?([a-zA-Z])?(?:{((?:[^}]|}})*)})?)"
# Mapping from color arguments to values for tty.set_color
-color_when_values = {
- 'always': True,
- 'auto': None,
- 'never': False
-}
+color_when_values = {"always": True, "auto": None, "never": False}
# Force color; None: Only color if stdout is a tty
# True: Always colorize output, False: Never colorize output
@@ -114,7 +118,7 @@ def _color_when_value(when):
if when in color_when_values:
return color_when_values[when]
elif when not in color_when_values.values():
- raise ValueError('Invalid color setting: %s' % when)
+ raise ValueError("Invalid color setting: %s" % when)
return when
@@ -146,7 +150,6 @@ def color_when(value):
class match_to_ansi(object):
-
def __init__(self, color=True):
self.color = _color_when_value(color)
@@ -155,7 +158,7 @@ class match_to_ansi(object):
if self.color:
return "\033[%sm" % s
else:
- return ''
+ return ""
def __call__(self, match):
"""Convert a match object generated by ``color_re`` into an ansi
@@ -164,22 +167,22 @@ class match_to_ansi(object):
style, color, text = match.groups()
m = match.group(0)
- if m == '@@':
- return '@'
- elif m == '@.':
+ if m == "@@":
+ return "@"
+ elif m == "@.":
return self.escape(0)
- elif m == '@':
- raise ColorParseError("Incomplete color format: '%s' in %s"
- % (m, match.string))
+ elif m == "@":
+ raise ColorParseError("Incomplete color format: '%s' in %s" % (m, match.string))
string = styles[style]
if color:
if color not in colors:
- raise ColorParseError("Invalid color specifier: '%s' in '%s'"
- % (color, match.string))
- string += ';' + str(colors[color])
+ raise ColorParseError(
+ "Invalid color specifier: '%s' in '%s'" % (color, match.string)
+ )
+ string += ";" + str(colors[color])
- colored_text = ''
+ colored_text = ""
if text:
colored_text = text + self.escape(0)
@@ -199,28 +202,28 @@ def colorize(string, **kwargs):
color (bool): If False, output will be plain text without control
codes, for output to non-console devices.
"""
- color = _color_when_value(kwargs.get('color', get_color_when()))
+ color = _color_when_value(kwargs.get("color", get_color_when()))
string = re.sub(color_re, match_to_ansi(color), string)
- string = string.replace('}}', '}')
+ string = string.replace("}}", "}")
return string
def clen(string):
"""Return the length of a string, excluding ansi color sequences."""
- return len(re.sub(r'\033[^m]*m', '', string))
+ return len(re.sub(r"\033[^m]*m", "", string))
def cextra(string):
"""Length of extra color characters in a string"""
- return len(''.join(re.findall(r'\033[^m]*m', string)))
+ return len("".join(re.findall(r"\033[^m]*m", string)))
def cwrite(string, stream=None, color=None):
"""Replace all color expressions in string with ANSI control
- codes and write the result to the stream. If color is
- False, this will write plain text with no color. If True,
- then it will always write colored output. If not supplied,
- then it will be set based on stream.isatty().
+ codes and write the result to the stream. If color is
+ False, this will write plain text with no color. If True,
+ then it will always write colored output. If not supplied,
+ then it will be set based on stream.isatty().
"""
stream = sys.stdout if stream is None else stream
if color is None:
@@ -251,20 +254,19 @@ def cescape(string):
(str): the string with color codes escaped
"""
string = six.text_type(string)
- string = string.replace('@', '@@')
- string = string.replace('}', '}}')
+ string = string.replace("@", "@@")
+ string = string.replace("}", "}}")
return string
class ColorStream(object):
-
def __init__(self, stream, color=None):
self._stream = stream
self._color = color
def write(self, string, **kwargs):
- raw = kwargs.get('raw', False)
- raw_write = getattr(self._stream, 'write')
+ raw = kwargs.get("raw", False)
+ raw_write = getattr(self._stream, "write")
color = self._color
if self._color is None:
@@ -275,6 +277,6 @@ class ColorStream(object):
raw_write(colorize(string, color=color))
def writelines(self, sequence, **kwargs):
- raw = kwargs.get('raw', False)
+ raw = kwargs.get("raw", False)
for string in sequence:
self.write(string, self.color, raw=raw)
diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py
index 51b9caa332..e155fa1d26 100644
--- a/lib/spack/llnl/util/tty/log.py
+++ b/lib/spack/llnl/util/tty/log.py
@@ -31,21 +31,22 @@ import llnl.util.tty as tty
termios = None # type: Optional[ModuleType]
try:
import termios as term_mod
+
termios = term_mod
except ImportError:
pass
# Use this to strip escape sequences
-_escape = re.compile(r'\x1b[^m]*m|\x1b\[?1034h|\x1b\][0-9]+;[^\x07]*\x07')
+_escape = re.compile(r"\x1b[^m]*m|\x1b\[?1034h|\x1b\][0-9]+;[^\x07]*\x07")
# control characters for enabling/disabling echo
#
# We use control characters to ensure that echo enable/disable are inline
# with the other output. We always follow these with a newline to ensure
# one per line the following newline is ignored in output.
-xon, xoff = '\x11\n', '\x13\n'
-control = re.compile('(\x11\n|\x13\n)')
+xon, xoff = "\x11\n", "\x13\n"
+control = re.compile("(\x11\n|\x13\n)")
@contextmanager
@@ -59,17 +60,13 @@ def ignore_signal(signum):
def _is_background_tty(stream):
- """True if the stream is a tty and calling process is in the background.
- """
- return (
- stream.isatty() and
- os.getpgrp() != os.tcgetpgrp(stream.fileno())
- )
+ """True if the stream is a tty and calling process is in the background."""
+ return stream.isatty() and os.getpgrp() != os.tcgetpgrp(stream.fileno())
def _strip(line):
"""Strip color and control characters from a line."""
- return _escape.sub('', line)
+ return _escape.sub("", line)
class keyboard_input(object):
@@ -147,6 +144,7 @@ class keyboard_input(object):
a TTY, ``keyboard_input`` has no effect.
"""
+
def __init__(self, stream):
"""Create a context manager that will enable keyboard input on stream.
@@ -204,7 +202,7 @@ class keyboard_input(object):
bg = self._is_background()
# restore sanity if flags are amiss -- see diagram in class docs
- if not bg and any(flags): # fg, but input not enabled
+ if not bg and any(flags): # fg, but input not enabled
self._enable_keyboard_input()
elif bg and not all(flags): # bg, but input enabled
self._restore_default_terminal_settings()
@@ -228,8 +226,7 @@ class keyboard_input(object):
# Install a signal handler to disable/enable keyboard input
# when the process moves between foreground and background.
- self.old_handlers[signal.SIGTSTP] = signal.signal(
- signal.SIGTSTP, self._tstp_handler)
+ self.old_handlers[signal.SIGTSTP] = signal.signal(signal.SIGTSTP, self._tstp_handler)
# add an atexit handler to ensure the terminal is restored
atexit.register(self._restore_default_terminal_settings)
@@ -258,6 +255,7 @@ class Unbuffered(object):
This is implemented by forcing a flush after each write.
"""
+
def __init__(self, stream):
self.stream = stream
@@ -302,6 +300,7 @@ class FileWrapper(object):
yet), or neither. When unwrapped, it returns an open file (or file-like)
object.
"""
+
def __init__(self, file_like):
# This records whether the file-like object returned by "unwrap" is
# purely in-memory. In that case a subprocess will need to explicitly
@@ -325,9 +324,9 @@ class FileWrapper(object):
if self.open:
if self.file_like:
if sys.version_info < (3,):
- self.file = open(self.file_like, 'w')
+ self.file = open(self.file_like, "w")
else:
- self.file = open(self.file_like, 'w', encoding='utf-8') # novm
+ self.file = open(self.file_like, "w", encoding="utf-8") # novm
else:
self.file = StringIO()
return self.file
@@ -343,8 +342,9 @@ class FileWrapper(object):
class MultiProcessFd(object):
"""Return an object which stores a file descriptor and can be passed as an
- argument to a function run with ``multiprocessing.Process``, such that
- the file descriptor is available in the subprocess."""
+ argument to a function run with ``multiprocessing.Process``, such that
+ the file descriptor is available in the subprocess."""
+
def __init__(self, fd):
self._connection = None
self._fd = None
@@ -434,7 +434,7 @@ def log_output(*args, **kwargs):
This method is actually a factory serving a per platform
(unix vs windows) log_output class
"""
- if sys.platform == 'win32':
+ if sys.platform == "win32":
return winlog(*args, **kwargs)
else:
return nixlog(*args, **kwargs)
@@ -454,8 +454,9 @@ class nixlog(object):
work within test frameworks like nose and pytest.
"""
- def __init__(self, file_like=None, echo=False, debug=0, buffer=False,
- env=None, filter_fn=None):
+ def __init__(
+ self, file_like=None, echo=False, debug=0, buffer=False, env=None, filter_fn=None
+ ):
"""Create a new output log context manager.
Args:
@@ -524,8 +525,7 @@ class nixlog(object):
raise RuntimeError("Can't re-enter the same log_output!")
if self.file_like is None:
- raise RuntimeError(
- "file argument must be set by either __init__ or __call__")
+ raise RuntimeError("file argument must be set by either __init__ or __call__")
# set up a stream for the daemon to write to
self.log_file = FileWrapper(self.file_like)
@@ -555,9 +555,7 @@ class nixlog(object):
input_multiprocess_fd = None
try:
if sys.stdin.isatty():
- input_multiprocess_fd = MultiProcessFd(
- os.dup(sys.stdin.fileno())
- )
+ input_multiprocess_fd = MultiProcessFd(os.dup(sys.stdin.fileno()))
except BaseException:
# just don't forward input if this fails
pass
@@ -566,9 +564,14 @@ class nixlog(object):
self.process = multiprocessing.Process(
target=_writer_daemon,
args=(
- input_multiprocess_fd, read_multiprocess_fd, write_fd,
- self.echo, self.log_file, child_pipe, self.filter_fn
- )
+ input_multiprocess_fd,
+ read_multiprocess_fd,
+ write_fd,
+ self.echo,
+ self.log_file,
+ child_pipe,
+ self.filter_fn,
+ ),
)
self.process.daemon = True # must set before start()
self.process.start()
@@ -609,7 +612,7 @@ class nixlog(object):
self._saved_stderr = sys.stderr
# create a file object for the pipe; redirect to it.
- pipe_fd_out = os.fdopen(write_fd, 'w')
+ pipe_fd_out = os.fdopen(write_fd, "w")
sys.stdout = pipe_fd_out
sys.stderr = pipe_fd_out
@@ -674,8 +677,7 @@ class nixlog(object):
def force_echo(self):
"""Context manager to force local echo, even if echo is off."""
if not self._active:
- raise RuntimeError(
- "Can't call force_echo() outside log_output region!")
+ raise RuntimeError("Can't call force_echo() outside log_output region!")
# This uses the xon/xoff to highlight regions to be echoed in the
# output. We us these control characters rather than, say, a
@@ -691,25 +693,26 @@ class nixlog(object):
class StreamWrapper:
- """ Wrapper class to handle redirection of io streams """
+ """Wrapper class to handle redirection of io streams"""
+
def __init__(self, sys_attr):
self.sys_attr = sys_attr
self.saved_stream = None
- if sys.platform.startswith('win32'):
+ if sys.platform.startswith("win32"):
if sys.version_info < (3, 5):
- libc = ctypes.CDLL(ctypes.util.find_library('c'))
+ libc = ctypes.CDLL(ctypes.util.find_library("c"))
else:
- if hasattr(sys, 'gettotalrefcount'): # debug build
- libc = ctypes.CDLL('ucrtbased')
+ if hasattr(sys, "gettotalrefcount"): # debug build
+ libc = ctypes.CDLL("ucrtbased")
else:
- libc = ctypes.CDLL('api-ms-win-crt-stdio-l1-1-0')
+ libc = ctypes.CDLL("api-ms-win-crt-stdio-l1-1-0")
- kernel32 = ctypes.WinDLL('kernel32')
+ kernel32 = ctypes.WinDLL("kernel32")
# https://docs.microsoft.com/en-us/windows/console/getstdhandle
- if self.sys_attr == 'stdout':
+ if self.sys_attr == "stdout":
STD_HANDLE = -11
- elif self.sys_attr == 'stderr':
+ elif self.sys_attr == "stderr":
STD_HANDLE = -12
else:
raise KeyError(self.sys_attr)
@@ -728,7 +731,7 @@ class StreamWrapper:
def redirect_stream(self, to_fd):
"""Redirect stdout to the given file descriptor."""
# Flush the C-level buffer stream
- if sys.platform.startswith('win32'):
+ if sys.platform.startswith("win32"):
self.libc.fflush(None)
else:
self.libc.fflush(self.c_stream)
@@ -739,13 +742,13 @@ class StreamWrapper:
# Make orig_stream_fd point to the same file as to_fd
os.dup2(to_fd, self.orig_stream_fd)
# Set sys_stream to a new stream that points to the redirected fd
- new_buffer = open(self.orig_stream_fd, 'wb')
+ new_buffer = open(self.orig_stream_fd, "wb")
new_stream = io.TextIOWrapper(new_buffer)
setattr(sys, self.sys_attr, new_stream)
self.sys_stream = getattr(sys, self.sys_attr)
def flush(self):
- if sys.platform.startswith('win32'):
+ if sys.platform.startswith("win32"):
self.libc.fflush(None)
else:
self.libc.fflush(self.c_stream)
@@ -768,14 +771,16 @@ class winlog(object):
Does not support the use of 'v' toggling as nixlog does.
"""
- def __init__(self, file_like=None, echo=False, debug=0, buffer=False,
- env=None, filter_fn=None):
+
+ def __init__(
+ self, file_like=None, echo=False, debug=0, buffer=False, env=None, filter_fn=None
+ ):
self.env = env
self.debug = debug
self.echo = echo
self.logfile = file_like
- self.stdout = StreamWrapper('stdout')
- self.stderr = StreamWrapper('stderr')
+ self.stdout = StreamWrapper("stdout")
+ self.stderr = StreamWrapper("stderr")
self._active = False
self._ioflag = False
self.old_stdout = sys.stdout
@@ -786,8 +791,7 @@ class winlog(object):
raise RuntimeError("Can't re-enter the same log_output!")
if self.logfile is None:
- raise RuntimeError(
- "file argument must be set by __init__ ")
+ raise RuntimeError("file argument must be set by __init__ ")
# Open both write and reading on logfile
if type(self.logfile) == StringIO:
@@ -796,8 +800,8 @@ class winlog(object):
sys.stdout = self.logfile
sys.stderr = self.logfile
else:
- self.writer = open(self.logfile, mode='wb+')
- self.reader = open(self.logfile, mode='rb+')
+ self.writer = open(self.logfile, mode="wb+")
+ self.reader = open(self.logfile, mode="rb+")
# Dup stdout so we can still write to it after redirection
self.echo_writer = open(os.dup(sys.stdout.fileno()), "w")
@@ -811,7 +815,7 @@ class winlog(object):
# if echo: write line to user
try:
while True:
- is_killed = _kill.wait(.1)
+ is_killed = _kill.wait(0.1)
# Flush buffered build output to file
# stdout/err fds refer to log file
self.stderr.flush()
@@ -819,7 +823,7 @@ class winlog(object):
line = reader.readline()
if self.echo and line:
- echo_writer.write('{0}'.format(line.decode()))
+ echo_writer.write("{0}".format(line.decode()))
echo_writer.flush()
if is_killed:
@@ -829,8 +833,9 @@ class winlog(object):
self._active = True
with replace_environment(self.env):
- self._thread = Thread(target=background_reader,
- args=(self.reader, self.echo_writer, self._kill))
+ self._thread = Thread(
+ target=background_reader, args=(self.reader, self.echo_writer, self._kill)
+ )
self._thread.start()
return self
@@ -854,13 +859,19 @@ class winlog(object):
def force_echo(self):
"""Context manager to force local echo, even if echo is off."""
if not self._active:
- raise RuntimeError(
- "Can't call force_echo() outside log_output region!")
+ raise RuntimeError("Can't call force_echo() outside log_output region!")
yield
-def _writer_daemon(stdin_multiprocess_fd, read_multiprocess_fd, write_fd, echo,
- log_file_wrapper, control_pipe, filter_fn):
+def _writer_daemon(
+ stdin_multiprocess_fd,
+ read_multiprocess_fd,
+ write_fd,
+ echo,
+ log_file_wrapper,
+ control_pipe,
+ filter_fn,
+):
"""Daemon used by ``log_output`` to write to a log file and to ``stdout``.
The daemon receives output from the parent process and writes it both
@@ -913,16 +924,16 @@ def _writer_daemon(stdin_multiprocess_fd, read_multiprocess_fd, write_fd, echo,
# write_fd to terminate the reading loop, so we close the file descriptor
# here. Forking is the process spawning method everywhere except Mac OS
# for Python >= 3.8 and on Windows
- if sys.version_info < (3, 8) or sys.platform != 'darwin':
+ if sys.version_info < (3, 8) or sys.platform != "darwin":
os.close(write_fd)
# Use line buffering (3rd param = 1) since Python 3 has a bug
# that prevents unbuffered text I/O.
if sys.version_info < (3,):
- in_pipe = os.fdopen(read_multiprocess_fd.fd, 'r', 1)
+ in_pipe = os.fdopen(read_multiprocess_fd.fd, "r", 1)
else:
# Python 3.x before 3.7 does not open with UTF-8 encoding by default
- in_pipe = os.fdopen(read_multiprocess_fd.fd, 'r', 1, encoding='utf-8')
+ in_pipe = os.fdopen(read_multiprocess_fd.fd, "r", 1, encoding="utf-8")
if stdin_multiprocess_fd:
stdin = os.fdopen(stdin_multiprocess_fd.fd)
@@ -931,7 +942,7 @@ def _writer_daemon(stdin_multiprocess_fd, read_multiprocess_fd, write_fd, echo,
# list of streams to select from
istreams = [in_pipe, stdin] if stdin else [in_pipe]
- force_echo = False # parent can force echo for certain output
+ force_echo = False # parent can force echo for certain output
log_file = log_file_wrapper.unwrap()
@@ -954,7 +965,7 @@ def _writer_daemon(stdin_multiprocess_fd, read_multiprocess_fd, write_fd, echo,
# check and the read, so we ignore SIGTTIN here.
with ignore_signal(signal.SIGTTIN):
try:
- if stdin.read(1) == 'v':
+ if stdin.read(1) == "v":
echo = not echo
except IOError as e:
# If SIGTTIN is ignored, the system gives EIO
@@ -972,14 +983,14 @@ def _writer_daemon(stdin_multiprocess_fd, read_multiprocess_fd, write_fd, echo,
line = _retry(in_pipe.readline)()
except UnicodeDecodeError:
# installs like --test=root gpgme produce non-UTF8 logs
- line = '<line lost: output was not encoded as UTF-8>\n'
+ line = "<line lost: output was not encoded as UTF-8>\n"
if not line:
return
line_count += 1
# find control characters and strip them.
- clean_line, num_controls = control.subn('', line)
+ clean_line, num_controls = control.subn("", line)
# Echo to stdout if requested or forced.
if echo or force_echo:
@@ -1043,6 +1054,7 @@ def _retry(function):
relevant for this file.
"""
+
def wrapped(*args, **kwargs):
while True:
try:
@@ -1055,6 +1067,7 @@ def _retry(function):
if e.args[0] == errno.EINTR:
continue
raise
+
return wrapped
diff --git a/lib/spack/llnl/util/tty/pty.py b/lib/spack/llnl/util/tty/pty.py
index 1a5731a5c7..8d7213a533 100644
--- a/lib/spack/llnl/util/tty/pty.py
+++ b/lib/spack/llnl/util/tty/pty.py
@@ -30,6 +30,7 @@ from spack.util.executable import which
termios = None
try:
import termios as term_mod
+
termios = term_mod
except ImportError:
pass
@@ -42,8 +43,8 @@ class ProcessController(object):
minion) similar to the way a shell would, by sending signals and I/O.
"""
- def __init__(self, pid, controller_fd,
- timeout=1, sleep_time=1e-1, debug=False):
+
+ def __init__(self, pid, controller_fd, timeout=1, sleep_time=1e-1, debug=False):
"""Create a controller to manipulate the process with id ``pid``
Args:
@@ -84,18 +85,19 @@ class ProcessController(object):
def horizontal_line(self, name):
"""Labled horizontal line for debugging."""
if self.debug:
- sys.stderr.write(
- "------------------------------------------- %s\n" % name
- )
+ sys.stderr.write("------------------------------------------- %s\n" % name)
def status(self):
"""Print debug message with status info for the minion."""
if self.debug:
canon, echo = self.get_canon_echo_attrs()
- sys.stderr.write("canon: %s, echo: %s\n" % (
- "on" if canon else "off",
- "on" if echo else "off",
- ))
+ sys.stderr.write(
+ "canon: %s, echo: %s\n"
+ % (
+ "on" if canon else "off",
+ "on" if echo else "off",
+ )
+ )
sys.stderr.write("input: %s\n" % self.input_on())
sys.stderr.write("bg: %s\n" % self.background())
sys.stderr.write("\n")
@@ -137,7 +139,7 @@ class ProcessController(object):
def wait(self, condition):
start = time.time()
- while (((time.time() - start) < self.timeout) and not condition()):
+ while ((time.time() - start) < self.timeout) and not condition():
time.sleep(1e-2)
assert condition()
@@ -219,6 +221,7 @@ class PseudoShell(object):
|_________________________________________________________|
"""
+
def __init__(self, controller_function, minion_function):
self.proc = None
self.controller_function = controller_function
@@ -242,8 +245,12 @@ class PseudoShell(object):
"""
self.proc = multiprocessing.Process(
target=PseudoShell._set_up_and_run_controller_function,
- args=(self.controller_function, self.minion_function,
- self.controller_timeout, self.sleep_time),
+ args=(
+ self.controller_function,
+ self.minion_function,
+ self.controller_timeout,
+ self.sleep_time,
+ ),
kwargs=kwargs,
)
self.proc.start()
@@ -255,7 +262,8 @@ class PseudoShell(object):
@staticmethod
def _set_up_and_run_minion_function(
- tty_name, stdout_fd, stderr_fd, ready, minion_function, **kwargs):
+ tty_name, stdout_fd, stderr_fd, ready, minion_function, **kwargs
+ ):
"""Minion process wrapper for PseudoShell.
Handles the mechanics of setting up a PTY, then calls
@@ -273,8 +281,7 @@ class PseudoShell(object):
os.close(stdin_fd)
if kwargs.get("debug"):
- sys.stderr.write(
- "minion: stdin.isatty(): %s\n" % sys.stdin.isatty())
+ sys.stderr.write("minion: stdin.isatty(): %s\n" % sys.stdin.isatty())
# tell the parent that we're really running
if kwargs.get("debug"):
@@ -288,15 +295,15 @@ class PseudoShell(object):
@staticmethod
def _set_up_and_run_controller_function(
- controller_function, minion_function, controller_timeout,
- sleep_time, **kwargs):
+ controller_function, minion_function, controller_timeout, sleep_time, **kwargs
+ ):
"""Set up a pty, spawn a minion process, execute controller_function.
Handles the mechanics of setting up a PTY, then calls
``controller_function``.
"""
- os.setsid() # new session; this process is the controller
+ os.setsid() # new session; this process is the controller
controller_fd, minion_fd = os.openpty()
pty_name = os.ttyname(minion_fd)
@@ -305,11 +312,10 @@ class PseudoShell(object):
pty_fd = os.open(pty_name, os.O_RDWR)
os.close(pty_fd)
- ready = multiprocessing.Value('i', False)
+ ready = multiprocessing.Value("i", False)
minion_process = multiprocessing.Process(
target=PseudoShell._set_up_and_run_minion_function,
- args=(pty_name, sys.stdout.fileno(), sys.stderr.fileno(),
- ready, minion_function),
+ args=(pty_name, sys.stdout.fileno(), sys.stderr.fileno(), ready, minion_function),
kwargs=kwargs,
)
minion_process.start()
@@ -329,8 +335,7 @@ class PseudoShell(object):
minion_pgid = os.getpgid(minion_process.pid)
sys.stderr.write("minion pid: %d\n" % minion_process.pid)
sys.stderr.write("minion pgid: %d\n" % minion_pgid)
- sys.stderr.write(
- "minion sid: %d\n" % os.getsid(minion_process.pid))
+ sys.stderr.write("minion sid: %d\n" % os.getsid(minion_process.pid))
sys.stderr.write("\n")
sys.stderr.flush()
# set up controller to ignore SIGTSTP, like a shell
@@ -339,7 +344,8 @@ class PseudoShell(object):
# call the controller function once the minion is ready
try:
controller = ProcessController(
- minion_process.pid, controller_fd, debug=kwargs.get("debug"))
+ minion_process.pid, controller_fd, debug=kwargs.get("debug")
+ )
controller.timeout = controller_timeout
controller.sleep_time = sleep_time
error = controller_function(minion_process, controller, **kwargs)