diff options
Diffstat (limited to 'lib/spack/llnl/util')
-rw-r--r-- | lib/spack/llnl/util/argparsewriter.py | 86 | ||||
-rw-r--r-- | lib/spack/llnl/util/compat.py | 28 | ||||
-rw-r--r-- | lib/spack/llnl/util/filesystem.py | 444 | ||||
-rw-r--r-- | lib/spack/llnl/util/lang.py | 198 | ||||
-rw-r--r-- | lib/spack/llnl/util/link_tree.py | 136 | ||||
-rw-r--r-- | lib/spack/llnl/util/lock.py | 164 | ||||
-rw-r--r-- | lib/spack/llnl/util/multiproc.py | 4 | ||||
-rw-r--r-- | lib/spack/llnl/util/symlink.py | 16 | ||||
-rw-r--r-- | lib/spack/llnl/util/tty/__init__.py | 126 | ||||
-rw-r--r-- | lib/spack/llnl/util/tty/colify.py | 74 | ||||
-rw-r--r-- | lib/spack/llnl/util/tty/color.py | 90 | ||||
-rw-r--r-- | lib/spack/llnl/util/tty/log.py | 145 | ||||
-rw-r--r-- | lib/spack/llnl/util/tty/pty.py | 54 |
13 files changed, 801 insertions, 764 deletions
diff --git a/lib/spack/llnl/util/argparsewriter.py b/lib/spack/llnl/util/argparsewriter.py index eb35d26aa8..a8db508c2f 100644 --- a/lib/spack/llnl/util/argparsewriter.py +++ b/lib/spack/llnl/util/argparsewriter.py @@ -29,8 +29,8 @@ class Command(object): - optionals: list of optional arguments (list) - subcommands: list of subcommand parsers (list) """ - def __init__(self, prog, description, usage, - positionals, optionals, subcommands): + + def __init__(self, prog, description, usage, positionals, optionals, subcommands): self.prog = prog self.description = description self.usage = usage @@ -71,15 +71,15 @@ class ArgparseWriter(argparse.HelpFormatter): """ self.parser = parser - split_prog = parser.prog.split(' ') + split_prog = parser.prog.split(" ") split_prog[-1] = prog - prog = ' '.join(split_prog) + prog = " ".join(split_prog) description = parser.description fmt = parser._get_formatter() actions = parser._actions groups = parser._mutually_exclusive_groups - usage = fmt._format_usage(None, actions, groups, '').strip() + usage = fmt._format_usage(None, actions, groups, "").strip() # Go through actions and split them into optionals, positionals, # and subcommands @@ -90,8 +90,8 @@ class ArgparseWriter(argparse.HelpFormatter): if action.option_strings: flags = action.option_strings dest_flags = fmt._format_action_invocation(action) - help = self._expand_help(action) if action.help else '' - help = help.replace('\n', ' ') + help = self._expand_help(action) if action.help else "" + help = help.replace("\n", " ") optionals.append((flags, dest_flags, help)) elif isinstance(action, argparse._SubParsersAction): for subaction in action._choices_actions: @@ -100,20 +100,19 @@ class ArgparseWriter(argparse.HelpFormatter): # Look for aliases of the form 'name (alias, ...)' if self.aliases: - match = re.match(r'(.*) \((.*)\)', subaction.metavar) + match = re.match(r"(.*) \((.*)\)", subaction.metavar) if match: - aliases = match.group(2).split(', ') + aliases = match.group(2).split(", ") for alias in aliases: subparser = action._name_parser_map[alias] subcommands.append((subparser, alias)) else: args = fmt._format_action_invocation(action) - help = self._expand_help(action) if action.help else '' - help = help.replace('\n', ' ') + help = self._expand_help(action) if action.help else "" + help = help.replace("\n", " ") positionals.append((args, help)) - return Command( - prog, description, usage, positionals, optionals, subcommands) + return Command(prog, description, usage, positionals, optionals, subcommands) def format(self, cmd): """Returns the string representation of a single node in the @@ -161,14 +160,13 @@ class ArgparseWriter(argparse.HelpFormatter): raise -_rst_levels = ['=', '-', '^', '~', ':', '`'] +_rst_levels = ["=", "-", "^", "~", ":", "`"] class ArgparseRstWriter(ArgparseWriter): """Write argparse output as rst sections.""" - def __init__(self, prog, out=None, aliases=False, - rst_levels=_rst_levels): + def __init__(self, prog, out=None, aliases=False, rst_levels=_rst_levels): """Create a new ArgparseRstWriter. Parameters: @@ -217,11 +215,12 @@ class ArgparseRstWriter(ArgparseWriter): {1} {2} -""".format(prog.replace(' ', '-'), prog, - self.rst_levels[self.level] * len(prog)) +""".format( + prog.replace(" ", "-"), prog, self.rst_levels[self.level] * len(prog) + ) def description(self, description): - return description + '\n\n' + return description + "\n\n" def usage(self, usage): return """\ @@ -229,33 +228,39 @@ class ArgparseRstWriter(ArgparseWriter): {0} -""".format(usage) +""".format( + usage + ) def begin_positionals(self): - return '\n**Positional arguments**\n\n' + return "\n**Positional arguments**\n\n" def positional(self, name, help): return """\ {0} {1} -""".format(name, help) +""".format( + name, help + ) def end_positionals(self): - return '' + return "" def begin_optionals(self): - return '\n**Optional arguments**\n\n' + return "\n**Optional arguments**\n\n" def optional(self, opts, help): return """\ ``{0}`` {1} -""".format(opts, help) +""".format( + opts, help + ) def end_optionals(self): - return '' + return "" def begin_subcommands(self, subcommands): string = """ @@ -267,11 +272,10 @@ class ArgparseRstWriter(ArgparseWriter): """ for cmd, _ in subcommands: - prog = re.sub(r'^[^ ]* ', '', cmd.prog) - string += ' * :ref:`{0} <{1}>`\n'.format( - prog, cmd.prog.replace(' ', '-')) + prog = re.sub(r"^[^ ]* ", "", cmd.prog) + string += " * :ref:`{0} <{1}>`\n".format(prog, cmd.prog.replace(" ", "-")) - return string + '\n' + return string + "\n" class ArgparseCompletionWriter(ArgparseWriter): @@ -306,9 +310,11 @@ class ArgparseCompletionWriter(ArgparseWriter): # Flatten lists of lists optionals = [x for xx in optionals for x in xx] - return (self.start_function(cmd.prog) + - self.body(positionals, optionals, subcommands) + - self.end_function(cmd.prog)) + return ( + self.start_function(cmd.prog) + + self.body(positionals, optionals, subcommands) + + self.end_function(cmd.prog) + ) def start_function(self, prog): """Returns the syntax needed to begin a function definition. @@ -319,8 +325,8 @@ class ArgparseCompletionWriter(ArgparseWriter): Returns: str: the function definition beginning """ - name = prog.replace('-', '_').replace(' ', '_') - return '\n_{0}() {{'.format(name) + name = prog.replace("-", "_").replace(" ", "_") + return "\n_{0}() {{".format(name) def end_function(self, prog=None): """Returns the syntax needed to end a function definition. @@ -331,7 +337,7 @@ class ArgparseCompletionWriter(ArgparseWriter): Returns: str: the function definition ending """ - return '}\n' + return "}\n" def body(self, positionals, optionals, subcommands): """Returns the body of the function. @@ -344,7 +350,7 @@ class ArgparseCompletionWriter(ArgparseWriter): Returns: str: the function body """ - return '' + return "" def positionals(self, positionals): """Returns the syntax for reporting positional arguments. @@ -355,7 +361,7 @@ class ArgparseCompletionWriter(ArgparseWriter): Returns: str: the syntax for positional arguments """ - return '' + return "" def optionals(self, optionals): """Returns the syntax for reporting optional flags. @@ -366,7 +372,7 @@ class ArgparseCompletionWriter(ArgparseWriter): Returns: str: the syntax for optional flags """ - return '' + return "" def subcommands(self, subcommands): """Returns the syntax for reporting subcommands. @@ -377,4 +383,4 @@ class ArgparseCompletionWriter(ArgparseWriter): Returns: str: the syntax for subcommand parsers """ - return '' + return "" diff --git a/lib/spack/llnl/util/compat.py b/lib/spack/llnl/util/compat.py index ca914d0fb6..ebe509f3a7 100644 --- a/lib/spack/llnl/util/compat.py +++ b/lib/spack/llnl/util/compat.py @@ -18,22 +18,22 @@ else: map = map zip = zip from itertools import zip_longest as zip_longest # novm # noqa: F401 - from urllib.parse import urlencode as urlencode # novm # noqa: F401 - from urllib.request import urlopen as urlopen # novm # noqa: F401 + from urllib.parse import urlencode as urlencode # novm # noqa: F401 + from urllib.request import urlopen as urlopen # novm # noqa: F401 if sys.version_info >= (3, 3): - from collections.abc import Hashable as Hashable # novm - from collections.abc import Iterable as Iterable # novm - from collections.abc import Mapping as Mapping # novm - from collections.abc import MutableMapping as MutableMapping # novm + from collections.abc import Hashable as Hashable # novm + from collections.abc import Iterable as Iterable # novm + from collections.abc import Mapping as Mapping # novm + from collections.abc import MutableMapping as MutableMapping # novm from collections.abc import MutableSequence as MutableSequence # novm - from collections.abc import MutableSet as MutableSet # novm - from collections.abc import Sequence as Sequence # novm + from collections.abc import MutableSet as MutableSet # novm + from collections.abc import Sequence as Sequence # novm else: - from collections import Hashable as Hashable # noqa: F401 - from collections import Iterable as Iterable # noqa: F401 - from collections import Mapping as Mapping # noqa: F401 - from collections import MutableMapping as MutableMapping # noqa: F401 + from collections import Hashable as Hashable # noqa: F401 + from collections import Iterable as Iterable # noqa: F401 + from collections import Mapping as Mapping # noqa: F401 + from collections import MutableMapping as MutableMapping # noqa: F401 from collections import MutableSequence as MutableSequence # noqa: F401 - from collections import MutableSet as MutableSet # noqa: F401 - from collections import Sequence as Sequence # noqa: F401 + from collections import MutableSet as MutableSet # noqa: F401 + from collections import Sequence as Sequence # noqa: F401 diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 185f45d131..1740fb71c0 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -27,7 +27,7 @@ from llnl.util.symlink import symlink from spack.util.executable import Executable from spack.util.path import path_to_os_path, system_path_filter -is_windows = _platform == 'win32' +is_windows = _platform == "win32" if not is_windows: import grp @@ -37,56 +37,57 @@ else: __all__ = [ - 'FileFilter', - 'FileList', - 'HeaderList', - 'LibraryList', - 'ancestor', - 'can_access', - 'change_sed_delimiter', - 'copy_mode', - 'filter_file', - 'find', - 'find_headers', - 'find_all_headers', - 'find_libraries', - 'find_system_libraries', - 'fix_darwin_install_name', - 'force_remove', - 'force_symlink', - 'getuid', - 'chgrp', - 'chmod_x', - 'copy', - 'install', - 'copy_tree', - 'install_tree', - 'is_exe', - 'join_path', - 'last_modification_time_recursive', - 'library_extensions', - 'mkdirp', - 'partition_path', - 'prefixes', - 'remove_dead_links', - 'remove_directory_contents', - 'remove_if_dead_link', - 'remove_linked_tree', - 'rename', - 'set_executable', - 'set_install_permissions', - 'touch', - 'touchp', - 'traverse_tree', - 'unset_executable_mode', - 'working_dir', - 'keep_modification_time' + "FileFilter", + "FileList", + "HeaderList", + "LibraryList", + "ancestor", + "can_access", + "change_sed_delimiter", + "copy_mode", + "filter_file", + "find", + "find_headers", + "find_all_headers", + "find_libraries", + "find_system_libraries", + "fix_darwin_install_name", + "force_remove", + "force_symlink", + "getuid", + "chgrp", + "chmod_x", + "copy", + "install", + "copy_tree", + "install_tree", + "is_exe", + "join_path", + "last_modification_time_recursive", + "library_extensions", + "mkdirp", + "partition_path", + "prefixes", + "remove_dead_links", + "remove_directory_contents", + "remove_if_dead_link", + "remove_linked_tree", + "rename", + "set_executable", + "set_install_permissions", + "touch", + "touchp", + "traverse_tree", + "unset_executable_mode", + "working_dir", + "keep_modification_time", ] def getuid(): if is_windows: import ctypes + if ctypes.windll.shell32.IsUserAnAdmin() == 0: return 1 return 0 @@ -111,7 +112,7 @@ def path_contains_subdirectory(path, root): #: This generates the library filenames that may appear on any OS. -library_extensions = ['a', 'la', 'so', 'tbd', 'dylib'] +library_extensions = ["a", "la", "so", "tbd", "dylib"] def possible_library_filenames(library_names): @@ -120,8 +121,9 @@ def possible_library_filenames(library_names): """ lib_extensions = library_extensions return set( - '.'.join((lib, extension)) for lib, extension in - itertools.product(library_names, lib_extensions)) + ".".join((lib, extension)) + for lib, extension in itertools.product(library_names, lib_extensions) + ) def paths_containing_libs(paths, library_names): @@ -174,19 +176,21 @@ def filter_file(regex, repl, *filenames, **kwargs): file is copied verbatim. Default is to filter until the end of the file. """ - string = kwargs.get('string', False) - backup = kwargs.get('backup', False) - ignore_absent = kwargs.get('ignore_absent', False) - stop_at = kwargs.get('stop_at', None) + string = kwargs.get("string", False) + backup = kwargs.get("backup", False) + ignore_absent = kwargs.get("ignore_absent", False) + stop_at = kwargs.get("stop_at", None) # Allow strings to use \1, \2, etc. for replacement, like sed if not callable(repl): - unescaped = repl.replace(r'\\', '\\') + unescaped = repl.replace(r"\\", "\\") def replace_groups_with_groupid(m): def groupid_to_group(x): return m.group(int(x.group(1))) - return re.sub(r'\\([1-9])', groupid_to_group, unescaped) + + return re.sub(r"\\([1-9])", groupid_to_group, unescaped) + repl = replace_groups_with_groupid if string: @@ -217,16 +221,16 @@ def filter_file(regex, repl, *filenames, **kwargs): try: extra_kwargs = {} if sys.version_info > (3, 0): - extra_kwargs = {'errors': 'surrogateescape'} + extra_kwargs = {"errors": "surrogateescape"} # Open as a text file and filter until the end of the file is # reached or we found a marker in the line if it was specified - with open(tmp_filename, mode='r', **extra_kwargs) as input_file: - with open(filename, mode='w', **extra_kwargs) as output_file: + with open(tmp_filename, mode="r", **extra_kwargs) as input_file: + with open(filename, mode="w", **extra_kwargs) as output_file: # Using iter and readline is a workaround needed not to # disable input_file.tell(), which will happen if we call # input_file.next() implicitly via the for loop - for line in iter(input_file.readline, ''): + for line in iter(input_file.readline, ""): if stop_at is not None: current_position = input_file.tell() if stop_at == line.strip(): @@ -240,9 +244,9 @@ def filter_file(regex, repl, *filenames, **kwargs): # If we stopped filtering at some point, reopen the file in # binary mode and copy verbatim the remaining part if current_position and stop_at: - with open(tmp_filename, mode='rb') as input_file: + with open(tmp_filename, mode="rb") as input_file: input_file.seek(current_position) - with open(filename, mode='ab') as output_file: + with open(filename, mode="ab") as output_file: output_file.writelines(input_file.readlines()) except BaseException: @@ -281,26 +285,26 @@ def change_sed_delimiter(old_delim, new_delim, *filenames): new_delim (str): The delimiter to replace with *filenames: One or more files to search and replace """ - assert(len(old_delim) == 1) - assert(len(new_delim) == 1) + assert len(old_delim) == 1 + assert len(new_delim) == 1 # TODO: handle these cases one day? - assert(old_delim != '"') - assert(old_delim != "'") - assert(new_delim != '"') - assert(new_delim != "'") + assert old_delim != '"' + assert old_delim != "'" + assert new_delim != '"' + assert new_delim != "'" whole_lines = "^s@([^@]*)@(.*)@[gIp]$" - whole_lines = whole_lines.replace('@', old_delim) + whole_lines = whole_lines.replace("@", old_delim) single_quoted = r"'s@((?:\\'|[^@'])*)@((?:\\'|[^'])*)@[gIp]?'" - single_quoted = single_quoted.replace('@', old_delim) + single_quoted = single_quoted.replace("@", old_delim) double_quoted = r'"s@((?:\\"|[^@"])*)@((?:\\"|[^"])*)@[gIp]?"' - double_quoted = double_quoted.replace('@', old_delim) + double_quoted = double_quoted.replace("@", old_delim) - repl = r's@\1@\2@g' - repl = repl.replace('@', new_delim) + repl = r"s@\1@\2@g" + repl = repl.replace("@", new_delim) filenames = path_to_os_path(*filenames) for f in filenames: filter_file(whole_lines, repl, f) @@ -324,8 +328,7 @@ def exploding_archive_catch(stage): # Expand all tarballs in their own directory to contain # exploding tarballs. - tarball_container = os.path.join(stage.path, - "spack-expanded-archive") + tarball_container = os.path.join(stage.path, "spack-expanded-archive") mkdirp(tarball_container) orig_dir = os.getcwd() os.chdir(tarball_container) @@ -349,7 +352,7 @@ def exploding_archive_handler(tarball_container, stage): where archive is being expanded """ files = os.listdir(tarball_container) - non_hidden = [f for f in files if not f.startswith('.')] + non_hidden = [f for f in files if not f.startswith(".")] if len(non_hidden) == 1: src = os.path.join(tarball_container, non_hidden[0]) if os.path.isdir(src): @@ -377,11 +380,13 @@ def get_owner_uid(path, err_msg=None): p_stat = os.stat(path) if p_stat.st_mode & stat.S_IRWXU != stat.S_IRWXU: - tty.error("Expected {0} to support mode {1}, but it is {2}" - .format(path, stat.S_IRWXU, p_stat.st_mode)) + tty.error( + "Expected {0} to support mode {1}, but it is {2}".format( + path, stat.S_IRWXU, p_stat.st_mode + ) + ) - raise OSError(errno.EACCES, - err_msg.format(path, path) if err_msg else "") + raise OSError(errno.EACCES, err_msg.format(path, path) if err_msg else "") else: p_stat = os.stat(path) @@ -389,8 +394,8 @@ def get_owner_uid(path, err_msg=None): owner_uid = p_stat.st_uid else: sid = win32security.GetFileSecurity( - path, win32security.OWNER_SECURITY_INFORMATION) \ - .GetSecurityDescriptorOwner() + path, win32security.OWNER_SECURITY_INFORMATION + ).GetSecurityDescriptorOwner() owner_uid = win32security.LookupAccountSid(None, sid)[0] return owner_uid @@ -460,8 +465,7 @@ def chmod_x(entry, perms): @system_path_filter def copy_mode(src, dest): - """Set the mode of dest to that of src unless it is a link. - """ + """Set the mode of dest to that of src unless it is a link.""" if os.path.islink(dest): return src_mode = os.stat(src).st_mode @@ -504,17 +508,17 @@ def copy(src, dest, _permissions=False): not a directory """ if _permissions: - tty.debug('Installing {0} to {1}'.format(src, dest)) + tty.debug("Installing {0} to {1}".format(src, dest)) else: - tty.debug('Copying {0} to {1}'.format(src, dest)) + tty.debug("Copying {0} to {1}".format(src, dest)) files = glob.glob(src) if not files: raise IOError("No such file or directory: '{0}'".format(src)) if len(files) > 1 and not os.path.isdir(dest): raise ValueError( - "'{0}' matches multiple files but '{1}' is not a directory".format( - src, dest)) + "'{0}' matches multiple files but '{1}' is not a directory".format(src, dest) + ) for src in files: # Expand dest to its eventual full path if it is a directory. @@ -592,9 +596,9 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False): ValueError: if *src* is a parent directory of *dest* """ if _permissions: - tty.debug('Installing {0} to {1}'.format(src, dest)) + tty.debug("Installing {0} to {1}".format(src, dest)) else: - tty.debug('Copying {0} to {1}'.format(src, dest)) + tty.debug("Copying {0} to {1}".format(src, dest)) abs_dest = os.path.abspath(dest) if not abs_dest.endswith(os.path.sep): @@ -612,15 +616,20 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False): # Stop early to avoid unnecessary recursion if being asked to copy # from a parent directory. if abs_dest.startswith(abs_src): - raise ValueError('Cannot copy ancestor directory {0} into {1}'. - format(abs_src, abs_dest)) + raise ValueError( + "Cannot copy ancestor directory {0} into {1}".format(abs_src, abs_dest) + ) mkdirp(abs_dest) - for s, d in traverse_tree(abs_src, abs_dest, order='pre', - follow_symlinks=not symlinks, - ignore=ignore, - follow_nonexisting=True): + for s, d in traverse_tree( + abs_src, + abs_dest, + order="pre", + follow_symlinks=not symlinks, + ignore=ignore, + follow_nonexisting=True, + ): if os.path.islink(s): link_target = resolve_link_target_relative_to_the_link(s) if symlinks: @@ -628,8 +637,7 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False): if os.path.isabs(target): new_target = re.sub(abs_src, abs_dest, target) if new_target != target: - tty.debug("Redirecting link {0} to {1}" - .format(target, new_target)) + tty.debug("Redirecting link {0} to {1}".format(target, new_target)) target = new_target symlink(target, d) @@ -679,10 +687,9 @@ def get_filetype(path_name): """ Return the output of file path_name as a string to identify file type. """ - file = Executable('file') - file.add_default_env('LC_ALL', 'C') - output = file('-b', '-h', '%s' % path_name, - output=str, error=str) + file = Executable("file") + file.add_default_env("LC_ALL", "C") + output = file("-b", "-h", "%s" % path_name, output=str, error=str) return output.strip() @@ -703,8 +710,8 @@ def is_nonsymlink_exe_with_shebang(path): return False # Should start with a shebang - with open(path, 'rb') as f: - return f.read(2) == b'#!' + with open(path, "rb") as f: + return f.read(2) == b"#!" except (IOError, OSError): return False @@ -736,16 +743,16 @@ def mkdirp(*paths, **kwargs): intermediate get the same permissions specified in the arguments to mkdirp -- default value is 'args' """ - mode = kwargs.get('mode', None) - group = kwargs.get('group', None) - default_perms = kwargs.get('default_perms', 'args') + mode = kwargs.get("mode", None) + group = kwargs.get("group", None) + default_perms = kwargs.get("default_perms", "args") paths = path_to_os_path(*paths) for path in paths: if not os.path.exists(path): try: # detect missing intermediate folders intermediate_folders = [] - last_parent = '' + last_parent = "" intermediate_path = os.path.dirname(path) @@ -772,10 +779,10 @@ def mkdirp(*paths, **kwargs): # ones and if mode_intermediate has been specified, otherwise # intermediate folders list is not populated at all and default # OS mode will be used - if default_perms == 'args': + if default_perms == "args": intermediate_mode = mode intermediate_group = group - elif default_perms == 'parents': + elif default_perms == "parents": stat_info = os.stat(last_parent) intermediate_mode = stat_info.st_mode intermediate_group = stat_info.st_gid @@ -788,10 +795,8 @@ def mkdirp(*paths, **kwargs): if intermediate_mode is not None: os.chmod(intermediate_path, intermediate_mode) if intermediate_group is not None: - chgrp_if_not_world_writable(intermediate_path, - intermediate_group) - os.chmod(intermediate_path, - intermediate_mode) # reset sticky bit after + chgrp_if_not_world_writable(intermediate_path, intermediate_group) + os.chmod(intermediate_path, intermediate_mode) # reset sticky bit after except OSError as e: if e.errno != errno.EEXIST or not os.path.isdir(path): @@ -803,7 +808,7 @@ def mkdirp(*paths, **kwargs): @system_path_filter def force_remove(*paths): """Remove files without printing errors. Like ``rm -f``, does NOT - remove directories.""" + remove directories.""" for path in paths: try: os.remove(path) @@ -814,7 +819,7 @@ def force_remove(*paths): @contextmanager @system_path_filter def working_dir(dirname, **kwargs): - if kwargs.get('create', False): + if kwargs.get("create", False): mkdirp(dirname) orig_dir = os.getcwd() @@ -847,19 +852,17 @@ def replace_directory_transaction(directory_name): # Check the input is indeed a directory with absolute path. # Raise before anything is done to avoid moving the wrong directory directory_name = os.path.abspath(directory_name) - assert os.path.isdir(directory_name), 'Not a directory: ' + directory_name + assert os.path.isdir(directory_name), "Not a directory: " + directory_name # Note: directory_name is normalized here, meaning the trailing slash is dropped, # so dirname is the directory's parent not the directory itself. - tmpdir = tempfile.mkdtemp( - dir=os.path.dirname(directory_name), - prefix='.backup') + tmpdir = tempfile.mkdtemp(dir=os.path.dirname(directory_name), prefix=".backup") # We have to jump through hoops to support Windows, since # os.rename(directory_name, tmpdir) errors there. - backup_dir = os.path.join(tmpdir, 'backup') + backup_dir = os.path.join(tmpdir, "backup") os.rename(directory_name, backup_dir) - tty.debug('Directory moved [src={0}, dest={1}]'.format(directory_name, backup_dir)) + tty.debug("Directory moved [src={0}, dest={1}]".format(directory_name, backup_dir)) try: yield backup_dir @@ -874,12 +877,12 @@ def replace_directory_transaction(directory_name): except Exception as outer_exception: raise CouldNotRestoreDirectoryBackup(inner_exception, outer_exception) - tty.debug('Directory recovered [{0}]'.format(directory_name)) + tty.debug("Directory recovered [{0}]".format(directory_name)) raise else: # Otherwise delete the temporary directory shutil.rmtree(tmpdir, ignore_errors=True) - tty.debug('Temporary directory deleted [{0}]'.format(tmpdir)) + tty.debug("Temporary directory deleted [{0}]".format(tmpdir)) @system_path_filter @@ -904,7 +907,7 @@ def hash_directory(directory, ignore=[]): # TODO: if caching big files becomes an issue, convert this to # TODO: read in chunks. Currently it's used only for testing # TODO: purposes. - with open(filename, 'rb') as f: + with open(filename, "rb") as f: md5_hash.update(f.read()) return md5_hash.hexdigest() @@ -916,15 +919,15 @@ def write_tmp_and_move(filename): """Write to a temporary file, then move into place.""" dirname = os.path.dirname(filename) basename = os.path.basename(filename) - tmp = os.path.join(dirname, '.%s.tmp' % basename) - with open(tmp, 'w') as f: + tmp = os.path.join(dirname, ".%s.tmp" % basename) + with open(tmp, "w") as f: yield f shutil.move(tmp, filename) @contextmanager @system_path_filter -def open_if_filename(str_or_file, mode='r'): +def open_if_filename(str_or_file, mode="r"): """Takes either a path or a file object, and opens it if it is a path. If it's a file object, just yields the file object. @@ -940,9 +943,9 @@ def open_if_filename(str_or_file, mode='r'): def touch(path): """Creates an empty file at the specified path.""" if is_windows: - perms = (os.O_WRONLY | os.O_CREAT) + perms = os.O_WRONLY | os.O_CREAT else: - perms = (os.O_WRONLY | os.O_CREAT | os.O_NONBLOCK | os.O_NOCTTY) + perms = os.O_WRONLY | os.O_CREAT | os.O_NONBLOCK | os.O_NOCTTY fd = None try: fd = os.open(path, perms) @@ -954,8 +957,7 @@ def touch(path): @system_path_filter def touchp(path): - """Like ``touch``, but creates any parent directories needed for the file. - """ + """Like ``touch``, but creates any parent directories needed for the file.""" mkdirp(os.path.dirname(path)) touch(path) @@ -990,8 +992,7 @@ def ancestor(dir, n=1): def get_single_file(directory): fnames = os.listdir(directory) if len(fnames) != 1: - raise ValueError("Expected exactly 1 file, got {0}" - .format(str(len(fnames)))) + raise ValueError("Expected exactly 1 file, got {0}".format(str(len(fnames)))) return fnames[0] @@ -1025,7 +1026,7 @@ def can_access(file_name): @system_path_filter -def traverse_tree(source_root, dest_root, rel_path='', **kwargs): +def traverse_tree(source_root, dest_root, rel_path="", **kwargs): """Traverse two filesystem trees simultaneously. Walks the LinkTree directory in pre or post order. Yields each @@ -1057,16 +1058,16 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): ``src`` that do not exit in ``dest``. Default is True follow_links (bool): Whether to descend into symlinks in ``src`` """ - follow_nonexisting = kwargs.get('follow_nonexisting', True) - follow_links = kwargs.get('follow_link', False) + follow_nonexisting = kwargs.get("follow_nonexisting", True) + follow_links = kwargs.get("follow_link", False) # Yield in pre or post order? - order = kwargs.get('order', 'pre') - if order not in ('pre', 'post'): + order = kwargs.get("order", "pre") + if order not in ("pre", "post"): raise ValueError("Order must be 'pre' or 'post'.") # List of relative paths to ignore under the src root. - ignore = kwargs.get('ignore', None) or (lambda filename: False) + ignore = kwargs.get("ignore", None) or (lambda filename: False) # Don't descend into ignored directories if ignore(rel_path): @@ -1076,7 +1077,7 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): dest_path = os.path.join(dest_root, rel_path) # preorder yields directories before children - if order == 'pre': + if order == "pre": yield (source_path, dest_path) for f in os.listdir(source_path): @@ -1088,14 +1089,12 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): # TODO: for symlinks, os.path.isdir looks for the link target. If the # target is relative to the link, then that may not resolve properly # relative to our cwd - see resolve_link_target_relative_to_the_link - if os.path.isdir(source_child) and ( - follow_links or not os.path.islink(source_child)): + if os.path.isdir(source_child) and (follow_links or not os.path.islink(source_child)): # When follow_nonexisting isn't set, don't descend into dirs # in source that do not exist in dest if follow_nonexisting or os.path.exists(dest_child): - tuples = traverse_tree( - source_root, dest_root, rel_child, **kwargs) + tuples = traverse_tree(source_root, dest_root, rel_child, **kwargs) for t in tuples: yield t @@ -1103,7 +1102,7 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): elif not ignore(os.path.join(rel_path, f)): yield (source_child, dest_child) - if order == 'post': + if order == "post": yield (source_path, dest_path) @@ -1134,7 +1133,7 @@ def lexists_islink_isdir(path): return True, is_link, is_dir -def visit_directory_tree(root, visitor, rel_path='', depth=0): +def visit_directory_tree(root, visitor, rel_path="", depth=0): """ Recurses the directory root depth-first through a visitor pattern @@ -1172,8 +1171,7 @@ def visit_directory_tree(root, visitor, rel_path='', depth=0): try: isdir = f.is_dir() except OSError as e: - if is_windows and hasattr(e, 'winerror')\ - and e.winerror == 5 and islink: + if is_windows and hasattr(e, "winerror") and e.winerror == 5 and islink: # if path is a symlink, determine destination and # evaluate file vs directory link_target = resolve_link_target_relative_to_the_link(f) @@ -1221,9 +1219,11 @@ def set_executable(path): def last_modification_time_recursive(path): path = os.path.abspath(path) times = [os.stat(path).st_mtime] - times.extend(os.stat(os.path.join(root, name)).st_mtime - for root, dirs, files in os.walk(path) - for name in dirs + files) + times.extend( + os.stat(os.path.join(root, name)).st_mtime + for root, dirs, files in os.walk(path) + for name in dirs + files + ) return max(times) @@ -1282,18 +1282,23 @@ def readonly_file_handler(ignore_errors=False): and will raise a separate error if it is ever invoked (by accident) on a non-Windows system. """ + def error_remove_readonly(func, path, exc): if not is_windows: raise RuntimeError("This method should only be invoked on Windows") excvalue = exc[1] - if is_windows and func in (os.rmdir, os.remove, os.unlink) and\ - excvalue.errno == errno.EACCES: + if ( + is_windows + and func in (os.rmdir, os.remove, os.unlink) + and excvalue.errno == errno.EACCES + ): # change the file to be readable,writable,executable: 0777 os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # retry func(path) elif not ignore_errors: raise + return error_remove_readonly @@ -1309,13 +1314,13 @@ def remove_linked_tree(path): Parameters: path (str): Directory to be removed """ - kwargs = {'ignore_errors': True} + kwargs = {"ignore_errors": True} # Windows readonly files cannot be removed by Python # directly. if is_windows: - kwargs['ignore_errors'] = False - kwargs['onerror'] = readonly_file_handler(ignore_errors=True) + kwargs["ignore_errors"] = False + kwargs["onerror"] = readonly_file_handler(ignore_errors=True) if os.path.exists(path): if os.path.islink(path): @@ -1344,9 +1349,7 @@ def safe_remove(*files_or_dirs): # Sort them so that shorter paths like "/foo/bar" come before # nested paths like "/foo/bar/baz.yaml". This simplifies the # handling of temporary copies below - sorted_matches = sorted([ - os.path.abspath(x) for x in itertools.chain(*glob_matches) - ], key=len) + sorted_matches = sorted([os.path.abspath(x) for x in itertools.chain(*glob_matches)], key=len) # Copy files and directories in a temporary location removed, dst_root = {}, tempfile.mkdtemp() @@ -1361,7 +1364,7 @@ def safe_remove(*files_or_dirs): continue # The monotonic ID is a simple way to make the filename # or directory name unique in the temporary folder - basename = os.path.basename(file_or_dir) + '-{0}'.format(id) + basename = os.path.basename(file_or_dir) + "-{0}".format(id) temporary_path = os.path.join(dst_root, basename) shutil.move(file_or_dir, temporary_path) removed[file_or_dir] = temporary_path @@ -1390,11 +1393,11 @@ def fix_darwin_install_name(path): libs = glob.glob(join_path(path, "*.dylib")) for lib in libs: # fix install name first: - install_name_tool = Executable('install_name_tool') - install_name_tool('-id', lib, lib) - otool = Executable('otool') - long_deps = otool('-L', lib, output=str).split('\n') - deps = [dep.partition(' ')[0][1::] for dep in long_deps[2:-1]] + install_name_tool = Executable("install_name_tool") + install_name_tool("-id", lib, lib) + otool = Executable("otool") + long_deps = otool("-L", lib, output=str).split("\n") + deps = [dep.partition(" ")[0][1::] for dep in long_deps[2:-1]] # fix all dependencies: for dep in deps: for loc in libs: @@ -1404,7 +1407,7 @@ def fix_darwin_install_name(path): # but we don't know builddir (nor how symbolic links look # in builddir). We thus only compare the basenames. if os.path.basename(dep) == os.path.basename(loc): - install_name_tool('-change', dep, loc, lib) + install_name_tool("-change", dep, loc, lib) break @@ -1534,9 +1537,7 @@ class FileList(Sequence): Returns: list: A list of directories """ - return list(dedupe( - os.path.dirname(x) for x in self.files if os.path.dirname(x) - )) + return list(dedupe(os.path.dirname(x) for x in self.files if os.path.dirname(x))) @property def basenames(self): @@ -1572,11 +1573,11 @@ class FileList(Sequence): def __len__(self): return len(self.files) - def joined(self, separator=' '): + def joined(self, separator=" "): return separator.join(self.files) def __repr__(self): - return self.__class__.__name__ + '(' + repr(self.files) + ')' + return self.__class__.__name__ + "(" + repr(self.files) + ")" def __str__(self): return self.joined() @@ -1593,7 +1594,7 @@ class HeaderList(FileList): # as "xinclude" will cause false matches. # Avoid matching paths such as <prefix>/include/something/detail/include, # e.g. in the CUDA Toolkit which ships internal libc++ headers. - include_regex = re.compile(r'(.*?)(\binclude\b)(.*)') + include_regex = re.compile(r"(.*?)(\binclude\b)(.*)") def __init__(self, files): super(HeaderList, self).__init__(files) @@ -1658,7 +1659,7 @@ class HeaderList(FileList): name = x # Valid extensions include: ['.cuh', '.hpp', '.hh', '.h'] - for ext in ['.cuh', '.hpp', '.hh', '.h']: + for ext in [".cuh", ".hpp", ".hh", ".h"]: i = name.rfind(ext) if i != -1: names.append(name[:i]) @@ -1680,7 +1681,7 @@ class HeaderList(FileList): Returns: str: A joined list of include flags """ - return ' '.join(['-I' + x for x in self.directories]) + return " ".join(["-I" + x for x in self.directories]) @property def macro_definitions(self): @@ -1695,7 +1696,7 @@ class HeaderList(FileList): Returns: str: A joined list of macro definitions """ - return ' '.join(self._macro_definitions) + return " ".join(self._macro_definitions) @property def cpp_flags(self): @@ -1713,7 +1714,7 @@ class HeaderList(FileList): """ cpp_flags = self.include_flags if self.macro_definitions: - cpp_flags += ' ' + self.macro_definitions + cpp_flags += " " + self.macro_definitions return cpp_flags def add_macro(self, macro): @@ -1752,24 +1753,30 @@ def find_headers(headers, root, recursive=False): if isinstance(headers, six.string_types): headers = [headers] elif not isinstance(headers, Sequence): - message = '{0} expects a string or sequence of strings as the ' - message += 'first argument [got {1} instead]' + message = "{0} expects a string or sequence of strings as the " + message += "first argument [got {1} instead]" message = message.format(find_headers.__name__, type(headers)) raise TypeError(message) # Construct the right suffix for the headers suffixes = [ # C - 'h', + "h", # C++ - 'hpp', 'hxx', 'hh', 'H', 'txx', 'tcc', 'icc', + "hpp", + "hxx", + "hh", + "H", + "txx", + "tcc", + "icc", # Fortran - 'mod', 'inc', + "mod", + "inc", ] # List of headers we are searching with suffixes - headers = ['{0}.{1}'.format(header, suffix) for header in headers - for suffix in suffixes] + headers = ["{0}.{1}".format(header, suffix) for header in headers for suffix in suffixes] return HeaderList(find(root, headers, recursive)) @@ -1785,7 +1792,7 @@ def find_all_headers(root): Returns: List of all headers found in ``root`` and subdirectories. """ - return find_headers('*', root=root, recursive=True) + return find_headers("*", root=root, recursive=True) class LibraryList(FileList): @@ -1819,11 +1826,11 @@ class LibraryList(FileList): for x in self.basenames: name = x - if x.startswith('lib'): + if x.startswith("lib"): name = x[3:] # Valid extensions include: ['.dylib', '.so', '.a'] - for ext in ['.dylib', '.so', '.a']: + for ext in [".dylib", ".so", ".a"]: i = name.rfind(ext) if i != -1: names.append(name[:i]) @@ -1845,7 +1852,7 @@ class LibraryList(FileList): Returns: str: A joined list of search flags """ - return ' '.join(['-L' + x for x in self.directories]) + return " ".join(["-L" + x for x in self.directories]) @property def link_flags(self): @@ -1858,7 +1865,7 @@ class LibraryList(FileList): Returns: str: A joined list of link flags """ - return ' '.join(['-l' + name for name in self.names]) + return " ".join(["-l" + name for name in self.names]) @property def ld_flags(self): @@ -1871,7 +1878,7 @@ class LibraryList(FileList): Returns: str: A joined list of search flags and link flags """ - return self.search_flags + ' ' + self.link_flags + return self.search_flags + " " + self.link_flags def find_system_libraries(libraries, shared=True): @@ -1908,20 +1915,19 @@ def find_system_libraries(libraries, shared=True): if isinstance(libraries, six.string_types): libraries = [libraries] elif not isinstance(libraries, Sequence): - message = '{0} expects a string or sequence of strings as the ' - message += 'first argument [got {1} instead]' - message = message.format(find_system_libraries.__name__, - type(libraries)) + message = "{0} expects a string or sequence of strings as the " + message += "first argument [got {1} instead]" + message = message.format(find_system_libraries.__name__, type(libraries)) raise TypeError(message) libraries_found = [] search_locations = [ - '/lib64', - '/lib', - '/usr/lib64', - '/usr/lib', - '/usr/local/lib64', - '/usr/local/lib', + "/lib64", + "/lib", + "/usr/lib64", + "/usr/lib", + "/usr/local/lib64", + "/usr/local/lib", ] for library in libraries: @@ -1962,24 +1968,23 @@ def find_libraries(libraries, root, shared=True, recursive=False): if isinstance(libraries, six.string_types): libraries = [libraries] elif not isinstance(libraries, Sequence): - message = '{0} expects a string or sequence of strings as the ' - message += 'first argument [got {1} instead]' + message = "{0} expects a string or sequence of strings as the " + message += "first argument [got {1} instead]" message = message.format(find_libraries.__name__, type(libraries)) raise TypeError(message) # Construct the right suffix for the library if shared: # Used on both Linux and macOS - suffixes = ['so'] - if sys.platform == 'darwin': + suffixes = ["so"] + if sys.platform == "darwin": # Only used on macOS - suffixes.append('dylib') + suffixes.append("dylib") else: - suffixes = ['a'] + suffixes = ["a"] # List of libraries we are searching with suffixes - libraries = ['{0}.{1}'.format(lib, suffix) for lib in libraries - for suffix in suffixes] + libraries = ["{0}.{1}".format(lib, suffix) for lib in libraries for suffix in suffixes] if not recursive: # If not recursive, look for the libraries directly in root @@ -1989,7 +1994,7 @@ def find_libraries(libraries, root, shared=True, recursive=False): # perform first non-recursive search in root/lib then in root/lib64 and # finally search all of root recursively. The search stops when the first # match is found. - for subdir in ('lib', 'lib64'): + for subdir in ("lib", "lib64"): dirname = join_path(root, subdir) if not os.path.isdir(dirname): continue @@ -2045,10 +2050,11 @@ def files_in(*search_paths): """ files = [] for d in filter(can_access_dir, search_paths): - files.extend(filter( - lambda x: os.path.isfile(x[1]), - [(f, os.path.join(d, f)) for f in os.listdir(d)] - )) + files.extend( + filter( + lambda x: os.path.isfile(x[1]), [(f, os.path.join(d, f)) for f in os.listdir(d)] + ) + ) return files @@ -2078,7 +2084,7 @@ def search_paths_for_executables(*path_hints): path = os.path.abspath(path) executable_paths.append(path) - bin_dir = os.path.join(path, 'bin') + bin_dir = os.path.join(path, "bin") if os.path.isdir(bin_dir): executable_paths.append(bin_dir) @@ -2106,11 +2112,11 @@ def search_paths_for_libraries(*path_hints): path = os.path.abspath(path) library_paths.append(path) - lib_dir = os.path.join(path, 'lib') + lib_dir = os.path.join(path, "lib") if os.path.isdir(lib_dir): library_paths.append(lib_dir) - lib64_dir = os.path.join(path, 'lib64') + lib64_dir = os.path.join(path, "lib64") if os.path.isdir(lib64_dir): library_paths.append(lib64_dir) @@ -2140,13 +2146,13 @@ def partition_path(path, entry=None): # Handle drive letters e.g. C:/ on Windows entries[0] = entries[0] + sep i = entries.index(entry) - if '' in entries: + if "" in entries: i -= 1 - return paths[:i], paths[i], paths[i + 1:] + return paths[:i], paths[i], paths[i + 1 :] except ValueError: pass - return paths, '', [] + return paths, "", [] @system_path_filter @@ -2181,7 +2187,7 @@ def prefixes(path): elif parts[0].endswith(":"): # Handle drive letters e.g. C:/ on Windows parts[0] = parts[0] + sep - paths = [os.path.join(*parts[:i + 1]) for i in range(len(parts))] + paths = [os.path.join(*parts[: i + 1]) for i in range(len(parts))] try: paths.remove(sep) @@ -2189,7 +2195,7 @@ def prefixes(path): pass try: - paths.remove('.') + paths.remove(".") except ValueError: pass diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 463310b7a2..314566e97a 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -21,7 +21,7 @@ from six import string_types from llnl.util.compat import MutableMapping, MutableSequence, zip_longest # Ignore emacs backups when listing modules -ignore_modules = [r'^\.#', '~$'] +ignore_modules = [r"^\.#", "~$"] def index_by(objects, *funcs): @@ -91,9 +91,9 @@ def index_by(objects, *funcs): def caller_locals(): """This will return the locals of the *parent* of the caller. - This allows a function to insert variables into its caller's - scope. Yes, this is some black magic, and yes it's useful - for implementing things like depends_on and provides. + This allows a function to insert variables into its caller's + scope. Yes, this is some black magic, and yes it's useful + for implementing things like depends_on and provides. """ # Passing zero here skips line context for speed. stack = inspect.stack(0) @@ -105,7 +105,7 @@ def caller_locals(): def get_calling_module_name(): """Make sure that the caller is a class definition, and return the - enclosing module's name. + enclosing module's name. """ # Passing zero here skips line context for speed. stack = inspect.stack(0) @@ -115,12 +115,13 @@ def get_calling_module_name(): finally: del stack - if '__module__' not in caller_locals: - raise RuntimeError("Must invoke get_calling_module_name() " - "from inside a class definition!") + if "__module__" not in caller_locals: + raise RuntimeError( + "Must invoke get_calling_module_name() " "from inside a class definition!" + ) - module_name = caller_locals['__module__'] - base_name = module_name.split('.')[-1] + module_name = caller_locals["__module__"] + base_name = module_name.split(".")[-1] return base_name @@ -128,8 +129,8 @@ def attr_required(obj, attr_name): """Ensure that a class has a required attribute.""" if not hasattr(obj, attr_name): raise RequiredAttributeError( - "No required attribute '%s' in class '%s'" - % (attr_name, obj.__class__.__name__)) + "No required attribute '%s' in class '%s'" % (attr_name, obj.__class__.__name__) + ) def attr_setdefault(obj, name, value): @@ -201,33 +202,35 @@ def memoized(func): # TypeError is raised when indexing into a dict if the key is unhashable. raise six.raise_from( UnhashableArguments( - "args + kwargs '{}' was not hashable for function '{}'" - .format(key, func.__name__), + "args + kwargs '{}' was not hashable for function '{}'".format( + key, func.__name__ + ), ), - e) + e, + ) return _memoized_function def list_modules(directory, **kwargs): """Lists all of the modules, excluding ``__init__.py``, in a - particular directory. Listed packages have no particular - order.""" - list_directories = kwargs.setdefault('directories', True) + particular directory. Listed packages have no particular + order.""" + list_directories = kwargs.setdefault("directories", True) for name in os.listdir(directory): - if name == '__init__.py': + if name == "__init__.py": continue path = os.path.join(directory, name) if list_directories and os.path.isdir(path): - init_py = os.path.join(path, '__init__.py') + init_py = os.path.join(path, "__init__.py") if os.path.isfile(init_py): yield name - elif name.endswith('.py'): + elif name.endswith(".py"): if not any(re.search(pattern, name) for pattern in ignore_modules): - yield re.sub('.py$', '', name) + yield re.sub(".py$", "", name) def decorator_with_or_without_args(decorator): @@ -257,41 +260,34 @@ def decorator_with_or_without_args(decorator): def key_ordering(cls): """Decorates a class with extra methods that implement rich comparison - operations and ``__hash__``. The decorator assumes that the class - implements a function called ``_cmp_key()``. The rich comparison - operations will compare objects using this key, and the ``__hash__`` - function will return the hash of this key. + operations and ``__hash__``. The decorator assumes that the class + implements a function called ``_cmp_key()``. The rich comparison + operations will compare objects using this key, and the ``__hash__`` + function will return the hash of this key. - If a class already has ``__eq__``, ``__ne__``, ``__lt__``, ``__le__``, - ``__gt__``, or ``__ge__`` defined, this decorator will overwrite them. + If a class already has ``__eq__``, ``__ne__``, ``__lt__``, ``__le__``, + ``__gt__``, or ``__ge__`` defined, this decorator will overwrite them. - Raises: - TypeError: If the class does not have a ``_cmp_key`` method + Raises: + TypeError: If the class does not have a ``_cmp_key`` method """ + def setter(name, value): value.__name__ = name setattr(cls, name, value) - if not has_method(cls, '_cmp_key'): + if not has_method(cls, "_cmp_key"): raise TypeError("'%s' doesn't define _cmp_key()." % cls.__name__) - setter('__eq__', - lambda s, o: - (s is o) or (o is not None and s._cmp_key() == o._cmp_key())) - setter('__lt__', - lambda s, o: o is not None and s._cmp_key() < o._cmp_key()) - setter('__le__', - lambda s, o: o is not None and s._cmp_key() <= o._cmp_key()) + setter("__eq__", lambda s, o: (s is o) or (o is not None and s._cmp_key() == o._cmp_key())) + setter("__lt__", lambda s, o: o is not None and s._cmp_key() < o._cmp_key()) + setter("__le__", lambda s, o: o is not None and s._cmp_key() <= o._cmp_key()) - setter('__ne__', - lambda s, o: - (s is not o) and (o is None or s._cmp_key() != o._cmp_key())) - setter('__gt__', - lambda s, o: o is None or s._cmp_key() > o._cmp_key()) - setter('__ge__', - lambda s, o: o is None or s._cmp_key() >= o._cmp_key()) + setter("__ne__", lambda s, o: (s is not o) and (o is None or s._cmp_key() != o._cmp_key())) + setter("__gt__", lambda s, o: o is None or s._cmp_key() > o._cmp_key()) + setter("__ge__", lambda s, o: o is None or s._cmp_key() >= o._cmp_key()) - setter('__hash__', lambda self: hash(self._cmp_key())) + setter("__hash__", lambda self: hash(self._cmp_key())) return cls @@ -458,8 +454,7 @@ def lazy_lexicographic_ordering(cls, set_hash=True): def le(self, other): if self is other: return True - return (other is not None) and not lazy_lt(other._cmp_iter, - self._cmp_iter) + return (other is not None) and not lazy_lt(other._cmp_iter, self._cmp_iter) def ge(self, other): if self is other: @@ -489,7 +484,7 @@ def lazy_lexicographic_ordering(cls, set_hash=True): @lazy_lexicographic_ordering class HashableMap(MutableMapping): """This is a hashable, comparable dictionary. Hash is performed on - a tuple of the values in the dictionary.""" + a tuple of the values in the dictionary.""" def __init__(self): self.dict = {} @@ -527,7 +522,7 @@ class HashableMap(MutableMapping): def in_function(function_name): """True if the caller was called from some function with - the supplied Name, False otherwise.""" + the supplied Name, False otherwise.""" stack = inspect.stack() try: for elt in stack[2:]: @@ -540,24 +535,25 @@ def in_function(function_name): def check_kwargs(kwargs, fun): """Helper for making functions with kwargs. Checks whether the kwargs - are empty after all of them have been popped off. If they're - not, raises an error describing which kwargs are invalid. + are empty after all of them have been popped off. If they're + not, raises an error describing which kwargs are invalid. - Example:: + Example:: - def foo(self, **kwargs): - x = kwargs.pop('x', None) - y = kwargs.pop('y', None) - z = kwargs.pop('z', None) - check_kwargs(kwargs, self.foo) + def foo(self, **kwargs): + x = kwargs.pop('x', None) + y = kwargs.pop('y', None) + z = kwargs.pop('z', None) + check_kwargs(kwargs, self.foo) - # This raises a TypeError: - foo(w='bad kwarg') + # This raises a TypeError: + foo(w='bad kwarg') """ if kwargs: raise TypeError( "'%s' is an invalid keyword argument for function %s()." - % (next(iter(kwargs)), fun.__name__)) + % (next(iter(kwargs)), fun.__name__) + ) def match_predicate(*args): @@ -573,6 +569,7 @@ def match_predicate(*args): * any regex in a list or tuple of regexes matches. * any predicate in args matches. """ + def match(string): for arg in args: if isinstance(arg, string_types): @@ -585,9 +582,11 @@ def match_predicate(*args): if arg(string): return True else: - raise ValueError("args to match_predicate must be regex, " - "list of regexes, or callable.") + raise ValueError( + "args to match_predicate must be regex, " "list of regexes, or callable." + ) return False + return match @@ -647,7 +646,7 @@ def pretty_date(time, now=None): day_diff = diff.days if day_diff < 0: - return '' + return "" if day_diff == 0: if second_diff < 10: @@ -705,43 +704,40 @@ def pretty_string_to_date(date_str, now=None): now = now or datetime.now() # datetime formats - pattern[re.compile(r'^\d{4}$')] = lambda x: datetime.strptime(x, '%Y') - pattern[re.compile(r'^\d{4}-\d{2}$')] = lambda x: datetime.strptime( - x, '%Y-%m' + pattern[re.compile(r"^\d{4}$")] = lambda x: datetime.strptime(x, "%Y") + pattern[re.compile(r"^\d{4}-\d{2}$")] = lambda x: datetime.strptime(x, "%Y-%m") + pattern[re.compile(r"^\d{4}-\d{2}-\d{2}$")] = lambda x: datetime.strptime(x, "%Y-%m-%d") + pattern[re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}$")] = lambda x: datetime.strptime( + x, "%Y-%m-%d %H:%M" ) - pattern[re.compile(r'^\d{4}-\d{2}-\d{2}$')] = lambda x: datetime.strptime( - x, '%Y-%m-%d' + pattern[re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$")] = lambda x: datetime.strptime( + x, "%Y-%m-%d %H:%M:%S" ) - pattern[re.compile(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}$')] = \ - lambda x: datetime.strptime(x, '%Y-%m-%d %H:%M') - pattern[re.compile(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$')] = \ - lambda x: datetime.strptime(x, '%Y-%m-%d %H:%M:%S') - pretty_regex = re.compile( - r'(a|\d+)\s*(year|month|week|day|hour|minute|second)s?\s*ago') + pretty_regex = re.compile(r"(a|\d+)\s*(year|month|week|day|hour|minute|second)s?\s*ago") def _n_xxx_ago(x): how_many, time_period = pretty_regex.search(x).groups() - how_many = 1 if how_many == 'a' else int(how_many) + how_many = 1 if how_many == "a" else int(how_many) # timedelta natively supports time periods up to 'weeks'. # To apply month or year we convert to 30 and 365 days - if time_period == 'month': + if time_period == "month": how_many *= 30 - time_period = 'day' - elif time_period == 'year': + time_period = "day" + elif time_period == "year": how_many *= 365 - time_period = 'day' + time_period = "day" - kwargs = {(time_period + 's'): how_many} + kwargs = {(time_period + "s"): how_many} return now - timedelta(**kwargs) pattern[pretty_regex] = _n_xxx_ago # yesterday callback = lambda x: now - timedelta(days=1) - pattern[re.compile('^yesterday$')] = callback + pattern[re.compile("^yesterday$")] = callback for regexp, parser in pattern.items(): if bool(regexp.match(date_str)): @@ -752,7 +748,6 @@ def pretty_string_to_date(date_str, now=None): class RequiredAttributeError(ValueError): - def __init__(self, message): super(RequiredAttributeError, self).__init__(message) @@ -764,6 +759,7 @@ class ObjectWrapper(object): This class is modeled after the stackoverflow answer: * http://stackoverflow.com/a/1445289/771663 """ + def __init__(self, wrapped_object): wrapped_cls = type(wrapped_object) wrapped_name = wrapped_cls.__name__ @@ -807,7 +803,7 @@ class Singleton(object): # requested but not yet set. The final 'getattr' line here requires # 'instance'/'_instance' to be defined or it will enter an infinite # loop, so protect against that here. - if name in ['_instance', 'instance']: + if name in ["_instance", "instance"]: raise AttributeError() return getattr(self.instance, name) @@ -837,7 +833,7 @@ class LazyReference(object): self.ref_function = ref_function def __getattr__(self, name): - if name == 'ref_function': + if name == "ref_function": raise AttributeError() return getattr(self.ref_function(), name) @@ -875,8 +871,8 @@ def load_module_from_file(module_name, module_path): # This recipe is adapted from https://stackoverflow.com/a/67692/771663 if sys.version_info[0] == 3 and sys.version_info[1] >= 5: import importlib.util - spec = importlib.util.spec_from_file_location( # novm - module_name, module_path) + + spec = importlib.util.spec_from_file_location(module_name, module_path) # novm module = importlib.util.module_from_spec(spec) # novm # The module object needs to exist in sys.modules before the # loader executes the module code. @@ -893,6 +889,7 @@ def load_module_from_file(module_name, module_path): raise elif sys.version_info[0] == 2: import imp + module = imp.load_source(module_name, module_path) return module @@ -924,8 +921,10 @@ def uniq(sequence): def star(func): """Unpacks arguments for use with Multiprocessing mapping functions""" + def _wrapper(args): return func(*args) + return _wrapper @@ -934,22 +933,23 @@ class Devnull(object): See https://stackoverflow.com/a/2929954. """ + def write(self, *_): pass def elide_list(line_list, max_num=10): """Takes a long list and limits it to a smaller number of elements, - replacing intervening elements with '...'. For example:: + replacing intervening elements with '...'. For example:: - elide_list([1,2,3,4,5,6], 4) + elide_list([1,2,3,4,5,6], 4) - gives:: + gives:: - [1, 2, 3, '...', 6] + [1, 2, 3, '...', 6] """ if len(line_list) > max_num: - return line_list[:max_num - 1] + ['...'] + line_list[-1:] + return line_list[: max_num - 1] + ["..."] + line_list[-1:] else: return line_list @@ -972,7 +972,7 @@ def enum(**kwargs): Args: **kwargs: explicit dictionary of enums """ - return type('Enum', (object,), kwargs) + return type("Enum", (object,), kwargs) class TypedMutableSequence(MutableSequence): @@ -988,6 +988,7 @@ class TypedMutableSequence(MutableSequence): if isinstance(l, Foo): # do something """ + def __init__(self, iterable): self.data = list(iterable) @@ -1017,7 +1018,7 @@ class GroupedExceptionHandler(object): """A generic mechanism to coalesce multiple exceptions and preserve tracebacks.""" def __init__(self): - self.exceptions = [] # type: List[Tuple[str, Exception, List[str]]] + self.exceptions = [] # type: List[Tuple[str, Exception, List[str]]] def __bool__(self): """Whether any exceptions were handled.""" @@ -1036,17 +1037,15 @@ class GroupedExceptionHandler(object): # type: (bool) -> str """Print out an error message coalescing all the forwarded errors.""" each_exception_message = [ - '{0} raised {1}: {2}{3}'.format( + "{0} raised {1}: {2}{3}".format( context, exc.__class__.__name__, exc, - '\n{0}'.format(''.join(tb)) if with_tracebacks else '', + "\n{0}".format("".join(tb)) if with_tracebacks else "", ) for context, exc, tb in self.exceptions ] - return 'due to the following failures:\n{0}'.format( - '\n'.join(each_exception_message) - ) + return "due to the following failures:\n{0}".format("\n".join(each_exception_message)) class GroupedExceptionForwarder(object): @@ -1079,6 +1078,7 @@ class classproperty(object): the evaluation is injected at creation time and take an instance (could be None) and an owner (i.e. the class that originated the instance) """ + def __init__(self, callback): self.callback = callback diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py index 34cce1247c..947ca9c541 100644 --- a/lib/spack/llnl/util/link_tree.py +++ b/lib/spack/llnl/util/link_tree.py @@ -16,9 +16,9 @@ import llnl.util.tty as tty from llnl.util.filesystem import mkdirp, touch, traverse_tree from llnl.util.symlink import islink, symlink -__all__ = ['LinkTree'] +__all__ = ["LinkTree"] -empty_file_name = '.spack-empty' +empty_file_name = ".spack-empty" def remove_link(src, dest): @@ -38,6 +38,7 @@ class MergeConflict: project(src_a) == project(src_b) == dst """ + def __init__(self, dst, src_a=None, src_b=None): self.dst = dst self.src_a = src_a @@ -51,13 +52,14 @@ class SourceMergeVisitor(object): - A list of files to link in dst - A list of merge conflicts in dst/ """ + def __init__(self, ignore=None): self.ignore = ignore if ignore is not None else lambda f: False # When mapping <src root> to <dst root>/<projection>, we need # to prepend the <projection> bit to the relative path in the # destination dir. - self.projection = '' + self.projection = "" # When a file blocks another file, the conflict can sometimes # be resolved / ignored (e.g. <prefix>/LICENSE or @@ -88,10 +90,13 @@ class SourceMergeVisitor(object): elif proj_rel_path in self.files: # Can't create a dir where a file is. src_a_root, src_a_relpath = self.files[proj_rel_path] - self.fatal_conflicts.append(MergeConflict( - dst=proj_rel_path, - src_a=os.path.join(src_a_root, src_a_relpath), - src_b=os.path.join(root, rel_path))) + self.fatal_conflicts.append( + MergeConflict( + dst=proj_rel_path, + src_a=os.path.join(src_a_root, src_a_relpath), + src_b=os.path.join(root, rel_path), + ) + ) return False elif proj_rel_path in self.directories: # No new directory, carry on. @@ -147,17 +152,23 @@ class SourceMergeVisitor(object): elif proj_rel_path in self.directories: # Can't create a file where a dir is; fatal error src_a_root, src_a_relpath = self.directories[proj_rel_path] - self.fatal_conflicts.append(MergeConflict( - dst=proj_rel_path, - src_a=os.path.join(src_a_root, src_a_relpath), - src_b=os.path.join(root, rel_path))) + self.fatal_conflicts.append( + MergeConflict( + dst=proj_rel_path, + src_a=os.path.join(src_a_root, src_a_relpath), + src_b=os.path.join(root, rel_path), + ) + ) elif proj_rel_path in self.files: # In some cases we can resolve file-file conflicts src_a_root, src_a_relpath = self.files[proj_rel_path] - self.file_conflicts.append(MergeConflict( - dst=proj_rel_path, - src_a=os.path.join(src_a_root, src_a_relpath), - src_b=os.path.join(root, rel_path))) + self.file_conflicts.append( + MergeConflict( + dst=proj_rel_path, + src_a=os.path.join(src_a_root, src_a_relpath), + src_b=os.path.join(root, rel_path), + ) + ) else: # Otherwise register this file to be linked. self.files[proj_rel_path] = (root, rel_path) @@ -166,24 +177,27 @@ class SourceMergeVisitor(object): self.projection = os.path.normpath(projection) # Todo, is this how to check in general for empty projection? - if self.projection == '.': - self.projection = '' + if self.projection == ".": + self.projection = "" return # If there is a projection, we'll also create the directories # it consists of, and check whether that's causing conflicts. - path = '' + path = "" for part in self.projection.split(os.sep): path = os.path.join(path, part) if path not in self.files: - self.directories[path] = ('<projection>', path) + self.directories[path] = ("<projection>", path) else: # Can't create a dir where a file is. src_a_root, src_a_relpath = self.files[path] - self.fatal_conflicts.append(MergeConflict( - dst=path, - src_a=os.path.join(src_a_root, src_a_relpath), - src_b=os.path.join('<projection>', path))) + self.fatal_conflicts.append( + MergeConflict( + dst=path, + src_a=os.path.join(src_a_root, src_a_relpath), + src_b=os.path.join("<projection>", path), + ) + ) class DestinationMergeVisitor(object): @@ -200,6 +214,7 @@ class DestinationMergeVisitor(object): in the target prefix will never be merged with directories in the sources directories. """ + def __init__(self, source_merge_visitor): self.src = source_merge_visitor @@ -208,10 +223,11 @@ class DestinationMergeVisitor(object): # and don't traverse deeper if rel_path in self.src.files: src_a_root, src_a_relpath = self.src.files[rel_path] - self.src.fatal_conflicts.append(MergeConflict( - rel_path, - os.path.join(src_a_root, src_a_relpath), - os.path.join(root, rel_path))) + self.src.fatal_conflicts.append( + MergeConflict( + rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path) + ) + ) return False # If destination dir was also a src dir, remove the mkdir @@ -236,17 +252,19 @@ class DestinationMergeVisitor(object): # Always conflict if rel_path in self.src.directories: src_a_root, src_a_relpath = self.src.directories[rel_path] - self.src.fatal_conflicts.append(MergeConflict( - rel_path, - os.path.join(src_a_root, src_a_relpath), - os.path.join(root, rel_path))) + self.src.fatal_conflicts.append( + MergeConflict( + rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path) + ) + ) if rel_path in self.src.files: src_a_root, src_a_relpath = self.src.files[rel_path] - self.src.fatal_conflicts.append(MergeConflict( - rel_path, - os.path.join(src_a_root, src_a_relpath), - os.path.join(root, rel_path))) + self.src.fatal_conflicts.append( + MergeConflict( + rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path) + ) + ) # Never descend into symlinked target dirs. return False @@ -258,17 +276,19 @@ class DestinationMergeVisitor(object): # Can't merge a file if target already exists if rel_path in self.src.directories: src_a_root, src_a_relpath = self.src.directories[rel_path] - self.src.fatal_conflicts.append(MergeConflict( - rel_path, - os.path.join(src_a_root, src_a_relpath), - os.path.join(root, rel_path))) + self.src.fatal_conflicts.append( + MergeConflict( + rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path) + ) + ) elif rel_path in self.src.files: src_a_root, src_a_relpath = self.src.files[rel_path] - self.src.fatal_conflicts.append(MergeConflict( - rel_path, - os.path.join(src_a_root, src_a_relpath), - os.path.join(root, rel_path))) + self.src.fatal_conflicts.append( + MergeConflict( + rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path) + ) + ) class LinkTree(object): @@ -281,30 +301,31 @@ class LinkTree(object): symlinked to, to prevent the source directory from ever being modified. """ + def __init__(self, source_root): if not os.path.exists(source_root): raise IOError("No such file or directory: '%s'", source_root) self._root = source_root - def find_conflict(self, dest_root, ignore=None, - ignore_file_conflicts=False): + def find_conflict(self, dest_root, ignore=None, ignore_file_conflicts=False): """Returns the first file in dest that conflicts with src""" ignore = ignore or (lambda x: False) conflicts = self.find_dir_conflicts(dest_root, ignore) if not ignore_file_conflicts: conflicts.extend( - dst for src, dst - in self.get_file_map(dest_root, ignore).items() - if os.path.exists(dst)) + dst + for src, dst in self.get_file_map(dest_root, ignore).items() + if os.path.exists(dst) + ) if conflicts: return conflicts[0] def find_dir_conflicts(self, dest_root, ignore): conflicts = [] - kwargs = {'follow_nonexisting': False, 'ignore': ignore} + kwargs = {"follow_nonexisting": False, "ignore": ignore} for src, dest in traverse_tree(self._root, dest_root, **kwargs): if os.path.isdir(src): if os.path.exists(dest) and not os.path.isdir(dest): @@ -315,7 +336,7 @@ class LinkTree(object): def get_file_map(self, dest_root, ignore): merge_map = {} - kwargs = {'follow_nonexisting': True, 'ignore': ignore} + kwargs = {"follow_nonexisting": True, "ignore": ignore} for src, dest in traverse_tree(self._root, dest_root, **kwargs): if not os.path.isdir(src): merge_map[src] = dest @@ -337,8 +358,7 @@ class LinkTree(object): touch(marker) def unmerge_directories(self, dest_root, ignore): - for src, dest in traverse_tree( - self._root, dest_root, ignore=ignore, order='post'): + for src, dest in traverse_tree(self._root, dest_root, ignore=ignore, order="post"): if os.path.isdir(src): if not os.path.exists(dest): continue @@ -354,8 +374,7 @@ class LinkTree(object): if os.path.exists(marker): os.remove(marker) - def merge(self, dest_root, ignore_conflicts=False, ignore=None, - link=symlink, relative=False): + def merge(self, dest_root, ignore_conflicts=False, ignore=None, link=symlink, relative=False): """Link all files in src into dest, creating directories if necessary. @@ -377,7 +396,8 @@ class LinkTree(object): ignore = lambda x: False conflict = self.find_conflict( - dest_root, ignore=ignore, ignore_file_conflicts=ignore_conflicts) + dest_root, ignore=ignore, ignore_file_conflicts=ignore_conflicts + ) if conflict: raise SingleMergeConflictError(conflict) @@ -416,8 +436,7 @@ class MergeConflictError(Exception): class SingleMergeConflictError(MergeConflictError): def __init__(self, path): - super(MergeConflictError, self).__init__( - "Package merge blocked by file: %s" % path) + super(MergeConflictError, self).__init__("Package merge blocked by file: %s" % path) class MergeConflictSummary(MergeConflictError): @@ -430,5 +449,6 @@ class MergeConflictSummary(MergeConflictError): # show the first 3 merge conflicts. for conflict in conflicts[:3]: msg += "\n `{0}` and `{1}` both project to `{2}`".format( - conflict.src_a, conflict.src_b, conflict.dst) + conflict.src_a, conflict.src_b, conflict.dst + ) super(MergeConflictSummary, self).__init__(msg) diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py index 1ff7ceec64..0682ce059a 100644 --- a/lib/spack/llnl/util/lock.py +++ b/lib/spack/llnl/util/lock.py @@ -15,22 +15,22 @@ import llnl.util.tty as tty import spack.util.string -if sys.platform != 'win32': +if sys.platform != "win32": import fcntl __all__ = [ - 'Lock', - 'LockDowngradeError', - 'LockUpgradeError', - 'LockTransaction', - 'WriteTransaction', - 'ReadTransaction', - 'LockError', - 'LockTimeoutError', - 'LockPermissionError', - 'LockROFileError', - 'CantCreateLockError' + "Lock", + "LockDowngradeError", + "LockUpgradeError", + "LockTransaction", + "WriteTransaction", + "ReadTransaction", + "LockError", + "LockTimeoutError", + "LockPermissionError", + "LockROFileError", + "CantCreateLockError", ] @@ -47,6 +47,7 @@ class OpenFile(object): the file descriptor from the file handle if needed -- or we could make this track file descriptors as well in the future. """ + def __init__(self, fh): self.fh = fh self.refs = 0 @@ -92,11 +93,11 @@ class OpenFileTracker(object): path (str): path to lock file we want a filehandle for """ # Open writable files as 'r+' so we can upgrade to write later - os_mode, fh_mode = (os.O_RDWR | os.O_CREAT), 'r+' + os_mode, fh_mode = (os.O_RDWR | os.O_CREAT), "r+" pid = os.getpid() open_file = None # OpenFile object, if there is one - stat = None # stat result for the lockfile, if it exists + stat = None # stat result for the lockfile, if it exists try: # see whether we've seen this inode/pid before @@ -109,7 +110,7 @@ class OpenFileTracker(object): raise # path does not exist -- fail if we won't be able to create it - parent = os.path.dirname(path) or '.' + parent = os.path.dirname(path) or "." if not os.access(parent, os.W_OK): raise CantCreateLockError(path) @@ -119,7 +120,7 @@ class OpenFileTracker(object): # we know path exists but not if it's writable. If it's read-only, # only open the file for reading (and fail if we're trying to get # an exclusive (write) lock on it) - os_mode, fh_mode = os.O_RDONLY, 'r' + os_mode, fh_mode = os.O_RDONLY, "r" fd = os.open(path, os_mode) fh = os.fdopen(fd, fh_mode) @@ -162,10 +163,10 @@ file_tracker = OpenFileTracker() def _attempts_str(wait_time, nattempts): # Don't print anything if we succeeded on the first try if nattempts <= 1: - return '' + return "" - attempts = spack.util.string.plural(nattempts, 'attempt') - return ' after {0:0.2f}s and {1}'.format(wait_time, attempts) + attempts = spack.util.string.plural(nattempts, "attempt") + return " after {0:0.2f}s and {1}".format(wait_time, attempts) class LockType(object): @@ -188,8 +189,7 @@ class LockType(object): @staticmethod def is_valid(op): - return op == LockType.READ \ - or op == LockType.WRITE + return op == LockType.READ or op == LockType.WRITE class Lock(object): @@ -207,8 +207,7 @@ class Lock(object): overlapping byte ranges in the same file). """ - def __init__(self, path, start=0, length=0, default_timeout=None, - debug=False, desc=''): + def __init__(self, path, start=0, length=0, default_timeout=None, debug=False, desc=""): """Construct a new lock on the file at ``path``. By default, the lock applies to the whole file. Optionally, @@ -243,7 +242,7 @@ class Lock(object): self.debug = debug # optional debug description - self.desc = ' ({0})'.format(desc) if desc else '' + self.desc = " ({0})".format(desc) if desc else "" # If the user doesn't set a default timeout, or if they choose # None, 0, etc. then lock attempts will not time out (unless the @@ -280,17 +279,17 @@ class Lock(object): def __repr__(self): """Formal representation of the lock.""" - rep = '{0}('.format(self.__class__.__name__) + rep = "{0}(".format(self.__class__.__name__) for attr, value in self.__dict__.items(): - rep += '{0}={1}, '.format(attr, value.__repr__()) - return '{0})'.format(rep.strip(', ')) + rep += "{0}={1}, ".format(attr, value.__repr__()) + return "{0})".format(rep.strip(", ")) def __str__(self): """Readable string (with key fields) of the lock.""" - location = '{0}[{1}:{2}]'.format(self.path, self._start, self._length) - timeout = 'timeout={0}'.format(self.default_timeout) - activity = '#reads={0}, #writes={1}'.format(self._reads, self._writes) - return '({0}, {1}, {2})'.format(location, timeout, activity) + location = "{0}[{1}:{2}]".format(self.path, self._start, self._length) + timeout = "timeout={0}".format(self.default_timeout) + activity = "#reads={0}, #writes={1}".format(self._reads, self._writes) + return "({0}, {1}, {2})".format(location, timeout, activity) def _lock(self, op, timeout=None): """This takes a lock using POSIX locks (``fcntl.lockf``). @@ -305,7 +304,7 @@ class Lock(object): assert LockType.is_valid(op) op_str = LockType.to_str(op) - self._log_acquiring('{0} LOCK'.format(op_str)) + self._log_acquiring("{0} LOCK".format(op_str)) timeout = timeout or self.default_timeout # Create file and parent directories if they don't exist. @@ -313,14 +312,16 @@ class Lock(object): self._ensure_parent_directory() self._file = file_tracker.get_fh(self.path) - if LockType.to_module(op) == fcntl.LOCK_EX and self._file.mode == 'r': + if LockType.to_module(op) == fcntl.LOCK_EX and self._file.mode == "r": # Attempt to upgrade to write lock w/a read-only file. # If the file were writable, we'd have opened it 'r+' raise LockROFileError(self.path) - self._log_debug("{0} locking [{1}:{2}]: timeout {3} sec" - .format(op_str.lower(), self._start, self._length, - timeout)) + self._log_debug( + "{0} locking [{1}:{2}]: timeout {3} sec".format( + op_str.lower(), self._start, self._length, timeout + ) + ) poll_intervals = iter(Lock._poll_interval_generator()) start_time = time.time() @@ -339,8 +340,7 @@ class Lock(object): total_wait_time = time.time() - start_time return total_wait_time, num_attempts - raise LockTimeoutError("Timed out waiting for a {0} lock." - .format(op_str.lower())) + raise LockTimeoutError("Timed out waiting for a {0} lock.".format(op_str.lower())) def _poll_lock(self, op): """Attempt to acquire the lock in a non-blocking manner. Return whether @@ -349,16 +349,19 @@ class Lock(object): module_op = LockType.to_module(op) try: # Try to get the lock (will raise if not available.) - fcntl.lockf(self._file, module_op | fcntl.LOCK_NB, - self._length, self._start, os.SEEK_SET) + fcntl.lockf( + self._file, module_op | fcntl.LOCK_NB, self._length, self._start, os.SEEK_SET + ) # help for debugging distributed locking if self.debug: # All locks read the owner PID and host self._read_log_debug_data() - self._log_debug('{0} locked {1} [{2}:{3}] (owner={4})' - .format(LockType.to_str(op), self.path, - self._start, self._length, self.pid)) + self._log_debug( + "{0} locked {1} [{2}:{3}] (owner={4})".format( + LockType.to_str(op), self.path, self._start, self._length, self.pid + ) + ) # Exclusive locks write their PID/host if module_op == fcntl.LOCK_EX: @@ -378,14 +381,13 @@ class Lock(object): # relative paths to lockfiles in the current directory have no parent if not parent: - return '.' + return "." try: os.makedirs(parent) except OSError as e: # makedirs can fail when diretory already exists. - if not (e.errno == errno.EEXIST and os.path.isdir(parent) or - e.errno == errno.EISDIR): + if not (e.errno == errno.EEXIST and os.path.isdir(parent) or e.errno == errno.EISDIR): raise return parent @@ -396,9 +398,9 @@ class Lock(object): line = self._file.read() if line: - pid, host = line.strip().split(',') - _, _, self.pid = pid.rpartition('=') - _, _, self.host = host.rpartition('=') + pid, host = line.strip().split(",") + _, _, self.pid = pid.rpartition("=") + _, _, self.host = host.rpartition("=") self.pid = int(self.pid) def _write_log_debug_data(self): @@ -423,8 +425,7 @@ class Lock(object): be masquerading as write locks, but this removes either. """ - fcntl.lockf(self._file, fcntl.LOCK_UN, - self._length, self._start, os.SEEK_SET) + fcntl.lockf(self._file, fcntl.LOCK_UN, self._length, self._start, os.SEEK_SET) file_tracker.release_fh(self.path) self._file = None @@ -449,7 +450,7 @@ class Lock(object): wait_time, nattempts = self._lock(LockType.READ, timeout=timeout) self._reads += 1 # Log if acquired, which includes counts when verbose - self._log_acquired('READ LOCK', wait_time, nattempts) + self._log_acquired("READ LOCK", wait_time, nattempts) return True else: # Increment the read count for nested lock tracking @@ -474,7 +475,7 @@ class Lock(object): wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout) self._writes += 1 # Log if acquired, which includes counts when verbose - self._log_acquired('WRITE LOCK', wait_time, nattempts) + self._log_acquired("WRITE LOCK", wait_time, nattempts) # return True only if we weren't nested in a read lock. # TODO: we may need to return two values: whether we got @@ -561,7 +562,7 @@ class Lock(object): """ assert self._reads > 0 - locktype = 'READ LOCK' + locktype = "READ LOCK" if self._reads == 1 and self._writes == 0: self._log_releasing(locktype) @@ -569,7 +570,7 @@ class Lock(object): release_fn = release_fn or true_fn result = release_fn() - self._unlock() # can raise LockError. + self._unlock() # can raise LockError. self._reads = 0 self._log_released(locktype) return result @@ -597,14 +598,14 @@ class Lock(object): assert self._writes > 0 release_fn = release_fn or true_fn - locktype = 'WRITE LOCK' + locktype = "WRITE LOCK" if self._writes == 1 and self._reads == 0: self._log_releasing(locktype) # we need to call release_fn before releasing the lock result = release_fn() - self._unlock() # can raise LockError. + self._unlock() # can raise LockError. self._writes = 0 self._log_released(locktype) return result @@ -625,56 +626,55 @@ class Lock(object): raise LockError("Attempting to cleanup active lock.") def _get_counts_desc(self): - return '(reads {0}, writes {1})'.format(self._reads, self._writes) \ - if tty.is_verbose() else '' + return ( + "(reads {0}, writes {1})".format(self._reads, self._writes) if tty.is_verbose() else "" + ) def _log_acquired(self, locktype, wait_time, nattempts): attempts_part = _attempts_str(wait_time, nattempts) now = datetime.now() - desc = 'Acquired at %s' % now.strftime("%H:%M:%S.%f") - self._log_debug(self._status_msg(locktype, '{0}{1}' - .format(desc, attempts_part))) + desc = "Acquired at %s" % now.strftime("%H:%M:%S.%f") + self._log_debug(self._status_msg(locktype, "{0}{1}".format(desc, attempts_part))) def _log_acquiring(self, locktype): - self._log_debug(self._status_msg(locktype, 'Acquiring'), level=3) + self._log_debug(self._status_msg(locktype, "Acquiring"), level=3) def _log_debug(self, *args, **kwargs): """Output lock debug messages.""" - kwargs['level'] = kwargs.get('level', 2) + kwargs["level"] = kwargs.get("level", 2) tty.debug(*args, **kwargs) def _log_downgraded(self, wait_time, nattempts): attempts_part = _attempts_str(wait_time, nattempts) now = datetime.now() - desc = 'Downgraded at %s' % now.strftime("%H:%M:%S.%f") - self._log_debug(self._status_msg('READ LOCK', '{0}{1}' - .format(desc, attempts_part))) + desc = "Downgraded at %s" % now.strftime("%H:%M:%S.%f") + self._log_debug(self._status_msg("READ LOCK", "{0}{1}".format(desc, attempts_part))) def _log_downgrading(self): - self._log_debug(self._status_msg('WRITE LOCK', 'Downgrading'), level=3) + self._log_debug(self._status_msg("WRITE LOCK", "Downgrading"), level=3) def _log_released(self, locktype): now = datetime.now() - desc = 'Released at %s' % now.strftime("%H:%M:%S.%f") + desc = "Released at %s" % now.strftime("%H:%M:%S.%f") self._log_debug(self._status_msg(locktype, desc)) def _log_releasing(self, locktype): - self._log_debug(self._status_msg(locktype, 'Releasing'), level=3) + self._log_debug(self._status_msg(locktype, "Releasing"), level=3) def _log_upgraded(self, wait_time, nattempts): attempts_part = _attempts_str(wait_time, nattempts) now = datetime.now() - desc = 'Upgraded at %s' % now.strftime("%H:%M:%S.%f") - self._log_debug(self._status_msg('WRITE LOCK', '{0}{1}'. - format(desc, attempts_part))) + desc = "Upgraded at %s" % now.strftime("%H:%M:%S.%f") + self._log_debug(self._status_msg("WRITE LOCK", "{0}{1}".format(desc, attempts_part))) def _log_upgrading(self): - self._log_debug(self._status_msg('READ LOCK', 'Upgrading'), level=3) + self._log_debug(self._status_msg("READ LOCK", "Upgrading"), level=3) def _status_msg(self, locktype, status): - status_desc = '[{0}] {1}'.format(status, self._get_counts_desc()) - return '{0}{1.desc}: {1.path}[{1._start}:{1._length}] {2}'.format( - locktype, self, status_desc) + status_desc = "[{0}] {1}".format(status, self._get_counts_desc()) + return "{0}{1.desc}: {1.path}[{1._start}:{1._length}] {2}".format( + locktype, self, status_desc + ) class LockTransaction(object): @@ -715,7 +715,7 @@ class LockTransaction(object): def __enter__(self): if self._enter() and self._acquire_fn: self._as = self._acquire_fn() - if hasattr(self._as, '__enter__'): + if hasattr(self._as, "__enter__"): return self._as.__enter__() else: return self._as @@ -727,7 +727,7 @@ class LockTransaction(object): if self._release_fn is not None: return self._release_fn(type, value, traceback) - if self._as and hasattr(self._as, '__exit__'): + if self._as and hasattr(self._as, "__exit__"): if self._as.__exit__(type, value, traceback): suppress = True @@ -739,6 +739,7 @@ class LockTransaction(object): class ReadTransaction(LockTransaction): """LockTransaction context manager that does a read and releases it.""" + def _enter(self): return self._lock.acquire_read(self._timeout) @@ -748,6 +749,7 @@ class ReadTransaction(LockTransaction): class WriteTransaction(LockTransaction): """LockTransaction context manager that does a write and releases it.""" + def _enter(self): return self._lock.acquire_write(self._timeout) @@ -761,6 +763,7 @@ class LockError(Exception): class LockDowngradeError(LockError): """Raised when unable to downgrade from a write to a read lock.""" + def __init__(self, path): msg = "Cannot downgrade lock from write to read on file: %s" % path super(LockDowngradeError, self).__init__(msg) @@ -776,6 +779,7 @@ class LockTimeoutError(LockError): class LockUpgradeError(LockError): """Raised when unable to upgrade from a read to a write lock.""" + def __init__(self, path): msg = "Cannot upgrade lock from read to write on file: %s" % path super(LockUpgradeError, self).__init__(msg) @@ -787,6 +791,7 @@ class LockPermissionError(LockError): class LockROFileError(LockPermissionError): """Tried to take an exclusive lock on a read-only file.""" + def __init__(self, path): msg = "Can't take write lock on read-only file: %s" % path super(LockROFileError, self).__init__(msg) @@ -794,6 +799,7 @@ class LockROFileError(LockPermissionError): class CantCreateLockError(LockPermissionError): """Attempt to create a lock in an unwritable location.""" + def __init__(self, path): msg = "cannot create lock '%s': " % path msg += "file does not exist and location is not writable" diff --git a/lib/spack/llnl/util/multiproc.py b/lib/spack/llnl/util/multiproc.py index 86b9e81bcc..e6a0091191 100644 --- a/lib/spack/llnl/util/multiproc.py +++ b/lib/spack/llnl/util/multiproc.py @@ -10,7 +10,7 @@ to pickle functions if they're passed indirectly as parameters. """ from multiprocessing import Semaphore, Value -__all__ = ['Barrier'] +__all__ = ["Barrier"] class Barrier: @@ -24,7 +24,7 @@ class Barrier: def __init__(self, n, timeout=None): self.n = n self.to = timeout - self.count = Value('i', 0) + self.count = Value("i", 0) self.mutex = Semaphore(1) self.turnstile1 = Semaphore(0) self.turnstile2 = Semaphore(1) diff --git a/lib/spack/llnl/util/symlink.py b/lib/spack/llnl/util/symlink.py index 6c55d74f66..103c5b4c38 100644 --- a/lib/spack/llnl/util/symlink.py +++ b/lib/spack/llnl/util/symlink.py @@ -11,7 +11,7 @@ from sys import platform as _platform from llnl.util import lang -is_windows = _platform == 'win32' +is_windows = _platform == "win32" if is_windows: from win32file import CreateHardLink @@ -47,7 +47,7 @@ def _win32_junction(path, link): # os.symlink will fail if link exists, emulate the behavior here if exists(link): - raise OSError(errno.EEXIST, 'File exists: %s -> %s' % (link, path)) + raise OSError(errno.EEXIST, "File exists: %s -> %s" % (link, path)) if not os.path.isabs(path): parent = os.path.join(link, os.pardir) @@ -61,13 +61,14 @@ def _win32_junction(path, link): def _win32_can_symlink(): tempdir = tempfile.mkdtemp() - dpath = join(tempdir, 'dpath') - fpath = join(tempdir, 'fpath.txt') + dpath = join(tempdir, "dpath") + fpath = join(tempdir, "fpath.txt") - dlink = join(tempdir, 'dlink') - flink = join(tempdir, 'flink.txt') + dlink = join(tempdir, "dlink") + flink = join(tempdir, "flink.txt") import llnl.util.filesystem as fs + fs.touchp(fpath) try: @@ -106,7 +107,6 @@ def _win32_is_junction(path): FILE_ATTRIBUTE_REPARSE_POINT = 0x400 res = GetFileAttributes(path) - return res != INVALID_FILE_ATTRIBUTES and \ - bool(res & FILE_ATTRIBUTE_REPARSE_POINT) + return res != INVALID_FILE_ATTRIBUTES and bool(res & FILE_ATTRIBUTE_REPARSE_POINT) return False diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py index de49f3f77f..ed847298ef 100644 --- a/lib/spack/llnl/util/tty/__init__.py +++ b/lib/spack/llnl/util/tty/__init__.py @@ -54,7 +54,7 @@ def is_stacktrace(): def set_debug(level=0): global _debug - assert level >= 0, 'Debug level must be a positive value' + assert level >= 0, "Debug level must be a positive value" _debug = level @@ -110,10 +110,7 @@ def output_filter(filter_fn): class SuppressOutput: """Class for disabling output in a scope using 'with' keyword""" - def __init__(self, - msg_enabled=True, - warn_enabled=True, - error_enabled=True): + def __init__(self, msg_enabled=True, warn_enabled=True, error_enabled=True): self._msg_enabled_initial = _msg_enabled self._warn_enabled_initial = _warn_enabled @@ -164,11 +161,10 @@ def get_timestamp(force=False): """Get a string timestamp""" if _debug or _timestamp or force: # Note inclusion of the PID is useful for parallel builds. - pid = ', {0}'.format(os.getpid()) if show_pid() else '' - return '[{0}{1}] '.format( - datetime.now().strftime("%Y-%m-%d-%H:%M:%S.%f"), pid) + pid = ", {0}".format(os.getpid()) if show_pid() else "" + return "[{0}{1}] ".format(datetime.now().strftime("%Y-%m-%d-%H:%M:%S.%f"), pid) else: - return '' + return "" def msg(message, *args, **kwargs): @@ -178,26 +174,14 @@ def msg(message, *args, **kwargs): if isinstance(message, Exception): message = "%s: %s" % (message.__class__.__name__, str(message)) - newline = kwargs.get('newline', True) + newline = kwargs.get("newline", True) st_text = "" if _stacktrace: st_text = process_stacktrace(2) if newline: - cprint( - "@*b{%s==>} %s%s" % ( - st_text, - get_timestamp(), - cescape(_output_filter(message)) - ) - ) + cprint("@*b{%s==>} %s%s" % (st_text, get_timestamp(), cescape(_output_filter(message)))) else: - cwrite( - "@*b{%s==>} %s%s" % ( - st_text, - get_timestamp(), - cescape(_output_filter(message)) - ) - ) + cwrite("@*b{%s==>} %s%s" % (st_text, get_timestamp(), cescape(_output_filter(message)))) for arg in args: print(indent + _output_filter(six.text_type(arg))) @@ -206,23 +190,19 @@ def info(message, *args, **kwargs): if isinstance(message, Exception): message = "%s: %s" % (message.__class__.__name__, str(message)) - format = kwargs.get('format', '*b') - stream = kwargs.get('stream', sys.stdout) - wrap = kwargs.get('wrap', False) - break_long_words = kwargs.get('break_long_words', False) - st_countback = kwargs.get('countback', 3) + format = kwargs.get("format", "*b") + stream = kwargs.get("stream", sys.stdout) + wrap = kwargs.get("wrap", False) + break_long_words = kwargs.get("break_long_words", False) + st_countback = kwargs.get("countback", 3) st_text = "" if _stacktrace: st_text = process_stacktrace(st_countback) cprint( - "@%s{%s==>} %s%s" % ( - format, - st_text, - get_timestamp(), - cescape(_output_filter(six.text_type(message))) - ), - stream=stream + "@%s{%s==>} %s%s" + % (format, st_text, get_timestamp(), cescape(_output_filter(six.text_type(message)))), + stream=stream, ) for arg in args: if wrap: @@ -230,27 +210,25 @@ def info(message, *args, **kwargs): _output_filter(six.text_type(arg)), initial_indent=indent, subsequent_indent=indent, - break_long_words=break_long_words + break_long_words=break_long_words, ) for line in lines: - stream.write(line + '\n') + stream.write(line + "\n") else: - stream.write( - indent + _output_filter(six.text_type(arg)) + '\n' - ) + stream.write(indent + _output_filter(six.text_type(arg)) + "\n") def verbose(message, *args, **kwargs): if _verbose: - kwargs.setdefault('format', 'c') + kwargs.setdefault("format", "c") info(message, *args, **kwargs) def debug(message, *args, **kwargs): - level = kwargs.get('level', 1) + level = kwargs.get("level", 1) if is_debug(level): - kwargs.setdefault('format', 'g') - kwargs.setdefault('stream', sys.stderr) + kwargs.setdefault("format", "g") + kwargs.setdefault("stream", sys.stderr) info(message, *args, **kwargs) @@ -258,8 +236,8 @@ def error(message, *args, **kwargs): if not error_enabled(): return - kwargs.setdefault('format', '*r') - kwargs.setdefault('stream', sys.stderr) + kwargs.setdefault("format", "*r") + kwargs.setdefault("stream", sys.stderr) info("Error: " + six.text_type(message), *args, **kwargs) @@ -267,27 +245,27 @@ def warn(message, *args, **kwargs): if not warn_enabled(): return - kwargs.setdefault('format', '*Y') - kwargs.setdefault('stream', sys.stderr) + kwargs.setdefault("format", "*Y") + kwargs.setdefault("stream", sys.stderr) info("Warning: " + six.text_type(message), *args, **kwargs) def die(message, *args, **kwargs): - kwargs.setdefault('countback', 4) + kwargs.setdefault("countback", 4) error(message, *args, **kwargs) sys.exit(1) def get_number(prompt, **kwargs): - default = kwargs.get('default', None) - abort = kwargs.get('abort', None) + default = kwargs.get("default", None) + abort = kwargs.get("abort", None) if default is not None and abort is not None: - prompt += ' (default is %s, %s to abort) ' % (default, abort) + prompt += " (default is %s, %s to abort) " % (default, abort) elif default is not None: - prompt += ' (default is %s) ' % default + prompt += " (default is %s) " % default elif abort is not None: - prompt += ' (%s to abort) ' % abort + prompt += " (%s to abort) " % abort number = None while number is None: @@ -310,17 +288,16 @@ def get_number(prompt, **kwargs): def get_yes_or_no(prompt, **kwargs): - default_value = kwargs.get('default', None) + default_value = kwargs.get("default", None) if default_value is None: - prompt += ' [y/n] ' + prompt += " [y/n] " elif default_value is True: - prompt += ' [Y/n] ' + prompt += " [Y/n] " elif default_value is False: - prompt += ' [y/N] ' + prompt += " [y/N] " else: - raise ValueError( - "default for get_yes_no() must be True, False, or None.") + raise ValueError("default for get_yes_no() must be True, False, or None.") result = None while result is None: @@ -331,9 +308,9 @@ def get_yes_or_no(prompt, **kwargs): if result is None: print("Please enter yes or no.") else: - if ans == 'y' or ans == 'yes': + if ans == "y" or ans == "yes": result = True - elif ans == 'n' or ans == 'no': + elif ans == "n" or ans == "no": result = False return result @@ -345,12 +322,12 @@ def hline(label=None, **kwargs): char (str): Char to draw the line with. Default '-' max_width (int): Maximum width of the line. Default is 64 chars. """ - char = kwargs.pop('char', '-') - max_width = kwargs.pop('max_width', 64) + char = kwargs.pop("char", "-") + max_width = kwargs.pop("max_width", 64) if kwargs: raise TypeError( - "'%s' is an invalid keyword argument for this function." - % next(kwargs.iterkeys())) + "'%s' is an invalid keyword argument for this function." % next(kwargs.iterkeys()) + ) rows, cols = terminal_size() if not cols: @@ -374,13 +351,14 @@ def hline(label=None, **kwargs): def terminal_size(): """Gets the dimensions of the console: (rows, cols).""" if _platform != "win32": + def ioctl_gwinsz(fd): try: - rc = struct.unpack('hh', fcntl.ioctl( - fd, termios.TIOCGWINSZ, '1234')) + rc = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234")) except BaseException: return return rc + rc = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2) if not rc: try: @@ -390,12 +368,14 @@ def terminal_size(): except BaseException: pass if not rc: - rc = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) + rc = (os.environ.get("LINES", 25), os.environ.get("COLUMNS", 80)) return int(rc[0]), int(rc[1]) else: if sys.version_info[0] < 3: - raise RuntimeError("Terminal size not obtainable on Windows with a\ -Python version older than 3") - rc = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) + raise RuntimeError( + "Terminal size not obtainable on Windows with a\ +Python version older than 3" + ) + rc = (os.environ.get("LINES", 25), os.environ.get("COLUMNS", 80)) return int(rc[0]), int(rc[1]) diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py index 65e56b5473..73c1daf0d5 100644 --- a/lib/spack/llnl/util/tty/colify.py +++ b/lib/spack/llnl/util/tty/colify.py @@ -18,29 +18,27 @@ from llnl.util.tty.color import cextra, clen class ColumnConfig: - def __init__(self, cols): self.cols = cols self.line_length = 0 self.valid = True - self.widths = [0] * cols # does not include ansi colors + self.widths = [0] * cols # does not include ansi colors def __repr__(self): - attrs = [(a, getattr(self, a)) - for a in dir(self) if not a.startswith("__")] + attrs = [(a, getattr(self, a)) for a in dir(self) if not a.startswith("__")] return "<Config: %s>" % ", ".join("%s: %r" % a for a in attrs) def config_variable_cols(elts, console_width, padding, cols=0): """Variable-width column fitting algorithm. - This function determines the most columns that can fit in the - screen width. Unlike uniform fitting, where all columns take - the width of the longest element in the list, each column takes - the width of its own longest element. This packs elements more - efficiently on screen. + This function determines the most columns that can fit in the + screen width. Unlike uniform fitting, where all columns take + the width of the longest element in the list, each column takes + the width of its own longest element. This packs elements more + efficiently on screen. - If cols is nonzero, force + If cols is nonzero, force """ if cols < 0: raise ValueError("cols must be non-negative.") @@ -64,8 +62,8 @@ def config_variable_cols(elts, console_width, padding, cols=0): if conf.widths[col] < (length + p): conf.line_length += length + p - conf.widths[col] - conf.widths[col] = length + p - conf.valid = (conf.line_length < console_width) + conf.widths[col] = length + p + conf.valid = conf.line_length < console_width try: config = next(conf for conf in reversed(configs) if conf.valid) @@ -81,9 +79,9 @@ def config_variable_cols(elts, console_width, padding, cols=0): def config_uniform_cols(elts, console_width, padding, cols=0): """Uniform-width column fitting algorithm. - Determines the longest element in the list, and determines how - many columns of that width will fit on screen. Returns a - corresponding column config. + Determines the longest element in the list, and determines how + many columns of that width will fit on screen. Returns a + corresponding column config. """ if cols < 0: raise ValueError("cols must be non-negative.") @@ -122,18 +120,18 @@ def colify(elts, **options): and fit less data on the screen """ # Get keyword arguments or set defaults - cols = options.pop("cols", 0) - output = options.pop("output", sys.stdout) - indent = options.pop("indent", 0) - padding = options.pop("padding", 2) - tty = options.pop('tty', None) - method = options.pop("method", "variable") + cols = options.pop("cols", 0) + output = options.pop("output", sys.stdout) + indent = options.pop("indent", 0) + padding = options.pop("padding", 2) + tty = options.pop("tty", None) + method = options.pop("method", "variable") console_cols = options.pop("width", None) if options: raise TypeError( - "'%s' is an invalid keyword argument for this function." - % next(options.iterkeys())) + "'%s' is an invalid keyword argument for this function." % next(options.iterkeys()) + ) # elts needs to be an array of strings so we can count the elements elts = [text_type(elt) for elt in elts] @@ -141,10 +139,10 @@ def colify(elts, **options): return (0, ()) # environment size is of the form "<rows>x<cols>" - env_size = os.environ.get('COLIFY_SIZE') + env_size = os.environ.get("COLIFY_SIZE") if env_size: try: - r, c = env_size.split('x') + r, c = env_size.split("x") console_rows, console_cols = int(r), int(c) tty = True except BaseException: @@ -180,7 +178,7 @@ def colify(elts, **options): elt = col * rows + row width = config.widths[col] + cextra(elts[elt]) if col < cols - 1: - fmt = '%%-%ds' % width + fmt = "%%-%ds" % width output.write(fmt % elts[elt]) else: # Don't pad the rightmost column (sapces can wrap on @@ -198,15 +196,15 @@ def colify(elts, **options): def colify_table(table, **options): """Version of ``colify()`` for data expressed in rows, (list of lists). - Same as regular colify but: + Same as regular colify but: - 1. This takes a list of lists, where each sub-list must be the - same length, and each is interpreted as a row in a table. - Regular colify displays a sequential list of values in columns. + 1. This takes a list of lists, where each sub-list must be the + same length, and each is interpreted as a row in a table. + Regular colify displays a sequential list of values in columns. - 2. Regular colify will always print with 1 column when the output - is not a tty. This will always print with same dimensions of - the table argument. + 2. Regular colify will always print with 1 column when the output + is not a tty. This will always print with same dimensions of + the table argument. """ if table is None: @@ -221,20 +219,20 @@ def colify_table(table, **options): for row in table: yield row[i] - if 'cols' in options: + if "cols" in options: raise ValueError("Cannot override columsn in colify_table.") - options['cols'] = columns + options["cols"] = columns # don't reduce to 1 column for non-tty - options['tty'] = True + options["tty"] = True colify(transpose(), **options) def colified(elts, **options): """Invokes the ``colify()`` function but returns the result as a string - instead of writing it to an output string.""" + instead of writing it to an output string.""" sio = StringIO() - options['output'] = sio + options["output"] = sio colify(elts, **options) return sio.getvalue() diff --git a/lib/spack/llnl/util/tty/color.py b/lib/spack/llnl/util/tty/color.py index 99c0a5c7ac..1bc80331d4 100644 --- a/lib/spack/llnl/util/tty/color.py +++ b/lib/spack/llnl/util/tty/color.py @@ -76,29 +76,33 @@ class ColorParseError(Exception): # Text styles for ansi codes -styles = {'*': '1', # bold - '_': '4', # underline - None: '0'} # plain +styles = {"*": "1", "_": "4", None: "0"} # bold # underline # plain # Dim and bright ansi colors -colors = {'k': 30, 'K': 90, # black - 'r': 31, 'R': 91, # red - 'g': 32, 'G': 92, # green - 'y': 33, 'Y': 93, # yellow - 'b': 34, 'B': 94, # blue - 'm': 35, 'M': 95, # magenta - 'c': 36, 'C': 96, # cyan - 'w': 37, 'W': 97} # white +colors = { + "k": 30, + "K": 90, # black + "r": 31, + "R": 91, # red + "g": 32, + "G": 92, # green + "y": 33, + "Y": 93, # yellow + "b": 34, + "B": 94, # blue + "m": 35, + "M": 95, # magenta + "c": 36, + "C": 96, # cyan + "w": 37, + "W": 97, +} # white # Regex to be used for color formatting -color_re = r'@(?:@|\.|([*_])?([a-zA-Z])?(?:{((?:[^}]|}})*)})?)' +color_re = r"@(?:@|\.|([*_])?([a-zA-Z])?(?:{((?:[^}]|}})*)})?)" # Mapping from color arguments to values for tty.set_color -color_when_values = { - 'always': True, - 'auto': None, - 'never': False -} +color_when_values = {"always": True, "auto": None, "never": False} # Force color; None: Only color if stdout is a tty # True: Always colorize output, False: Never colorize output @@ -114,7 +118,7 @@ def _color_when_value(when): if when in color_when_values: return color_when_values[when] elif when not in color_when_values.values(): - raise ValueError('Invalid color setting: %s' % when) + raise ValueError("Invalid color setting: %s" % when) return when @@ -146,7 +150,6 @@ def color_when(value): class match_to_ansi(object): - def __init__(self, color=True): self.color = _color_when_value(color) @@ -155,7 +158,7 @@ class match_to_ansi(object): if self.color: return "\033[%sm" % s else: - return '' + return "" def __call__(self, match): """Convert a match object generated by ``color_re`` into an ansi @@ -164,22 +167,22 @@ class match_to_ansi(object): style, color, text = match.groups() m = match.group(0) - if m == '@@': - return '@' - elif m == '@.': + if m == "@@": + return "@" + elif m == "@.": return self.escape(0) - elif m == '@': - raise ColorParseError("Incomplete color format: '%s' in %s" - % (m, match.string)) + elif m == "@": + raise ColorParseError("Incomplete color format: '%s' in %s" % (m, match.string)) string = styles[style] if color: if color not in colors: - raise ColorParseError("Invalid color specifier: '%s' in '%s'" - % (color, match.string)) - string += ';' + str(colors[color]) + raise ColorParseError( + "Invalid color specifier: '%s' in '%s'" % (color, match.string) + ) + string += ";" + str(colors[color]) - colored_text = '' + colored_text = "" if text: colored_text = text + self.escape(0) @@ -199,28 +202,28 @@ def colorize(string, **kwargs): color (bool): If False, output will be plain text without control codes, for output to non-console devices. """ - color = _color_when_value(kwargs.get('color', get_color_when())) + color = _color_when_value(kwargs.get("color", get_color_when())) string = re.sub(color_re, match_to_ansi(color), string) - string = string.replace('}}', '}') + string = string.replace("}}", "}") return string def clen(string): """Return the length of a string, excluding ansi color sequences.""" - return len(re.sub(r'\033[^m]*m', '', string)) + return len(re.sub(r"\033[^m]*m", "", string)) def cextra(string): """Length of extra color characters in a string""" - return len(''.join(re.findall(r'\033[^m]*m', string))) + return len("".join(re.findall(r"\033[^m]*m", string))) def cwrite(string, stream=None, color=None): """Replace all color expressions in string with ANSI control - codes and write the result to the stream. If color is - False, this will write plain text with no color. If True, - then it will always write colored output. If not supplied, - then it will be set based on stream.isatty(). + codes and write the result to the stream. If color is + False, this will write plain text with no color. If True, + then it will always write colored output. If not supplied, + then it will be set based on stream.isatty(). """ stream = sys.stdout if stream is None else stream if color is None: @@ -251,20 +254,19 @@ def cescape(string): (str): the string with color codes escaped """ string = six.text_type(string) - string = string.replace('@', '@@') - string = string.replace('}', '}}') + string = string.replace("@", "@@") + string = string.replace("}", "}}") return string class ColorStream(object): - def __init__(self, stream, color=None): self._stream = stream self._color = color def write(self, string, **kwargs): - raw = kwargs.get('raw', False) - raw_write = getattr(self._stream, 'write') + raw = kwargs.get("raw", False) + raw_write = getattr(self._stream, "write") color = self._color if self._color is None: @@ -275,6 +277,6 @@ class ColorStream(object): raw_write(colorize(string, color=color)) def writelines(self, sequence, **kwargs): - raw = kwargs.get('raw', False) + raw = kwargs.get("raw", False) for string in sequence: self.write(string, self.color, raw=raw) diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index 51b9caa332..e155fa1d26 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -31,21 +31,22 @@ import llnl.util.tty as tty termios = None # type: Optional[ModuleType] try: import termios as term_mod + termios = term_mod except ImportError: pass # Use this to strip escape sequences -_escape = re.compile(r'\x1b[^m]*m|\x1b\[?1034h|\x1b\][0-9]+;[^\x07]*\x07') +_escape = re.compile(r"\x1b[^m]*m|\x1b\[?1034h|\x1b\][0-9]+;[^\x07]*\x07") # control characters for enabling/disabling echo # # We use control characters to ensure that echo enable/disable are inline # with the other output. We always follow these with a newline to ensure # one per line the following newline is ignored in output. -xon, xoff = '\x11\n', '\x13\n' -control = re.compile('(\x11\n|\x13\n)') +xon, xoff = "\x11\n", "\x13\n" +control = re.compile("(\x11\n|\x13\n)") @contextmanager @@ -59,17 +60,13 @@ def ignore_signal(signum): def _is_background_tty(stream): - """True if the stream is a tty and calling process is in the background. - """ - return ( - stream.isatty() and - os.getpgrp() != os.tcgetpgrp(stream.fileno()) - ) + """True if the stream is a tty and calling process is in the background.""" + return stream.isatty() and os.getpgrp() != os.tcgetpgrp(stream.fileno()) def _strip(line): """Strip color and control characters from a line.""" - return _escape.sub('', line) + return _escape.sub("", line) class keyboard_input(object): @@ -147,6 +144,7 @@ class keyboard_input(object): a TTY, ``keyboard_input`` has no effect. """ + def __init__(self, stream): """Create a context manager that will enable keyboard input on stream. @@ -204,7 +202,7 @@ class keyboard_input(object): bg = self._is_background() # restore sanity if flags are amiss -- see diagram in class docs - if not bg and any(flags): # fg, but input not enabled + if not bg and any(flags): # fg, but input not enabled self._enable_keyboard_input() elif bg and not all(flags): # bg, but input enabled self._restore_default_terminal_settings() @@ -228,8 +226,7 @@ class keyboard_input(object): # Install a signal handler to disable/enable keyboard input # when the process moves between foreground and background. - self.old_handlers[signal.SIGTSTP] = signal.signal( - signal.SIGTSTP, self._tstp_handler) + self.old_handlers[signal.SIGTSTP] = signal.signal(signal.SIGTSTP, self._tstp_handler) # add an atexit handler to ensure the terminal is restored atexit.register(self._restore_default_terminal_settings) @@ -258,6 +255,7 @@ class Unbuffered(object): This is implemented by forcing a flush after each write. """ + def __init__(self, stream): self.stream = stream @@ -302,6 +300,7 @@ class FileWrapper(object): yet), or neither. When unwrapped, it returns an open file (or file-like) object. """ + def __init__(self, file_like): # This records whether the file-like object returned by "unwrap" is # purely in-memory. In that case a subprocess will need to explicitly @@ -325,9 +324,9 @@ class FileWrapper(object): if self.open: if self.file_like: if sys.version_info < (3,): - self.file = open(self.file_like, 'w') + self.file = open(self.file_like, "w") else: - self.file = open(self.file_like, 'w', encoding='utf-8') # novm + self.file = open(self.file_like, "w", encoding="utf-8") # novm else: self.file = StringIO() return self.file @@ -343,8 +342,9 @@ class FileWrapper(object): class MultiProcessFd(object): """Return an object which stores a file descriptor and can be passed as an - argument to a function run with ``multiprocessing.Process``, such that - the file descriptor is available in the subprocess.""" + argument to a function run with ``multiprocessing.Process``, such that + the file descriptor is available in the subprocess.""" + def __init__(self, fd): self._connection = None self._fd = None @@ -434,7 +434,7 @@ def log_output(*args, **kwargs): This method is actually a factory serving a per platform (unix vs windows) log_output class """ - if sys.platform == 'win32': + if sys.platform == "win32": return winlog(*args, **kwargs) else: return nixlog(*args, **kwargs) @@ -454,8 +454,9 @@ class nixlog(object): work within test frameworks like nose and pytest. """ - def __init__(self, file_like=None, echo=False, debug=0, buffer=False, - env=None, filter_fn=None): + def __init__( + self, file_like=None, echo=False, debug=0, buffer=False, env=None, filter_fn=None + ): """Create a new output log context manager. Args: @@ -524,8 +525,7 @@ class nixlog(object): raise RuntimeError("Can't re-enter the same log_output!") if self.file_like is None: - raise RuntimeError( - "file argument must be set by either __init__ or __call__") + raise RuntimeError("file argument must be set by either __init__ or __call__") # set up a stream for the daemon to write to self.log_file = FileWrapper(self.file_like) @@ -555,9 +555,7 @@ class nixlog(object): input_multiprocess_fd = None try: if sys.stdin.isatty(): - input_multiprocess_fd = MultiProcessFd( - os.dup(sys.stdin.fileno()) - ) + input_multiprocess_fd = MultiProcessFd(os.dup(sys.stdin.fileno())) except BaseException: # just don't forward input if this fails pass @@ -566,9 +564,14 @@ class nixlog(object): self.process = multiprocessing.Process( target=_writer_daemon, args=( - input_multiprocess_fd, read_multiprocess_fd, write_fd, - self.echo, self.log_file, child_pipe, self.filter_fn - ) + input_multiprocess_fd, + read_multiprocess_fd, + write_fd, + self.echo, + self.log_file, + child_pipe, + self.filter_fn, + ), ) self.process.daemon = True # must set before start() self.process.start() @@ -609,7 +612,7 @@ class nixlog(object): self._saved_stderr = sys.stderr # create a file object for the pipe; redirect to it. - pipe_fd_out = os.fdopen(write_fd, 'w') + pipe_fd_out = os.fdopen(write_fd, "w") sys.stdout = pipe_fd_out sys.stderr = pipe_fd_out @@ -674,8 +677,7 @@ class nixlog(object): def force_echo(self): """Context manager to force local echo, even if echo is off.""" if not self._active: - raise RuntimeError( - "Can't call force_echo() outside log_output region!") + raise RuntimeError("Can't call force_echo() outside log_output region!") # This uses the xon/xoff to highlight regions to be echoed in the # output. We us these control characters rather than, say, a @@ -691,25 +693,26 @@ class nixlog(object): class StreamWrapper: - """ Wrapper class to handle redirection of io streams """ + """Wrapper class to handle redirection of io streams""" + def __init__(self, sys_attr): self.sys_attr = sys_attr self.saved_stream = None - if sys.platform.startswith('win32'): + if sys.platform.startswith("win32"): if sys.version_info < (3, 5): - libc = ctypes.CDLL(ctypes.util.find_library('c')) + libc = ctypes.CDLL(ctypes.util.find_library("c")) else: - if hasattr(sys, 'gettotalrefcount'): # debug build - libc = ctypes.CDLL('ucrtbased') + if hasattr(sys, "gettotalrefcount"): # debug build + libc = ctypes.CDLL("ucrtbased") else: - libc = ctypes.CDLL('api-ms-win-crt-stdio-l1-1-0') + libc = ctypes.CDLL("api-ms-win-crt-stdio-l1-1-0") - kernel32 = ctypes.WinDLL('kernel32') + kernel32 = ctypes.WinDLL("kernel32") # https://docs.microsoft.com/en-us/windows/console/getstdhandle - if self.sys_attr == 'stdout': + if self.sys_attr == "stdout": STD_HANDLE = -11 - elif self.sys_attr == 'stderr': + elif self.sys_attr == "stderr": STD_HANDLE = -12 else: raise KeyError(self.sys_attr) @@ -728,7 +731,7 @@ class StreamWrapper: def redirect_stream(self, to_fd): """Redirect stdout to the given file descriptor.""" # Flush the C-level buffer stream - if sys.platform.startswith('win32'): + if sys.platform.startswith("win32"): self.libc.fflush(None) else: self.libc.fflush(self.c_stream) @@ -739,13 +742,13 @@ class StreamWrapper: # Make orig_stream_fd point to the same file as to_fd os.dup2(to_fd, self.orig_stream_fd) # Set sys_stream to a new stream that points to the redirected fd - new_buffer = open(self.orig_stream_fd, 'wb') + new_buffer = open(self.orig_stream_fd, "wb") new_stream = io.TextIOWrapper(new_buffer) setattr(sys, self.sys_attr, new_stream) self.sys_stream = getattr(sys, self.sys_attr) def flush(self): - if sys.platform.startswith('win32'): + if sys.platform.startswith("win32"): self.libc.fflush(None) else: self.libc.fflush(self.c_stream) @@ -768,14 +771,16 @@ class winlog(object): Does not support the use of 'v' toggling as nixlog does. """ - def __init__(self, file_like=None, echo=False, debug=0, buffer=False, - env=None, filter_fn=None): + + def __init__( + self, file_like=None, echo=False, debug=0, buffer=False, env=None, filter_fn=None + ): self.env = env self.debug = debug self.echo = echo self.logfile = file_like - self.stdout = StreamWrapper('stdout') - self.stderr = StreamWrapper('stderr') + self.stdout = StreamWrapper("stdout") + self.stderr = StreamWrapper("stderr") self._active = False self._ioflag = False self.old_stdout = sys.stdout @@ -786,8 +791,7 @@ class winlog(object): raise RuntimeError("Can't re-enter the same log_output!") if self.logfile is None: - raise RuntimeError( - "file argument must be set by __init__ ") + raise RuntimeError("file argument must be set by __init__ ") # Open both write and reading on logfile if type(self.logfile) == StringIO: @@ -796,8 +800,8 @@ class winlog(object): sys.stdout = self.logfile sys.stderr = self.logfile else: - self.writer = open(self.logfile, mode='wb+') - self.reader = open(self.logfile, mode='rb+') + self.writer = open(self.logfile, mode="wb+") + self.reader = open(self.logfile, mode="rb+") # Dup stdout so we can still write to it after redirection self.echo_writer = open(os.dup(sys.stdout.fileno()), "w") @@ -811,7 +815,7 @@ class winlog(object): # if echo: write line to user try: while True: - is_killed = _kill.wait(.1) + is_killed = _kill.wait(0.1) # Flush buffered build output to file # stdout/err fds refer to log file self.stderr.flush() @@ -819,7 +823,7 @@ class winlog(object): line = reader.readline() if self.echo and line: - echo_writer.write('{0}'.format(line.decode())) + echo_writer.write("{0}".format(line.decode())) echo_writer.flush() if is_killed: @@ -829,8 +833,9 @@ class winlog(object): self._active = True with replace_environment(self.env): - self._thread = Thread(target=background_reader, - args=(self.reader, self.echo_writer, self._kill)) + self._thread = Thread( + target=background_reader, args=(self.reader, self.echo_writer, self._kill) + ) self._thread.start() return self @@ -854,13 +859,19 @@ class winlog(object): def force_echo(self): """Context manager to force local echo, even if echo is off.""" if not self._active: - raise RuntimeError( - "Can't call force_echo() outside log_output region!") + raise RuntimeError("Can't call force_echo() outside log_output region!") yield -def _writer_daemon(stdin_multiprocess_fd, read_multiprocess_fd, write_fd, echo, - log_file_wrapper, control_pipe, filter_fn): +def _writer_daemon( + stdin_multiprocess_fd, + read_multiprocess_fd, + write_fd, + echo, + log_file_wrapper, + control_pipe, + filter_fn, +): """Daemon used by ``log_output`` to write to a log file and to ``stdout``. The daemon receives output from the parent process and writes it both @@ -913,16 +924,16 @@ def _writer_daemon(stdin_multiprocess_fd, read_multiprocess_fd, write_fd, echo, # write_fd to terminate the reading loop, so we close the file descriptor # here. Forking is the process spawning method everywhere except Mac OS # for Python >= 3.8 and on Windows - if sys.version_info < (3, 8) or sys.platform != 'darwin': + if sys.version_info < (3, 8) or sys.platform != "darwin": os.close(write_fd) # Use line buffering (3rd param = 1) since Python 3 has a bug # that prevents unbuffered text I/O. if sys.version_info < (3,): - in_pipe = os.fdopen(read_multiprocess_fd.fd, 'r', 1) + in_pipe = os.fdopen(read_multiprocess_fd.fd, "r", 1) else: # Python 3.x before 3.7 does not open with UTF-8 encoding by default - in_pipe = os.fdopen(read_multiprocess_fd.fd, 'r', 1, encoding='utf-8') + in_pipe = os.fdopen(read_multiprocess_fd.fd, "r", 1, encoding="utf-8") if stdin_multiprocess_fd: stdin = os.fdopen(stdin_multiprocess_fd.fd) @@ -931,7 +942,7 @@ def _writer_daemon(stdin_multiprocess_fd, read_multiprocess_fd, write_fd, echo, # list of streams to select from istreams = [in_pipe, stdin] if stdin else [in_pipe] - force_echo = False # parent can force echo for certain output + force_echo = False # parent can force echo for certain output log_file = log_file_wrapper.unwrap() @@ -954,7 +965,7 @@ def _writer_daemon(stdin_multiprocess_fd, read_multiprocess_fd, write_fd, echo, # check and the read, so we ignore SIGTTIN here. with ignore_signal(signal.SIGTTIN): try: - if stdin.read(1) == 'v': + if stdin.read(1) == "v": echo = not echo except IOError as e: # If SIGTTIN is ignored, the system gives EIO @@ -972,14 +983,14 @@ def _writer_daemon(stdin_multiprocess_fd, read_multiprocess_fd, write_fd, echo, line = _retry(in_pipe.readline)() except UnicodeDecodeError: # installs like --test=root gpgme produce non-UTF8 logs - line = '<line lost: output was not encoded as UTF-8>\n' + line = "<line lost: output was not encoded as UTF-8>\n" if not line: return line_count += 1 # find control characters and strip them. - clean_line, num_controls = control.subn('', line) + clean_line, num_controls = control.subn("", line) # Echo to stdout if requested or forced. if echo or force_echo: @@ -1043,6 +1054,7 @@ def _retry(function): relevant for this file. """ + def wrapped(*args, **kwargs): while True: try: @@ -1055,6 +1067,7 @@ def _retry(function): if e.args[0] == errno.EINTR: continue raise + return wrapped diff --git a/lib/spack/llnl/util/tty/pty.py b/lib/spack/llnl/util/tty/pty.py index 1a5731a5c7..8d7213a533 100644 --- a/lib/spack/llnl/util/tty/pty.py +++ b/lib/spack/llnl/util/tty/pty.py @@ -30,6 +30,7 @@ from spack.util.executable import which termios = None try: import termios as term_mod + termios = term_mod except ImportError: pass @@ -42,8 +43,8 @@ class ProcessController(object): minion) similar to the way a shell would, by sending signals and I/O. """ - def __init__(self, pid, controller_fd, - timeout=1, sleep_time=1e-1, debug=False): + + def __init__(self, pid, controller_fd, timeout=1, sleep_time=1e-1, debug=False): """Create a controller to manipulate the process with id ``pid`` Args: @@ -84,18 +85,19 @@ class ProcessController(object): def horizontal_line(self, name): """Labled horizontal line for debugging.""" if self.debug: - sys.stderr.write( - "------------------------------------------- %s\n" % name - ) + sys.stderr.write("------------------------------------------- %s\n" % name) def status(self): """Print debug message with status info for the minion.""" if self.debug: canon, echo = self.get_canon_echo_attrs() - sys.stderr.write("canon: %s, echo: %s\n" % ( - "on" if canon else "off", - "on" if echo else "off", - )) + sys.stderr.write( + "canon: %s, echo: %s\n" + % ( + "on" if canon else "off", + "on" if echo else "off", + ) + ) sys.stderr.write("input: %s\n" % self.input_on()) sys.stderr.write("bg: %s\n" % self.background()) sys.stderr.write("\n") @@ -137,7 +139,7 @@ class ProcessController(object): def wait(self, condition): start = time.time() - while (((time.time() - start) < self.timeout) and not condition()): + while ((time.time() - start) < self.timeout) and not condition(): time.sleep(1e-2) assert condition() @@ -219,6 +221,7 @@ class PseudoShell(object): |_________________________________________________________| """ + def __init__(self, controller_function, minion_function): self.proc = None self.controller_function = controller_function @@ -242,8 +245,12 @@ class PseudoShell(object): """ self.proc = multiprocessing.Process( target=PseudoShell._set_up_and_run_controller_function, - args=(self.controller_function, self.minion_function, - self.controller_timeout, self.sleep_time), + args=( + self.controller_function, + self.minion_function, + self.controller_timeout, + self.sleep_time, + ), kwargs=kwargs, ) self.proc.start() @@ -255,7 +262,8 @@ class PseudoShell(object): @staticmethod def _set_up_and_run_minion_function( - tty_name, stdout_fd, stderr_fd, ready, minion_function, **kwargs): + tty_name, stdout_fd, stderr_fd, ready, minion_function, **kwargs + ): """Minion process wrapper for PseudoShell. Handles the mechanics of setting up a PTY, then calls @@ -273,8 +281,7 @@ class PseudoShell(object): os.close(stdin_fd) if kwargs.get("debug"): - sys.stderr.write( - "minion: stdin.isatty(): %s\n" % sys.stdin.isatty()) + sys.stderr.write("minion: stdin.isatty(): %s\n" % sys.stdin.isatty()) # tell the parent that we're really running if kwargs.get("debug"): @@ -288,15 +295,15 @@ class PseudoShell(object): @staticmethod def _set_up_and_run_controller_function( - controller_function, minion_function, controller_timeout, - sleep_time, **kwargs): + controller_function, minion_function, controller_timeout, sleep_time, **kwargs + ): """Set up a pty, spawn a minion process, execute controller_function. Handles the mechanics of setting up a PTY, then calls ``controller_function``. """ - os.setsid() # new session; this process is the controller + os.setsid() # new session; this process is the controller controller_fd, minion_fd = os.openpty() pty_name = os.ttyname(minion_fd) @@ -305,11 +312,10 @@ class PseudoShell(object): pty_fd = os.open(pty_name, os.O_RDWR) os.close(pty_fd) - ready = multiprocessing.Value('i', False) + ready = multiprocessing.Value("i", False) minion_process = multiprocessing.Process( target=PseudoShell._set_up_and_run_minion_function, - args=(pty_name, sys.stdout.fileno(), sys.stderr.fileno(), - ready, minion_function), + args=(pty_name, sys.stdout.fileno(), sys.stderr.fileno(), ready, minion_function), kwargs=kwargs, ) minion_process.start() @@ -329,8 +335,7 @@ class PseudoShell(object): minion_pgid = os.getpgid(minion_process.pid) sys.stderr.write("minion pid: %d\n" % minion_process.pid) sys.stderr.write("minion pgid: %d\n" % minion_pgid) - sys.stderr.write( - "minion sid: %d\n" % os.getsid(minion_process.pid)) + sys.stderr.write("minion sid: %d\n" % os.getsid(minion_process.pid)) sys.stderr.write("\n") sys.stderr.flush() # set up controller to ignore SIGTSTP, like a shell @@ -339,7 +344,8 @@ class PseudoShell(object): # call the controller function once the minion is ready try: controller = ProcessController( - minion_process.pid, controller_fd, debug=kwargs.get("debug")) + minion_process.pid, controller_fd, debug=kwargs.get("debug") + ) controller.timeout = controller_timeout controller.sleep_time = sleep_time error = controller_function(minion_process, controller, **kwargs) |