diff options
Diffstat (limited to 'lib/spack/llnl/util/filesystem.py')
| -rw-r--r-- | lib/spack/llnl/util/filesystem.py | 98 |
1 files changed, 68 insertions, 30 deletions
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index bbe83de340..a23053df9c 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -18,11 +18,13 @@ import stat import sys import tempfile from contextlib import contextmanager +from itertools import accumulate from typing import Callable, Iterable, List, Match, Optional, Tuple, Union +import llnl.util.symlink from llnl.util import tty from llnl.util.lang import dedupe, memoized -from llnl.util.symlink import islink, symlink +from llnl.util.symlink import islink, readlink, resolve_link_target_relative_to_the_link, symlink from spack.util.executable import Executable, which from spack.util.path import path_to_os_path, system_path_filter @@ -101,7 +103,7 @@ if sys.version_info < (3, 7, 4): pass # follow symlinks (aka don't not follow symlinks) - follow = follow_symlinks or not (os.path.islink(src) and os.path.islink(dst)) + follow = follow_symlinks or not (islink(src) and islink(dst)) if follow: # use the real function if it exists def lookup(name): @@ -169,7 +171,7 @@ def rename(src, dst): if sys.platform == "win32": # Windows path existence checks will sometimes fail on junctions/links/symlinks # so check for that case - if os.path.exists(dst) or os.path.islink(dst): + if os.path.exists(dst) or islink(dst): os.remove(dst) os.rename(src, dst) @@ -566,7 +568,7 @@ def set_install_permissions(path): # If this points to a file maintained in a Spack prefix, it is assumed that # this function will be invoked on the target. If the file is outside a # Spack-maintained prefix, the permissions should not be modified. - if os.path.islink(path): + if islink(path): return if os.path.isdir(path): os.chmod(path, 0o755) @@ -635,7 +637,7 @@ def chmod_x(entry, perms): @system_path_filter def copy_mode(src, dest): """Set the mode of dest to that of src unless it is a link.""" - if os.path.islink(dest): + if islink(dest): return src_mode = os.stat(src).st_mode dest_mode = os.stat(dest).st_mode @@ -722,25 +724,11 @@ def install(src, dest): @system_path_filter -def resolve_link_target_relative_to_the_link(link): - """ - os.path.isdir uses os.path.exists, which for links will check - the existence of the link target. If the link target is relative to - the link, we need to construct a pathname that is valid from - our cwd (which may not be the same as the link's directory) - """ - target = os.readlink(link) - if os.path.isabs(target): - return target - link_dir = os.path.dirname(os.path.abspath(link)) - return os.path.join(link_dir, target) - - -@system_path_filter def copy_tree( src: str, dest: str, symlinks: bool = True, + allow_broken_symlinks: bool = sys.platform != "win32", ignore: Optional[Callable[[str], bool]] = None, _permissions: bool = False, ): @@ -763,6 +751,8 @@ def copy_tree( src (str): the directory to copy dest (str): the destination directory symlinks (bool): whether or not to preserve symlinks + allow_broken_symlinks (bool): whether or not to allow broken (dangling) symlinks, + On Windows, setting this to True will raise an exception. Defaults to true on unix. ignore (typing.Callable): function indicating which files to ignore _permissions (bool): for internal use only @@ -770,6 +760,8 @@ def copy_tree( IOError: if *src* does not match any files or directories ValueError: if *src* is a parent directory of *dest* """ + if allow_broken_symlinks and sys.platform == "win32": + raise llnl.util.symlink.SymlinkError("Cannot allow broken symlinks on Windows!") if _permissions: tty.debug("Installing {0} to {1}".format(src, dest)) else: @@ -783,6 +775,11 @@ def copy_tree( if not files: raise IOError("No such file or directory: '{0}'".format(src)) + # For Windows hard-links and junctions, the source path must exist to make a symlink. Add + # all symlinks to this list while traversing the tree, then when finished, make all + # symlinks at the end. + links = [] + for src in files: abs_src = os.path.abspath(src) if not abs_src.endswith(os.path.sep): @@ -805,7 +802,7 @@ def copy_tree( ignore=ignore, follow_nonexisting=True, ): - if os.path.islink(s): + if islink(s): link_target = resolve_link_target_relative_to_the_link(s) if symlinks: target = os.readlink(s) @@ -819,7 +816,9 @@ def copy_tree( tty.debug("Redirecting link {0} to {1}".format(target, new_target)) target = new_target - symlink(target, d) + links.append((target, d, s)) + continue + elif os.path.isdir(link_target): mkdirp(d) else: @@ -834,9 +833,17 @@ def copy_tree( set_install_permissions(d) copy_mode(s, d) + for target, d, s in links: + symlink(target, d, allow_broken_symlinks=allow_broken_symlinks) + if _permissions: + set_install_permissions(d) + copy_mode(s, d) + @system_path_filter -def install_tree(src, dest, symlinks=True, ignore=None): +def install_tree( + src, dest, symlinks=True, ignore=None, allow_broken_symlinks=sys.platform != "win32" +): """Recursively install an entire directory tree rooted at *src*. Same as :py:func:`copy_tree` with the addition of setting proper @@ -847,12 +854,21 @@ def install_tree(src, dest, symlinks=True, ignore=None): dest (str): the destination directory symlinks (bool): whether or not to preserve symlinks ignore (typing.Callable): function indicating which files to ignore + allow_broken_symlinks (bool): whether or not to allow broken (dangling) symlinks, + On Windows, setting this to True will raise an exception. Raises: IOError: if *src* does not match any files or directories ValueError: if *src* is a parent directory of *dest* """ - copy_tree(src, dest, symlinks=symlinks, ignore=ignore, _permissions=True) + copy_tree( + src, + dest, + symlinks=symlinks, + allow_broken_symlinks=allow_broken_symlinks, + ignore=ignore, + _permissions=True, + ) @system_path_filter @@ -1256,7 +1272,12 @@ def traverse_tree( Keyword Arguments: order (str): Whether to do pre- or post-order traversal. Accepted values are 'pre' and 'post' - ignore (typing.Callable): function indicating which files to ignore + ignore (typing.Callable): function indicating which files to ignore. This will also + ignore symlinks if they point to an ignored file (regardless of whether the symlink + is explicitly ignored); note this only supports one layer of indirection (i.e. if + you have x -> y -> z, and z is ignored but x/y are not, then y would be ignored + but not x). To avoid this, make sure the ignore function also ignores the symlink + paths too. follow_nonexisting (bool): Whether to descend into directories in ``src`` that do not exit in ``dest``. Default is True follow_links (bool): Whether to descend into symlinks in ``src`` @@ -1283,11 +1304,24 @@ def traverse_tree( dest_child = os.path.join(dest_path, f) rel_child = os.path.join(rel_path, f) + # If the source path is a link and the link's source is ignored, then ignore the link too, + # but only do this if the ignore is defined. + if ignore is not None: + if islink(source_child) and not follow_links: + target = readlink(source_child) + all_parents = accumulate(target.split(os.sep), lambda x, y: os.path.join(x, y)) + if any(map(ignore, all_parents)): + tty.warn( + f"Skipping {source_path} because the source or a part of the source's " + f"path is included in the ignores." + ) + continue + # Treat as a directory # TODO: for symlinks, os.path.isdir looks for the link target. If the # target is relative to the link, then that may not resolve properly # relative to our cwd - see resolve_link_target_relative_to_the_link - if os.path.isdir(source_child) and (follow_links or not os.path.islink(source_child)): + if os.path.isdir(source_child) and (follow_links or not islink(source_child)): # When follow_nonexisting isn't set, don't descend into dirs # in source that do not exist in dest if follow_nonexisting or os.path.exists(dest_child): @@ -1313,7 +1347,11 @@ def traverse_tree( def lexists_islink_isdir(path): """Computes the tuple (lexists(path), islink(path), isdir(path)) in a minimal - number of stat calls.""" + number of stat calls on unix. Use os.path and symlink.islink methods for windows.""" + if sys.platform == "win32": + if not os.path.lexists(path): + return False, False, False + return os.path.lexists(path), islink(path), os.path.isdir(path) # First try to lstat, so we know if it's a link or not. try: lst = os.lstat(path) @@ -1528,7 +1566,7 @@ def remove_if_dead_link(path): Parameters: path (str): The potential dead link """ - if os.path.islink(path) and not os.path.exists(path): + if islink(path) and not os.path.exists(path): os.unlink(path) @@ -1587,7 +1625,7 @@ def remove_linked_tree(path): kwargs["onerror"] = readonly_file_handler(ignore_errors=True) if os.path.exists(path): - if os.path.islink(path): + if islink(path): shutil.rmtree(os.path.realpath(path), **kwargs) os.unlink(path) else: @@ -2693,7 +2731,7 @@ def remove_directory_contents(dir): """Remove all contents of a directory.""" if os.path.exists(dir): for entry in [os.path.join(dir, entry) for entry in os.listdir(dir)]: - if os.path.isfile(entry) or os.path.islink(entry): + if os.path.isfile(entry) or islink(entry): os.unlink(entry) else: shutil.rmtree(entry) |