From 66f75407d136c0c06ed01774d5815728afba35ca Mon Sep 17 00:00:00 2001 From: David Alexander Date: Mon, 26 Jun 2023 14:38:14 -0600 Subject: Updates to symlinking for Windows (#34701) * Support hardlinks/junctions on Windows systems without developer mode enabled * Generally, use of llnl.util.symlink.symlink is preferred over os.symlink since it handles this automatically * Generally an error is now reported if a user attempts to create a symlink to a file that does not exist (this was previously allowed on Linux/Mac). * One exception to this: when Spack installs files from the source into their final prefix, dangling symlinks are allowed (on Linux/Mac - Windows does not allow this in any circumstance). The intent behind this is to avoid generating failures for installations on Linux/Mac that were succeeding before. * Because Windows is strict about forbidding dangling symlinks, `traverse_tree` has been updated to skip creating symlinks if they would point to a file that is ignored. This check is not transitive (i.e., a symlink to a symlink to an ignored file would not be caught appropriately) * Relocate function: resolve_link_target_relative_to_the_link (this is not otherwise modified) Co-authored-by: jamessmillie --- lib/spack/llnl/util/filesystem.py | 147 +++++++----- lib/spack/llnl/util/symlink.py | 336 ++++++++++++++++++++++----- lib/spack/spack/compilers/apple_clang.py | 6 +- lib/spack/spack/installer.py | 5 +- lib/spack/spack/test/architecture.py | 1 + lib/spack/spack/test/cache_fetch.py | 9 +- lib/spack/spack/test/compilers/basics.py | 10 +- lib/spack/spack/test/conftest.py | 4 +- lib/spack/spack/test/llnl/util/filesystem.py | 82 ++++++- lib/spack/spack/test/llnl/util/link_tree.py | 147 +++++++----- lib/spack/spack/test/llnl/util/symlink.py | 246 ++++++++++++++++++++ lib/spack/spack/test/mirror.py | 6 +- lib/spack/spack/test/packaging.py | 9 +- 13 files changed, 817 insertions(+), 191 deletions(-) create mode 100644 lib/spack/spack/test/llnl/util/symlink.py (limited to 'lib') diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 2b4cbe2424..346c44e66d 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -18,11 +18,13 @@ import stat import sys import tempfile from contextlib import contextmanager +from itertools import accumulate from typing import Callable, Iterable, List, Match, Optional, Tuple, Union +import llnl.util.symlink from llnl.util import tty from llnl.util.lang import dedupe, memoized -from llnl.util.symlink import islink, symlink +from llnl.util.symlink import islink, readlink, resolve_link_target_relative_to_the_link, symlink from spack.util.executable import Executable, which from spack.util.path import path_to_os_path, system_path_filter @@ -101,7 +103,7 @@ if sys.version_info < (3, 7, 4): pass # follow symlinks (aka don't not follow symlinks) - follow = follow_symlinks or not (os.path.islink(src) and os.path.islink(dst)) + follow = follow_symlinks or not (islink(src) and islink(dst)) if follow: # use the real function if it exists def lookup(name): @@ -169,7 +171,7 @@ def rename(src, dst): if sys.platform == "win32": # Windows path existence checks will sometimes fail on junctions/links/symlinks # so check for that case - if os.path.exists(dst) or os.path.islink(dst): + if os.path.exists(dst) or islink(dst): os.remove(dst) os.rename(src, dst) @@ -566,7 +568,7 @@ def set_install_permissions(path): # If this points to a file maintained in a Spack prefix, it is assumed that # this function will be invoked on the target. If the file is outside a # Spack-maintained prefix, the permissions should not be modified. - if os.path.islink(path): + if islink(path): return if os.path.isdir(path): os.chmod(path, 0o755) @@ -633,7 +635,7 @@ def chmod_x(entry, perms): @system_path_filter def copy_mode(src, dest): """Set the mode of dest to that of src unless it is a link.""" - if os.path.islink(dest): + if islink(dest): return src_mode = os.stat(src).st_mode dest_mode = os.stat(dest).st_mode @@ -719,26 +721,12 @@ def install(src, dest): copy(src, dest, _permissions=True) -@system_path_filter -def resolve_link_target_relative_to_the_link(link): - """ - os.path.isdir uses os.path.exists, which for links will check - the existence of the link target. If the link target is relative to - the link, we need to construct a pathname that is valid from - our cwd (which may not be the same as the link's directory) - """ - target = os.readlink(link) - if os.path.isabs(target): - return target - link_dir = os.path.dirname(os.path.abspath(link)) - return os.path.join(link_dir, target) - - @system_path_filter def copy_tree( src: str, dest: str, symlinks: bool = True, + allow_broken_symlinks: bool = False, ignore: Optional[Callable[[str], bool]] = None, _permissions: bool = False, ): @@ -761,6 +749,8 @@ def copy_tree( src (str): the directory to copy dest (str): the destination directory symlinks (bool): whether or not to preserve symlinks + allow_broken_symlinks (bool): whether or not to allow broken (dangling) symlinks, + On Windows, setting this to True will raise an exception. ignore (typing.Callable): function indicating which files to ignore _permissions (bool): for internal use only @@ -768,6 +758,8 @@ def copy_tree( IOError: if *src* does not match any files or directories ValueError: if *src* is a parent directory of *dest* """ + if allow_broken_symlinks and sys.platform == "win32": + raise llnl.util.symlink.SymlinkError("Cannot allow broken symlinks on Windows!") if _permissions: tty.debug("Installing {0} to {1}".format(src, dest)) else: @@ -781,6 +773,11 @@ def copy_tree( if not files: raise IOError("No such file or directory: '{0}'".format(src)) + # For Windows hard-links and junctions, the source path must exist to make a symlink. Add + # all symlinks to this list while traversing the tree, then when finished, make all + # symlinks at the end. + links = [] + for src in files: abs_src = os.path.abspath(src) if not abs_src.endswith(os.path.sep): @@ -803,21 +800,27 @@ def copy_tree( ignore=ignore, follow_nonexisting=True, ): - if os.path.islink(s): + if islink(s): link_target = resolve_link_target_relative_to_the_link(s) if symlinks: target = os.readlink(s) - if os.path.isabs(target): - def escaped_path(path): - return path.replace("\\", r"\\") + def escaped_path(path): + return path.replace("\\", r"\\") + if os.path.isabs(target): new_target = re.sub(escaped_path(abs_src), escaped_path(abs_dest), target) - if new_target != target: - tty.debug("Redirecting link {0} to {1}".format(target, new_target)) - target = new_target - symlink(target, d) + else: + new_target = re.sub(escaped_path(src), escaped_path(dest), target) + + if new_target != target: + tty.debug("Redirecting link {0} to {1}".format(target, new_target)) + target = new_target + + links.append((target, d, s)) + continue + elif os.path.isdir(link_target): mkdirp(d) else: @@ -832,9 +835,15 @@ def copy_tree( set_install_permissions(d) copy_mode(s, d) + for target, d, s in links: + symlink(target, d, allow_broken_symlinks=allow_broken_symlinks) + if _permissions: + set_install_permissions(d) + copy_mode(s, d) + @system_path_filter -def install_tree(src, dest, symlinks=True, ignore=None): +def install_tree(src, dest, symlinks=True, ignore=None, allow_broken_symlinks=False): """Recursively install an entire directory tree rooted at *src*. Same as :py:func:`copy_tree` with the addition of setting proper @@ -845,12 +854,21 @@ def install_tree(src, dest, symlinks=True, ignore=None): dest (str): the destination directory symlinks (bool): whether or not to preserve symlinks ignore (typing.Callable): function indicating which files to ignore + allow_broken_symlinks (bool): whether or not to allow broken (dangling) symlinks, + On Windows, setting this to True will raise an exception. Raises: IOError: if *src* does not match any files or directories ValueError: if *src* is a parent directory of *dest* """ - copy_tree(src, dest, symlinks=symlinks, ignore=ignore, _permissions=True) + copy_tree( + src, + dest, + symlinks=symlinks, + allow_broken_symlinks=allow_broken_symlinks, + ignore=ignore, + _permissions=True, + ) @system_path_filter @@ -1254,7 +1272,12 @@ def traverse_tree( Keyword Arguments: order (str): Whether to do pre- or post-order traversal. Accepted values are 'pre' and 'post' - ignore (typing.Callable): function indicating which files to ignore + ignore (typing.Callable): function indicating which files to ignore. This will also + ignore symlinks if they point to an ignored file (regardless of whether the symlink + is explicitly ignored); note this only supports one layer of indirection (i.e. if + you have x -> y -> z, and z is ignored but x/y are not, then y would be ignored + but not x). To avoid this, make sure the ignore function also ignores the symlink + paths too. follow_nonexisting (bool): Whether to descend into directories in ``src`` that do not exit in ``dest``. Default is True follow_links (bool): Whether to descend into symlinks in ``src`` @@ -1281,11 +1304,22 @@ def traverse_tree( dest_child = os.path.join(dest_path, f) rel_child = os.path.join(rel_path, f) + # If the source path is a link and the link's source is ignored, then ignore the link too. + if islink(source_child) and not follow_links: + target = readlink(source_child) + all_parents = accumulate(target.split(os.sep), lambda x, y: os.path.join(x, y)) + if any(map(ignore, all_parents)): + tty.warn( + f"Skipping {source_path} because the source or a part of the source's " + f"path is included in the ignores." + ) + continue + # Treat as a directory # TODO: for symlinks, os.path.isdir looks for the link target. If the # target is relative to the link, then that may not resolve properly # relative to our cwd - see resolve_link_target_relative_to_the_link - if os.path.isdir(source_child) and (follow_links or not os.path.islink(source_child)): + if os.path.isdir(source_child) and (follow_links or not islink(source_child)): # When follow_nonexisting isn't set, don't descend into dirs # in source that do not exist in dest if follow_nonexisting or os.path.exists(dest_child): @@ -1311,29 +1345,34 @@ def traverse_tree( def lexists_islink_isdir(path): """Computes the tuple (lexists(path), islink(path), isdir(path)) in a minimal - number of stat calls.""" - # First try to lstat, so we know if it's a link or not. - try: - lst = os.lstat(path) - except (IOError, OSError): - return False, False, False + number of stat calls on unix. Use os.path and symlink.islink methods for windows.""" + if sys.platform == "win32": + if not os.path.lexists(path): + return False, False, False + return os.path.lexists(path), islink(path), os.path.isdir(path) + else: + # First try to lstat, so we know if it's a link or not. + try: + lst = os.lstat(path) + except (IOError, OSError): + return False, False, False - is_link = stat.S_ISLNK(lst.st_mode) + is_link = stat.S_ISLNK(lst.st_mode) - # Check whether file is a dir. - if not is_link: - is_dir = stat.S_ISDIR(lst.st_mode) - return True, is_link, is_dir + # Check whether file is a dir. + if not is_link: + is_dir = stat.S_ISDIR(lst.st_mode) + return True, is_link, is_dir - # Check whether symlink points to a dir. - try: - st = os.stat(path) - is_dir = stat.S_ISDIR(st.st_mode) - except (IOError, OSError): - # Dangling symlink (i.e. it lexists but not exists) - is_dir = False + # Check whether symlink points to a dir. + try: + st = os.stat(path) + is_dir = stat.S_ISDIR(st.st_mode) + except (IOError, OSError): + # Dangling symlink (i.e. it lexists but not exists) + is_dir = False - return True, is_link, is_dir + return True, is_link, is_dir class BaseDirectoryVisitor(object): @@ -1526,7 +1565,7 @@ def remove_if_dead_link(path): Parameters: path (str): The potential dead link """ - if os.path.islink(path) and not os.path.exists(path): + if islink(path) and not os.path.exists(path): os.unlink(path) @@ -1585,7 +1624,7 @@ def remove_linked_tree(path): kwargs["onerror"] = readonly_file_handler(ignore_errors=True) if os.path.exists(path): - if os.path.islink(path): + if islink(path): shutil.rmtree(os.path.realpath(path), **kwargs) os.unlink(path) else: @@ -2686,7 +2725,7 @@ def remove_directory_contents(dir): """Remove all contents of a directory.""" if os.path.exists(dir): for entry in [os.path.join(dir, entry) for entry in os.listdir(dir)]: - if os.path.isfile(entry) or os.path.islink(entry): + if os.path.isfile(entry) or islink(entry): os.unlink(entry) else: shutil.rmtree(entry) diff --git a/lib/spack/llnl/util/symlink.py b/lib/spack/llnl/util/symlink.py index 69aacaf9f0..f6f250c3eb 100644 --- a/lib/spack/llnl/util/symlink.py +++ b/lib/spack/llnl/util/symlink.py @@ -2,77 +2,185 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import errno import os +import re import shutil +import subprocess import sys import tempfile -from os.path import exists, join -from llnl.util import lang +from llnl.util import lang, tty + +from spack.error import SpackError +from spack.util.path import system_path_filter if sys.platform == "win32": from win32file import CreateHardLink -def symlink(real_path, link_path): +def symlink(source_path: str, link_path: str, allow_broken_symlinks: bool = False): """ - Create a symbolic link. + Create a link. + + On non-Windows and Windows with System Administrator + privleges this will be a normal symbolic link via + os.symlink. + + On Windows without privledges the link will be a + junction for a directory and a hardlink for a file. + On Windows the various link types are: + + Symbolic Link: A link to a file or directory on the + same or different volume (drive letter) or even to + a remote file or directory (using UNC in its path). + Need System Administrator privileges to make these. - On Windows, use junctions if os.symlink fails. + Hard Link: A link to a file on the same volume (drive + letter) only. Every file (file's data) has at least 1 + hard link (file's name). But when this method creates + a new hard link there will be 2. Deleting all hard + links effectively deletes the file. Don't need System + Administrator privileges. + + Junction: A link to a directory on the same or different + volume (drive letter) but not to a remote directory. Don't + need System Administrator privileges. + + Parameters: + source_path (str): The real file or directory that the link points to. + Must be absolute OR relative to the link. + link_path (str): The path where the link will exist. + allow_broken_symlinks (bool): On Linux or Mac, don't raise an exception if the source_path + doesn't exist. This will still raise an exception on Windows. """ - if sys.platform != "win32": - os.symlink(real_path, link_path) - elif _win32_can_symlink(): - # Windows requires target_is_directory=True when the target is a dir. - os.symlink(real_path, link_path, target_is_directory=os.path.isdir(real_path)) + source_path = os.path.normpath(source_path) + win_source_path = source_path + link_path = os.path.normpath(link_path) + + # Never allow broken links on Windows. + if sys.platform == "win32" and allow_broken_symlinks: + raise ValueError("allow_broken_symlinks parameter cannot be True on Windows.") + + # Perform basic checks to make sure symlinking will succeed + if os.path.lexists(link_path): + raise SymlinkError(f"Link path ({link_path}) already exists. Cannot create link.") + + if not os.path.exists(source_path): + if os.path.isabs(source_path) and not allow_broken_symlinks: + # An absolute source path that does not exist will result in a broken link. + raise SymlinkError( + f"Source path ({source_path}) is absolute but does not exist. Resulting " + f"link would be broken so not making link." + ) + else: + # os.symlink can create a link when the given source path is relative to + # the link path. Emulate this behavior and check to see if the source exists + # relative to the link patg ahead of link creation to prevent broken + # links from being made. + link_parent_dir = os.path.dirname(link_path) + relative_path = os.path.join(link_parent_dir, source_path) + if os.path.exists(relative_path): + # In order to work on windows, the source path needs to be modified to be + # relative because hardlink/junction dont resolve relative paths the same way as + # os.symlink. This is ignored on other operating systems. + win_source_path = relative_path + elif not allow_broken_symlinks: + raise SymlinkError( + f"The source path ({source_path}) is not relative to the link path " + f"({link_path}). Resulting link would be broken so not making link." + ) + + # Create the symlink + if sys.platform == "win32" and not _windows_can_symlink(): + _windows_create_link(win_source_path, link_path) else: - try: - # Try to use junctions - _win32_junction(real_path, link_path) - except OSError as e: - if e.errno == errno.EEXIST: - # EEXIST error indicates that file we're trying to "link" - # is already present, don't bother trying to copy which will also fail - # just raise - raise - else: - # If all else fails, fall back to copying files - shutil.copyfile(real_path, link_path) + os.symlink(source_path, link_path, target_is_directory=os.path.isdir(source_path)) + + +def islink(path: str) -> bool: + """Override os.islink to give correct answer for spack logic. + + For Non-Windows: a link can be determined with the os.path.islink method. + Windows-only methods will return false for other operating systems. + + For Windows: spack considers symlinks, hard links, and junctions to + all be links, so if any of those are True, return True. + + Args: + path (str): path to check if it is a link. + Returns: + bool - whether the path is any kind link or not. + """ + return any([os.path.islink(path), _windows_is_junction(path), _windows_is_hardlink(path)]) + + +def _windows_is_hardlink(path: str) -> bool: + """Determines if a path is a windows hard link. This is accomplished + by looking at the number of links using os.stat. A non-hard-linked file + will have a st_nlink value of 1, whereas a hard link will have a value + larger than 1. Note that both the original and hard-linked file will + return True because they share the same inode. + + Args: + path (str): Windows path to check for a hard link + + Returns: + bool - Whether the path is a hard link or not. + """ + if sys.platform != "win32" or os.path.islink(path) or not os.path.exists(path): + return False + + return os.stat(path).st_nlink > 1 -def islink(path): - return os.path.islink(path) or _win32_is_junction(path) +def _windows_is_junction(path: str) -> bool: + """Determines if a path is a windows junction. A junction can be + determined using a bitwise AND operation between the file's + attribute bitmask and the known junction bitmask (0x400). -# '_win32' functions based on -# https://github.com/Erotemic/ubelt/blob/master/ubelt/util_links.py -def _win32_junction(path, link): - # junctions require absolute paths - if not os.path.isabs(link): - link = os.path.abspath(link) + Args: + path (str): A non-file path - # os.symlink will fail if link exists, emulate the behavior here - if exists(link): - raise OSError(errno.EEXIST, "File exists: %s -> %s" % (link, path)) + Returns: + bool - whether the path is a junction or not. + """ + if sys.platform != "win32" or os.path.islink(path) or os.path.isfile(path): + return False + + import ctypes.wintypes + + get_file_attributes = ctypes.windll.kernel32.GetFileAttributesW # type: ignore[attr-defined] + get_file_attributes.argtypes = (ctypes.wintypes.LPWSTR,) + get_file_attributes.restype = ctypes.wintypes.DWORD - if not os.path.isabs(path): - parent = os.path.join(link, os.pardir) - path = os.path.join(parent, path) - path = os.path.abspath(path) + invalid_file_attributes = 0xFFFFFFFF + reparse_point = 0x400 + file_attr = get_file_attributes(path) - CreateHardLink(link, path) + if file_attr == invalid_file_attributes: + return False + + return file_attr & reparse_point > 0 @lang.memoized -def _win32_can_symlink(): +def _windows_can_symlink() -> bool: + """ + Determines if windows is able to make a symlink depending on + the system configuration and the level of the user's permissions. + """ + if sys.platform != "win32": + tty.warn("windows_can_symlink method can't be used on non-Windows OS.") + return False + tempdir = tempfile.mkdtemp() - dpath = join(tempdir, "dpath") - fpath = join(tempdir, "fpath.txt") + dpath = os.path.join(tempdir, "dpath") + fpath = os.path.join(tempdir, "fpath.txt") - dlink = join(tempdir, "dlink") - flink = join(tempdir, "flink.txt") + dlink = os.path.join(tempdir, "dlink") + flink = os.path.join(tempdir, "flink.txt") import llnl.util.filesystem as fs @@ -96,24 +204,136 @@ def _win32_can_symlink(): return can_symlink_directories and can_symlink_files -def _win32_is_junction(path): +def _windows_create_link(source: str, link: str): """ - Determines if a path is a win32 junction + Attempts to create a Hard Link or Junction as an alternative + to a symbolic link. This is called when symbolic links cannot + be created. """ - if os.path.islink(path): - return False + if sys.platform != "win32": + raise SymlinkError("windows_create_link method can't be used on non-Windows OS.") + elif os.path.isdir(source): + _windows_create_junction(source=source, link=link) + elif os.path.isfile(source): + _windows_create_hard_link(path=source, link=link) + else: + raise SymlinkError( + f"Cannot create link from {source}. It is neither a file nor a directory." + ) + + +def _windows_create_junction(source: str, link: str): + """Duly verify that the path and link are eligible to create a junction, + then create the junction. + """ + if sys.platform != "win32": + raise SymlinkError("windows_create_junction method can't be used on non-Windows OS.") + elif not os.path.exists(source): + raise SymlinkError("Source path does not exist, cannot create a junction.") + elif os.path.lexists(link): + raise SymlinkError("Link path already exists, cannot create a junction.") + elif not os.path.isdir(source): + raise SymlinkError("Source path is not a directory, cannot create a junction.") + + import subprocess + + cmd = ["cmd", "/C", "mklink", "/J", link, source] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = proc.communicate() + tty.debug(out.decode()) + if proc.returncode != 0: + err = err.decode() + tty.error(err) + raise SymlinkError("Make junction command returned a non-zero return code.", err) + + +def _windows_create_hard_link(path: str, link: str): + """Duly verify that the path and link are eligible to create a hard + link, then create the hard link. + """ + if sys.platform != "win32": + raise SymlinkError("windows_create_hard_link method can't be used on non-Windows OS.") + elif not os.path.exists(path): + raise SymlinkError(f"File path {path} does not exist. Cannot create hard link.") + elif os.path.lexists(link): + raise SymlinkError(f"Link path ({link}) already exists. Cannot create hard link.") + elif not os.path.isfile(path): + raise SymlinkError(f"File path ({link}) is not a file. Cannot create hard link.") + else: + tty.debug(f"Creating hard link {link} pointing to {path}") + CreateHardLink(link, path) + + +def readlink(path: str): + """Spack utility to override of os.readlink method to work cross platform""" + if _windows_is_hardlink(path): + return _windows_read_hard_link(path) + elif _windows_is_junction(path): + return _windows_read_junction(path) + else: + return os.readlink(path) + + +def _windows_read_hard_link(link: str) -> str: + """Find all of the files that point to the same inode as the link""" + if sys.platform != "win32": + raise SymlinkError("Can't read hard link on non-Windows OS.") + link = os.path.abspath(link) + fsutil_cmd = ["fsutil", "hardlink", "list", link] + proc = subprocess.Popen(fsutil_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + out, err = proc.communicate() + if proc.returncode != 0: + raise SymlinkError(f"An error occurred while reading hard link: {err.decode()}") + + # fsutil response does not include the drive name, so append it back to each linked file. + drive, link_tail = os.path.splitdrive(os.path.abspath(link)) + links = set([os.path.join(drive, p) for p in out.decode().splitlines()]) + links.remove(link) + if len(links) == 1: + return links.pop() + elif len(links) > 1: + # TODO: How best to handle the case where 3 or more paths point to a single inode? + raise SymlinkError(f"Found multiple paths pointing to the same inode {links}") + else: + raise SymlinkError("Cannot determine hard link source path.") + + +def _windows_read_junction(link: str): + """Find the path that a junction points to.""" + if sys.platform != "win32": + raise SymlinkError("Can't read junction on non-Windows OS.") - if sys.platform == "win32": - import ctypes.wintypes + link = os.path.abspath(link) + link_basename = os.path.basename(link) + link_parent = os.path.dirname(link) + fsutil_cmd = ["dir", "/a:l", link_parent] + proc = subprocess.Popen(fsutil_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + out, err = proc.communicate() + if proc.returncode != 0: + raise SymlinkError(f"An error occurred while reading junction: {err.decode()}") + matches = re.search(rf"\s+{link_basename} \[(.*)]", out.decode()) + if matches: + return matches.group(1) + else: + raise SymlinkError("Could not find junction path.") - GetFileAttributes = ctypes.windll.kernel32.GetFileAttributesW - GetFileAttributes.argtypes = (ctypes.wintypes.LPWSTR,) - GetFileAttributes.restype = ctypes.wintypes.DWORD - INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF - FILE_ATTRIBUTE_REPARSE_POINT = 0x400 +@system_path_filter +def resolve_link_target_relative_to_the_link(link): + """ + os.path.isdir uses os.path.exists, which for links will check + the existence of the link target. If the link target is relative to + the link, we need to construct a pathname that is valid from + our cwd (which may not be the same as the link's directory) + """ + target = readlink(link) + if os.path.isabs(target): + return target + link_dir = os.path.dirname(os.path.abspath(link)) + return os.path.join(link_dir, target) - res = GetFileAttributes(path) - return res != INVALID_FILE_ATTRIBUTES and bool(res & FILE_ATTRIBUTE_REPARSE_POINT) - return False +class SymlinkError(SpackError): + """Exception class for errors raised while creating symlinks, + junctions and hard links + """ diff --git a/lib/spack/spack/compilers/apple_clang.py b/lib/spack/spack/compilers/apple_clang.py index cb3c5d2646..4e57cf63c9 100644 --- a/lib/spack/spack/compilers/apple_clang.py +++ b/lib/spack/spack/compilers/apple_clang.py @@ -7,8 +7,8 @@ import re import shutil import llnl.util.lang +import llnl.util.symlink as symlink import llnl.util.tty as tty -from llnl.util.symlink import symlink import spack.compiler import spack.compilers.clang @@ -216,11 +216,11 @@ class AppleClang(spack.compilers.clang.Clang): for fname in os.listdir(dev_dir): if fname in bins: os.unlink(os.path.join(dev_dir, fname)) - symlink( + symlink.symlink( os.path.join(spack.paths.build_env_path, "cc"), os.path.join(dev_dir, fname), ) - symlink(developer_root, xcode_link) + symlink.symlink(developer_root, xcode_link) env.set("DEVELOPER_DIR", xcode_link) diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index b178d30a71..7ebb3fcf8a 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -174,6 +174,7 @@ def _do_fake_install(pkg): # Install fake command fs.mkdirp(pkg.prefix.bin) fs.touch(os.path.join(pkg.prefix.bin, command)) + fs.touchp(pkg.install_log_path) if sys.platform != "win32": chmod = which("chmod") chmod("+x", os.path.join(pkg.prefix.bin, command)) @@ -1984,7 +1985,9 @@ class BuildProcessInstaller(object): src_target = os.path.join(pkg.spec.prefix, "share", pkg.name, "src") tty.debug("{0} Copying source to {1}".format(self.pre, src_target)) - fs.install_tree(pkg.stage.source_path, src_target) + fs.install_tree( + pkg.stage.source_path, src_target, allow_broken_symlinks=(sys.platform != "win32") + ) def _real_install(self): import spack.builder diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index f00d66ad32..c129251b59 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -229,6 +229,7 @@ def test_concretize_target_ranges(root_target_range, dep_target_range, result, m (["21.11", "21.9"], None, False), ], ) +@pytest.mark.skipif(sys.platform == "win32", reason="Cray does not use windows") def test_cray_platform_detection(versions, default, expected, tmpdir, monkeypatch, working_env): ex_path = str(tmpdir.join("fake_craype_dir")) fs.mkdirp(ex_path) diff --git a/lib/spack/spack/test/cache_fetch.py b/lib/spack/spack/test/cache_fetch.py index 7fb5efc895..f4bf0ba62d 100644 --- a/lib/spack/spack/test/cache_fetch.py +++ b/lib/spack/spack/test/cache_fetch.py @@ -31,13 +31,16 @@ def test_fetch_missing_cache(tmpdir, _fetch_method): @pytest.mark.parametrize("_fetch_method", ["curl", "urllib"]) def test_fetch(tmpdir, _fetch_method): """Ensure a fetch after expanding is effectively a no-op.""" - testpath = str(tmpdir) - cache = os.path.join(testpath, "cache.tar.gz") + cache_dir = tmpdir.join("cache") + stage_dir = tmpdir.join("stage") + mkdirp(cache_dir) + mkdirp(stage_dir) + cache = os.path.join(cache_dir, "cache.tar.gz") touch(cache) url = url_util.path_to_file_url(cache) with spack.config.override("config:url_fetch_method", _fetch_method): fetcher = CacheURLFetchStrategy(url=url) - with Stage(fetcher, path=testpath) as stage: + with Stage(fetcher, path=str(stage_dir)) as stage: source_path = stage.source_path mkdirp(source_path) fetcher.fetch() diff --git a/lib/spack/spack/test/compilers/basics.py b/lib/spack/spack/test/compilers/basics.py index 66c5fe451f..bf6bbdc399 100644 --- a/lib/spack/spack/test/compilers/basics.py +++ b/lib/spack/spack/test/compilers/basics.py @@ -11,6 +11,7 @@ from copy import copy import pytest import llnl.util.filesystem as fs +import llnl.util.symlink as symlink import spack.compiler import spack.compilers as compilers @@ -841,9 +842,12 @@ def test_apple_clang_setup_environment(mock_executable, monkeypatch): class MockPackage(object): use_xcode = False - apple_clang_cls = spack.compilers.class_for_compiler_name("apple-clang") + apple_clang = "apple-clang" + apple_clang_version = "11.0.0" + + apple_clang_cls = spack.compilers.class_for_compiler_name(apple_clang) compiler = apple_clang_cls( - spack.spec.CompilerSpec("apple-clang@=11.0.0"), + spack.spec.CompilerSpec(f"{apple_clang}@={apple_clang_version}"), "catalina", "x86_64", ["/usr/bin/clang", "/usr/bin/clang++", None, None], @@ -889,7 +893,7 @@ echo "/Library/Developer" # Set a few operations to noop monkeypatch.setattr(shutil, "copytree", noop) monkeypatch.setattr(os, "unlink", noop) - monkeypatch.setattr(os, "symlink", noop) + monkeypatch.setattr(symlink, "symlink", noop) monkeypatch.setattr(os, "listdir", _listdir) # Qt is so far the only package that uses this code path, change diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index f0e5f7fdb2..0b1506e9a2 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -27,7 +27,7 @@ import archspec.cpu.schema import llnl.util.lang import llnl.util.lock import llnl.util.tty as tty -from llnl.util.filesystem import copy_tree, mkdirp, remove_linked_tree, working_dir +from llnl.util.filesystem import copy_tree, mkdirp, remove_linked_tree, touchp, working_dir import spack.binary_distribution import spack.caches @@ -565,6 +565,8 @@ def mock_repo_path(): def _pkg_install_fn(pkg, spec, prefix): # sanity_check_prefix requires something in the install directory mkdirp(prefix.bin) + if not os.path.exists(spec.package.build_log_path): + touchp(spec.package.build_log_path) @pytest.fixture diff --git a/lib/spack/spack/test/llnl/util/filesystem.py b/lib/spack/spack/test/llnl/util/filesystem.py index 018137f406..193716328f 100644 --- a/lib/spack/spack/test/llnl/util/filesystem.py +++ b/lib/spack/spack/test/llnl/util/filesystem.py @@ -13,7 +13,7 @@ import sys import pytest import llnl.util.filesystem as fs -from llnl.util.symlink import islink, symlink +from llnl.util.symlink import SymlinkError, _windows_can_symlink, islink, symlink import spack.paths @@ -150,7 +150,6 @@ class TestInstall: fs.install("source/a/*/*", "dest/1") -@pytest.mark.skipif(sys.platform == "win32", reason="Skip test on Windows") class TestCopyTree: """Tests for ``filesystem.copy_tree``""" @@ -189,7 +188,7 @@ class TestCopyTree: def test_symlinks_true_ignore(self, stage): """Test copying when specifying relative paths that should be ignored""" with fs.working_dir(str(stage)): - ignore = lambda p: p in ["c/d/e", "a"] + ignore = lambda p: p in [os.path.join("c", "d", "e"), "a"] fs.copy_tree("source", "dest", symlinks=True, ignore=ignore) assert not os.path.exists("dest/a") assert os.path.exists("dest/c/d") @@ -231,7 +230,6 @@ class TestCopyTree: fs.copy_tree("source", "source/sub/directory") -@pytest.mark.skipif(sys.platform == "win32", reason="Skip test on Windows") class TestInstallTree: """Tests for ``filesystem.install_tree``""" @@ -275,6 +273,15 @@ class TestInstallTree: assert not os.path.islink("dest/2") check_added_exe_permissions("source/2", "dest/2") + @pytest.mark.skipif(sys.platform == "win32", reason="Broken symlinks not allowed on Windows") + def test_allow_broken_symlinks(self, stage): + """Test installing with a broken symlink.""" + with fs.working_dir(str(stage)): + symlink("nonexistant.txt", "source/broken", allow_broken_symlinks=True) + fs.install_tree("source", "dest", symlinks=True, allow_broken_symlinks=True) + assert os.path.islink("dest/broken") + assert not os.path.exists(os.readlink("dest/broken")) + def test_glob_src(self, stage): """Test using a glob as the source.""" @@ -712,6 +719,9 @@ def test_is_nonsymlink_exe_with_shebang(tmpdir): assert not fs.is_nonsymlink_exe_with_shebang("symlink_to_executable_script") +@pytest.mark.skipif( + sys.platform == "win32" and not _windows_can_symlink(), reason="Requires elevated privileges." +) def test_lexists_islink_isdir(tmpdir): root = str(tmpdir) @@ -730,19 +740,69 @@ def test_lexists_islink_isdir(tmpdir): with open(file, "wb") as f: f.write(b"file") - os.symlink("dir", symlink_to_dir) - os.symlink("file", symlink_to_file) - os.symlink("does_not_exist", dangling_symlink) - os.symlink("dangling_symlink", symlink_to_dangling_symlink) - os.symlink("symlink_to_dir", symlink_to_symlink_to_dir) - os.symlink("symlink_to_file", symlink_to_symlink_to_file) + symlink("dir", symlink_to_dir) + symlink("file", symlink_to_file) + + with pytest.raises(SymlinkError): + symlink("does_not_exist", dangling_symlink) + symlink("dangling_symlink", symlink_to_dangling_symlink) + + symlink("symlink_to_dir", symlink_to_symlink_to_dir) + symlink("symlink_to_file", symlink_to_symlink_to_file) assert fs.lexists_islink_isdir(dir) == (True, False, True) assert fs.lexists_islink_isdir(file) == (True, False, False) assert fs.lexists_islink_isdir(nonexistent) == (False, False, False) assert fs.lexists_islink_isdir(symlink_to_dir) == (True, True, True) assert fs.lexists_islink_isdir(symlink_to_file) == (True, True, False) - assert fs.lexists_islink_isdir(symlink_to_dangling_symlink) == (True, True, False) + assert fs.lexists_islink_isdir(symlink_to_dangling_symlink) == (False, False, False) + assert fs.lexists_islink_isdir(symlink_to_symlink_to_dir) == (True, True, True) + assert fs.lexists_islink_isdir(symlink_to_symlink_to_file) == (True, True, False) + + +@pytest.mark.skipif(_windows_can_symlink(), reason="Not to be run with elevated privileges.") +@pytest.mark.skipif(sys.platform != "win32", reason="For Windows Only") +def test_lexists_islink_isdir__win32_base(tmpdir): + """Run on windows without elevated privileges to test junctions and hard links which have + different results from the lexists_islink_isdir method. + """ + root = str(tmpdir) + + # Create a directory and a file, an a bunch of symlinks. + dir = os.path.join(root, "dir") + file = os.path.join(root, "file") + nonexistent = os.path.join(root, "does_not_exist") + symlink_to_dir = os.path.join(root, "symlink_to_dir") + symlink_to_file = os.path.join(root, "symlink_to_file") + dangling_symlink = os.path.join(root, "dangling_symlink") + symlink_to_dangling_symlink = os.path.join(root, "symlink_to_dangling_symlink") + symlink_to_symlink_to_dir = os.path.join(root, "symlink_to_symlink_to_dir") + symlink_to_symlink_to_file = os.path.join(root, "symlink_to_symlink_to_file") + + os.mkdir(dir) + assert fs.lexists_islink_isdir(dir) == (True, False, True) + + symlink("dir", symlink_to_dir) + assert fs.lexists_islink_isdir(dir) == (True, False, True) + assert fs.lexists_islink_isdir(symlink_to_dir) == (True, True, True) + + with open(file, "wb") as f: + f.write(b"file") + assert fs.lexists_islink_isdir(file) == (True, False, False) + + symlink("file", symlink_to_file) + assert fs.lexists_islink_isdir(file) == (True, True, False) + assert fs.lexists_islink_isdir(symlink_to_file) == (True, True, False) + + with pytest.raises(SymlinkError): + symlink("does_not_exist", dangling_symlink) + symlink("dangling_symlink", symlink_to_dangling_symlink) + + symlink("symlink_to_dir", symlink_to_symlink_to_dir) + symlink("symlink_to_file", symlink_to_symlink_to_file) + + assert fs.lexists_islink_isdir(nonexistent) == (False, False, False) + assert fs.lexists_islink_isdir(symlink_to_dangling_symlink) == (False, False, False) assert fs.lexists_islink_isdir(symlink_to_symlink_to_dir) == (True, True, True) assert fs.lexists_islink_isdir(symlink_to_symlink_to_file) == (True, True, False) diff --git a/lib/spack/spack/test/llnl/util/link_tree.py b/lib/spack/spack/test/llnl/util/link_tree.py index 3ae39bbb79..9d154014b1 100644 --- a/lib/spack/spack/test/llnl/util/link_tree.py +++ b/lib/spack/spack/test/llnl/util/link_tree.py @@ -4,12 +4,14 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os +import sys import pytest +import llnl.util.symlink from llnl.util.filesystem import mkdirp, touchp, visit_directory_tree, working_dir from llnl.util.link_tree import DestinationMergeVisitor, LinkTree, SourceMergeVisitor -from llnl.util.symlink import islink +from llnl.util.symlink import _windows_can_symlink, islink, readlink, symlink from spack.stage import Stage @@ -44,77 +46,116 @@ def link_tree(stage): def check_file_link(filename, expected_target): assert os.path.isfile(filename) assert islink(filename) - assert os.path.abspath(os.path.realpath(filename)) == os.path.abspath(expected_target) + if sys.platform != "win32" or llnl.util.symlink._windows_can_symlink(): + assert os.path.abspath(os.path.realpath(filename)) == os.path.abspath(expected_target) def check_dir(filename): assert os.path.isdir(filename) -def test_merge_to_new_directory(stage, link_tree): +@pytest.mark.parametrize("run_as_root", [True, False]) +def test_merge_to_new_directory(stage, link_tree, monkeypatch, run_as_root): + if sys.platform != "win32": + if run_as_root: + pass + else: + pytest.skip("Skipping duplicate test.") + elif _windows_can_symlink() or not run_as_root: + monkeypatch.setattr(llnl.util.symlink, "_windows_can_symlink", lambda: run_as_root) + else: + # Skip if trying to run as dev-mode without having dev-mode. + pytest.skip("Skipping portion of test which required dev-mode privileges.") + with working_dir(stage.path): link_tree.merge("dest") - check_file_link("dest/1", "source/1") - check_file_link("dest/a/b/2", "source/a/b/2") - check_file_link("dest/a/b/3", "source/a/b/3") - check_file_link("dest/c/4", "source/c/4") - check_file_link("dest/c/d/5", "source/c/d/5") - check_file_link("dest/c/d/6", "source/c/d/6") - check_file_link("dest/c/d/e/7", "source/c/d/e/7") - - assert os.path.isabs(os.readlink("dest/1")) - assert os.path.isabs(os.readlink("dest/a/b/2")) - assert os.path.isabs(os.readlink("dest/a/b/3")) - assert os.path.isabs(os.readlink("dest/c/4")) - assert os.path.isabs(os.readlink("dest/c/d/5")) - assert os.path.isabs(os.readlink("dest/c/d/6")) - assert os.path.isabs(os.readlink("dest/c/d/e/7")) + files = [ + ("dest/1", "source/1"), + ("dest/a/b/2", "source/a/b/2"), + ("dest/a/b/3", "source/a/b/3"), + ("dest/c/4", "source/c/4"), + ("dest/c/d/5", "source/c/d/5"), + ("dest/c/d/6", "source/c/d/6"), + ("dest/c/d/e/7", "source/c/d/e/7"), + ] + + for dest, source in files: + check_file_link(dest, source) + assert os.path.isabs(readlink(dest)) link_tree.unmerge("dest") assert not os.path.exists("dest") -def test_merge_to_new_directory_relative(stage, link_tree): +@pytest.mark.parametrize("run_as_root", [True, False]) +def test_merge_to_new_directory_relative(stage, link_tree, monkeypatch, run_as_root): + if sys.platform != "win32": + if run_as_root: + pass + else: + pytest.skip("Skipping duplicate test.") + elif _windows_can_symlink() or not run_as_root: + monkeypatch.setattr(llnl.util.symlink, "_windows_can_symlink", lambda: run_as_root) + else: + # Skip if trying to run as dev-mode without having dev-mode. + pytest.skip("Skipping portion of test which required dev-mode privileges.") + with working_dir(stage.path): link_tree.merge("dest", relative=True) - check_file_link("dest/1", "source/1") - check_file_link("dest/a/b/2", "source/a/b/2") - check_file_link("dest/a/b/3", "source/a/b/3") - check_file_link("dest/c/4", "source/c/4") - check_file_link("dest/c/d/5", "source/c/d/5") - check_file_link("dest/c/d/6", "source/c/d/6") - check_file_link("dest/c/d/e/7", "source/c/d/e/7") - - assert not os.path.isabs(os.readlink("dest/1")) - assert not os.path.isabs(os.readlink("dest/a/b/2")) - assert not os.path.isabs(os.readlink("dest/a/b/3")) - assert not os.path.isabs(os.readlink("dest/c/4")) - assert not os.path.isabs(os.readlink("dest/c/d/5")) - assert not os.path.isabs(os.readlink("dest/c/d/6")) - assert not os.path.isabs(os.readlink("dest/c/d/e/7")) + files = [ + ("dest/1", "source/1"), + ("dest/a/b/2", "source/a/b/2"), + ("dest/a/b/3", "source/a/b/3"), + ("dest/c/4", "source/c/4"), + ("dest/c/d/5", "source/c/d/5"), + ("dest/c/d/6", "source/c/d/6"), + ("dest/c/d/e/7", "source/c/d/e/7"), + ] + + for dest, source in files: + check_file_link(dest, source) + # Hard links/junctions are inherently absolute. + if sys.platform != "win32" or run_as_root: + assert not os.path.isabs(readlink(dest)) link_tree.unmerge("dest") assert not os.path.exists("dest") -def test_merge_to_existing_directory(stage, link_tree): +@pytest.mark.parametrize("run_as_root", [True, False]) +def test_merge_to_existing_directory(stage, link_tree, monkeypatch, run_as_root): + if sys.platform != "win32": + if run_as_root: + pass + else: + pytest.skip("Skipping duplicate test.") + elif _windows_can_symlink() or not run_as_root: + monkeypatch.setattr(llnl.util.symlink, "_windows_can_symlink", lambda: run_as_root) + else: + # Skip if trying to run as dev-mode without having dev-mode. + pytest.skip("Skipping portion of test which required dev-mode privileges.") + with working_dir(stage.path): touchp("dest/x") touchp("dest/a/b/y") link_tree.merge("dest") - check_file_link("dest/1", "source/1") - check_file_link("dest/a/b/2", "source/a/b/2") - check_file_link("dest/a/b/3", "source/a/b/3") - check_file_link("dest/c/4", "source/c/4") - check_file_link("dest/c/d/5", "source/c/d/5") - check_file_link("dest/c/d/6", "source/c/d/6") - check_file_link("dest/c/d/e/7", "source/c/d/e/7") + files = [ + ("dest/1", "source/1"), + ("dest/a/b/2", "source/a/b/2"), + ("dest/a/b/3", "source/a/b/3"), + ("dest/c/4", "source/c/4"), + ("dest/c/d/5", "source/c/d/5"), + ("dest/c/d/6", "source/c/d/6"), + ("dest/c/d/e/7", "source/c/d/e/7"), + ] + for dest, source in files: + check_file_link(dest, source) assert os.path.isfile("dest/x") assert os.path.isfile("dest/a/b/y") @@ -124,13 +165,8 @@ def test_merge_to_existing_directory(stage, link_tree): assert os.path.isfile("dest/x") assert os.path.isfile("dest/a/b/y") - assert not os.path.isfile("dest/1") - assert not os.path.isfile("dest/a/b/2") - assert not os.path.isfile("dest/a/b/3") - assert not os.path.isfile("dest/c/4") - assert not os.path.isfile("dest/c/d/5") - assert not os.path.isfile("dest/c/d/6") - assert not os.path.isfile("dest/c/d/e/7") + for dest, _ in files: + assert not os.path.isfile(dest) def test_merge_with_empty_directories(stage, link_tree): @@ -192,9 +228,9 @@ def test_source_merge_visitor_does_not_follow_symlinked_dirs_at_depth(tmpdir): os.mkdir(j("a", "b")) os.mkdir(j("a", "b", "c")) os.mkdir(j("a", "b", "c", "d")) - os.symlink(j("b"), j("a", "symlink_b")) - os.symlink(j("c"), j("a", "b", "symlink_c")) - os.symlink(j("d"), j("a", "b", "c", "symlink_d")) + symlink(j("b"), j("a", "symlink_b")) + symlink(j("c"), j("a", "b", "symlink_c")) + symlink(j("d"), j("a", "b", "c", "symlink_d")) with open(j("a", "b", "c", "d", "file"), "wb"): pass @@ -236,10 +272,11 @@ def test_source_merge_visitor_cant_be_cyclical(tmpdir): j = os.path.join with tmpdir.as_cwd(): os.mkdir(j("a")) - os.symlink(j("..", "b"), j("a", "symlink_b")) - os.symlink(j("symlink_b"), j("a", "symlink_b_b")) os.mkdir(j("b")) - os.symlink(j("..", "a"), j("b", "symlink_a")) + + symlink(j("..", "b"), j("a", "symlink_b")) + symlink(j("symlink_b"), j("a", "symlink_b_b")) + symlink(j("..", "a"), j("b", "symlink_a")) visitor = SourceMergeVisitor() visit_directory_tree(str(tmpdir), visitor) diff --git a/lib/spack/spack/test/llnl/util/symlink.py b/lib/spack/spack/test/llnl/util/symlink.py new file mode 100644 index 0000000000..28e4ea4ab9 --- /dev/null +++ b/lib/spack/spack/test/llnl/util/symlink.py @@ -0,0 +1,246 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Tests for ``llnl/util/symlink.py``""" +import os +import sys +import tempfile + +import pytest + +from llnl.util import symlink + + +def test_symlink_file(tmpdir): + """Test the symlink.symlink functionality on all operating systems for a file""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir) + link_file = str(tmpdir.join("link.txt")) + assert os.path.exists(link_file) is False + symlink.symlink(source_path=real_file, link_path=link_file) + assert os.path.exists(link_file) + assert symlink.islink(link_file) + + +def test_symlink_dir(tmpdir): + """Test the symlink.symlink functionality on all operating systems for a directory""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + real_dir = os.path.join(test_dir, "real_dir") + link_dir = os.path.join(test_dir, "link_dir") + os.mkdir(real_dir) + symlink.symlink(source_path=real_dir, link_path=link_dir) + assert os.path.exists(link_dir) + assert symlink.islink(link_dir) + + +def test_symlink_source_not_exists(tmpdir): + """Test the symlink.symlink method for the case where a source path does not exist""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + real_dir = os.path.join(test_dir, "real_dir") + link_dir = os.path.join(test_dir, "link_dir") + with pytest.raises(symlink.SymlinkError): + symlink.symlink(source_path=real_dir, link_path=link_dir) + + +def test_symlink_src_relative_to_link(tmpdir): + """Test the symlink.symlink functionality where the source value exists relative to the link + but not relative to the cwd""" + with tmpdir.as_cwd(): + subdir_1 = tmpdir.join("a") + subdir_2 = os.path.join(subdir_1, "b") + link_dir = os.path.join(subdir_1, "c") + + os.mkdir(subdir_1) + os.mkdir(subdir_2) + + fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=subdir_2) + link_file = os.path.join(subdir_1, "link.txt") + + symlink.symlink( + source_path=f"b/{os.path.basename(real_file)}", + link_path=f"a/{os.path.basename(link_file)}", + ) + assert os.path.exists(link_file) + assert symlink.islink(link_file) + # Check dirs + assert not os.path.lexists(link_dir) + symlink.symlink(source_path="b", link_path="a/c") + assert os.path.lexists(link_dir) + + +def test_symlink_src_not_relative_to_link(tmpdir): + """Test the symlink.symlink functionality where the source value does not exist relative to + the link and not relative to the cwd. NOTE that this symlink api call is EXPECTED to raise + a symlink.SymlinkError exception that we catch.""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + subdir_1 = os.path.join(test_dir, "a") + subdir_2 = os.path.join(subdir_1, "b") + link_dir = os.path.join(subdir_1, "c") + os.mkdir(subdir_1) + os.mkdir(subdir_2) + fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=subdir_2) + link_file = str(tmpdir.join("link.txt")) + # Expected SymlinkError because source path does not exist relative to link path + with pytest.raises(symlink.SymlinkError): + symlink.symlink( + source_path=f"d/{os.path.basename(real_file)}", + link_path=f"a/{os.path.basename(link_file)}", + ) + assert not os.path.exists(link_file) + # Check dirs + assert not os.path.lexists(link_dir) + with pytest.raises(symlink.SymlinkError): + symlink.symlink(source_path="d", link_path="a/c") + assert not os.path.lexists(link_dir) + + +def test_symlink_link_already_exists(tmpdir): + """Test the symlink.symlink method for the case where a link already exists""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + real_dir = os.path.join(test_dir, "real_dir") + link_dir = os.path.join(test_dir, "link_dir") + os.mkdir(real_dir) + symlink.symlink(real_dir, link_dir) + assert os.path.exists(link_dir) + with pytest.raises(symlink.SymlinkError): + symlink.symlink(source_path=real_dir, link_path=link_dir) + + +@pytest.mark.skipif(not symlink._windows_can_symlink(), reason="Test requires elevated privileges") +@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows") +def test_symlink_win_file(tmpdir): + """Check that symlink.symlink makes a symlink file when run with elevated permissions""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir) + link_file = str(tmpdir.join("link.txt")) + symlink.symlink(source_path=real_file, link_path=link_file) + # Verify that all expected conditions are met + assert os.path.exists(link_file) + assert symlink.islink(link_file) + assert os.path.islink(link_file) + assert not symlink._windows_is_hardlink(link_file) + assert not symlink._windows_is_junction(link_file) + + +@pytest.mark.skipif(not symlink._windows_can_symlink(), reason="Test requires elevated privileges") +@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows") +def test_symlink_win_dir(tmpdir): + """Check that symlink.symlink makes a symlink dir when run with elevated permissions""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + real_dir = os.path.join(test_dir, "real") + link_dir = os.path.join(test_dir, "link") + os.mkdir(real_dir) + symlink.symlink(source_path=real_dir, link_path=link_dir) + # Verify that all expected conditions are met + assert os.path.exists(link_dir) + assert symlink.islink(link_dir) + assert os.path.islink(link_dir) + assert not symlink._windows_is_hardlink(link_dir) + assert not symlink._windows_is_junction(link_dir) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows") +def test_windows_create_junction(tmpdir): + """Test the symlink._windows_create_junction method""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + junction_real_dir = os.path.join(test_dir, "real_dir") + junction_link_dir = os.path.join(test_dir, "link_dir") + os.mkdir(junction_real_dir) + symlink._windows_create_junction(junction_real_dir, junction_link_dir) + # Verify that all expected conditions are met + assert os.path.exists(junction_link_dir) + assert symlink._windows_is_junction(junction_link_dir) + assert symlink.islink(junction_link_dir) + assert not os.path.islink(junction_link_dir) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows") +def test_windows_create_hard_link(tmpdir): + """Test the symlink._windows_create_hard_link method""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir) + link_file = str(tmpdir.join("link.txt")) + symlink._windows_create_hard_link(real_file, link_file) + # Verify that all expected conditions are met + assert os.path.exists(link_file) + assert symlink._windows_is_hardlink(real_file) + assert symlink._windows_is_hardlink(link_file) + assert symlink.islink(link_file) + assert not os.path.islink(link_file) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows") +def test_windows_create_link_dir(tmpdir): + """Test the functionality of the windows_create_link method with a directory + which should result in making a junction. + """ + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + real_dir = os.path.join(test_dir, "real") + link_dir = os.path.join(test_dir, "link") + os.mkdir(real_dir) + symlink._windows_create_link(real_dir, link_dir) + # Verify that all expected conditions are met + assert os.path.exists(link_dir) + assert symlink.islink(link_dir) + assert not symlink._windows_is_hardlink(link_dir) + assert symlink._windows_is_junction(link_dir) + assert not os.path.islink(link_dir) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows") +def test_windows_create_link_file(tmpdir): + """Test the functionality of the windows_create_link method with a file + which should result in the creation of a hard link. It also tests the + functionality of the symlink islink infrastructure. + """ + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir) + link_file = str(tmpdir.join("link.txt")) + symlink._windows_create_link(real_file, link_file) + # Verify that all expected conditions are met + assert symlink._windows_is_hardlink(link_file) + assert symlink.islink(link_file) + assert not symlink._windows_is_junction(link_file) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows") +def test_windows_read_link(tmpdir): + """Makes sure symlink.readlink can read the link source for hard links and + junctions on windows.""" + with tmpdir.as_cwd(): + real_dir_1 = "real_dir_1" + real_dir_2 = "real_dir_2" + link_dir_1 = "link_dir_1" + link_dir_2 = "link_dir_2" + os.mkdir(real_dir_1) + os.mkdir(real_dir_2) + + # Create a file and a directory + _, real_file_1 = tempfile.mkstemp(prefix="real_1", suffix=".txt", dir=".") + _, real_file_2 = tempfile.mkstemp(prefix="real_2", suffix=".txt", dir=".") + link_file_1 = "link_1.txt" + link_file_2 = "link_2.txt" + + # Make hard link/junction + symlink._windows_create_hard_link(real_file_1, link_file_1) + symlink._windows_create_hard_link(real_file_2, link_file_2) + symlink._windows_create_junction(real_dir_1, link_dir_1) + symlink._windows_create_junction(real_dir_2, link_dir_2) + + assert symlink.readlink(link_file_1) == os.path.abspath(real_file_1) + assert symlink.readlink(link_file_2) == os.path.abspath(real_file_2) + assert symlink.readlink(link_dir_1) == os.path.abspath(real_dir_1) + assert symlink.readlink(link_dir_2) == os.path.abspath(real_dir_2) diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 2da1032d45..c66af4f125 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -9,7 +9,7 @@ import sys import pytest -from llnl.util.filesystem import resolve_link_target_relative_to_the_link +from llnl.util.symlink import resolve_link_target_relative_to_the_link import spack.mirror import spack.repo @@ -224,6 +224,9 @@ def test_mirror_with_url_patches(mock_packages, config, monkeypatch): def successful_apply(*args, **kwargs): pass + def successful_symlink(*args, **kwargs): + pass + with Stage("spack-mirror-test") as stage: mirror_root = os.path.join(stage.path, "test-mirror") @@ -231,6 +234,7 @@ def test_mirror_with_url_patches(mock_packages, config, monkeypatch): monkeypatch.setattr(spack.fetch_strategy.URLFetchStrategy, "expand", successful_expand) monkeypatch.setattr(spack.patch, "apply_patch", successful_apply) monkeypatch.setattr(spack.caches.MirrorCache, "store", record_store) + monkeypatch.setattr(spack.caches.MirrorCache, "symlink", successful_symlink) with spack.config.override("config:checksum", False): spack.mirror.create(mirror_root, list(spec.traverse())) diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py index 43193b578c..2999ff0608 100644 --- a/lib/spack/spack/test/packaging.py +++ b/lib/spack/spack/test/packaging.py @@ -16,7 +16,7 @@ from collections import OrderedDict import pytest -from llnl.util.filesystem import mkdirp +from llnl.util.filesystem import mkdirp, touchp from llnl.util.symlink import symlink import spack.binary_distribution as bindist @@ -188,8 +188,15 @@ def test_relocate_links(tmpdir): own_prefix_path = str(tmpdir.join("prefix_a", "file")) dep_prefix_path = str(tmpdir.join("prefix_b", "file")) + new_own_prefix_path = str(tmpdir.join("new_prefix_a", "file")) + new_dep_prefix_path = str(tmpdir.join("new_prefix_b", "file")) system_path = os.path.join(os.path.sep, "system", "path") + touchp(own_prefix_path) + touchp(new_own_prefix_path) + touchp(dep_prefix_path) + touchp(new_dep_prefix_path) + # Old prefixes to new prefixes prefix_to_prefix = OrderedDict( [ -- cgit v1.2.3-60-g2f50