From 638cc64571fd038f5ae7f47a664399b9f2629d66 Mon Sep 17 00:00:00 2001 From: scheibelp Date: Fri, 17 Aug 2018 22:08:38 -0400 Subject: install_tree: symlink handling and add 'ignore' option (#9019) Fixes #9001 #8289 added support for install_tree and copy_tree to merge into an existing directory structure. However, it did not properly handle relative symlinks and also removed support for the 'ignore' keyword. Additionally, some of the tests were overly-strict when checking the permissions on the copied files. This updates the install_tree/copy_tree methods and their tests: * copy_tree/install_tree now preserve relative link targets (if the symlink in the source directory structure is relative, the symlink created in the destination will be relative) * Added support for 'ignore' argument back to copy_tree/install_tree (removed in #8289). It is no longer the object output by shutil.ignore_patterns: you pass a function that accepts a path relative to the source and returns whether that path should be copied. * The openfoam packages (currently the only ones making use of the 'ignore' argument) are updated for the new API * When a symlink target is absolute, copy_tree and install_tree now rewrite the source prefix to be the destination prefix * copy_tree tests no longer check permissions: copy_tree doesn't enforce anything about permissions so its tests don't check for that * install_tree tests no longer check for exact permission matching since it can add file permissions --- lib/spack/llnl/util/filesystem.py | 68 +++++++++++++++++----- lib/spack/spack/test/llnl/util/filesystem.py | 37 ++++++++++-- .../repos/builtin/packages/foam-extend/package.py | 11 ++-- .../repos/builtin/packages/openfoam-com/package.py | 6 +- .../repos/builtin/packages/openfoam-org/package.py | 6 +- 5 files changed, 98 insertions(+), 30 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 40eb6a7aa8..7dacb9804f 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -306,7 +306,21 @@ def install(src, dest): copy(src, dest, _permissions=True) -def copy_tree(src, dest, symlinks=True, _permissions=False): +def resolve_link_target_relative_to_the_link(l): + """ + os.path.isdir uses os.path.exists, which for links will check + the existence of the link target. If the link target is relative to + the link, we need to construct a pathname that is valid from + our cwd (which may not be the same as the link's directory) + """ + target = os.readlink(l) + if os.path.isabs(target): + return target + link_dir = os.path.dirname(os.path.abspath(l)) + return os.path.join(link_dir, target) + + +def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False): """Recursively copy an entire directory tree rooted at *src*. If the destination directory *dest* does not already exist, it will @@ -317,10 +331,14 @@ def copy_tree(src, dest, symlinks=True, _permissions=False): will be copied as far as the platform allows; if false, the contents and metadata of the linked files are copied to the new tree. + If *ignore* is set, then each path relative to *src* will be passed to + this function; the function returns whether that path should be skipped. + Parameters: src (str): the directory to copy dest (str): the destination directory symlinks (bool): whether or not to preserve symlinks + ignore (function): function indicating which files to ignore _permissions (bool): for internal use only """ if _permissions: @@ -330,23 +348,41 @@ def copy_tree(src, dest, symlinks=True, _permissions=False): mkdirp(dest) - for s, d in traverse_tree(src, dest, order='pre', follow_nonexisting=True): - if symlinks and os.path.islink(s): - # Note that this won't rewrite absolute links into the old - # root to point at the new root. Should we handle that case? - target = os.readlink(s) - os.symlink(os.path.abspath(target), d) - elif os.path.isdir(s): - mkdirp(d) + src = os.path.abspath(src) + dest = os.path.abspath(dest) + + for s, d in traverse_tree(src, dest, order='pre', + follow_symlinks=not symlinks, + ignore=ignore, + follow_nonexisting=True): + if os.path.islink(s): + link_target = resolve_link_target_relative_to_the_link(s) + if symlinks: + target = os.readlink(s) + if os.path.isabs(target): + new_target = re.sub(src, dest, target) + if new_target != target: + tty.debug("Redirecting link {0} to {1}" + .format(target, new_target)) + target = new_target + + os.symlink(target, d) + elif os.path.isdir(link_target): + mkdirp(d) + else: + shutil.copyfile(s, d) else: - shutil.copyfile(s, d) + if os.path.isdir(s): + mkdirp(d) + else: + shutil.copyfile(s, d) if _permissions: set_install_permissions(d) copy_mode(s, d) -def install_tree(src, dest, symlinks=True): +def install_tree(src, dest, symlinks=True, ignore=None): """Recursively install an entire directory tree rooted at *src*. Same as :py:func:`copy_tree` with the addition of setting proper @@ -356,8 +392,9 @@ def install_tree(src, dest, symlinks=True): src (str): the directory to install dest (str): the destination directory symlinks (bool): whether or not to preserve symlinks + ignore (function): function indicating which files to ignore """ - copy_tree(src, dest, symlinks, _permissions=True) + copy_tree(src, dest, symlinks=symlinks, ignore=ignore, _permissions=True) def is_exe(path): @@ -564,7 +601,7 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): Keyword Arguments: order (str): Whether to do pre- or post-order traversal. Accepted values are 'pre' and 'post' - ignore (str): Predicate indicating which files to ignore + ignore (function): function indicating which files to ignore follow_nonexisting (bool): Whether to descend into directories in ``src`` that do not exit in ``dest``. Default is True follow_links (bool): Whether to descend into symlinks in ``src`` @@ -578,7 +615,7 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): raise ValueError("Order must be 'pre' or 'post'.") # List of relative paths to ignore under the src root. - ignore = kwargs.get('ignore', lambda filename: False) + ignore = kwargs.get('ignore', None) or (lambda filename: False) # Don't descend into ignored directories if ignore(rel_path): @@ -597,6 +634,9 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): rel_child = os.path.join(rel_path, f) # Treat as a directory + # TODO: for symlinks, os.path.isdir looks for the link target. If the + # target is relative to the link, then that may not resolve properly + # relative to our cwd - see resolve_link_target_relative_to_the_link if os.path.isdir(source_child) and ( follow_links or not os.path.islink(source_child)): diff --git a/lib/spack/spack/test/llnl/util/filesystem.py b/lib/spack/spack/test/llnl/util/filesystem.py index 7701185dbe..24ab02a613 100644 --- a/lib/spack/spack/test/llnl/util/filesystem.py +++ b/lib/spack/spack/test/llnl/util/filesystem.py @@ -26,6 +26,7 @@ import llnl.util.filesystem as fs import os +import stat import pytest @@ -45,8 +46,10 @@ def stage(tmpdir_factory): fs.touchp('source/c/d/6') fs.touchp('source/c/d/e/7') - # Create symlink + # Create symlinks os.symlink(os.path.abspath('source/1'), 'source/2') + os.symlink('b/2', 'source/a/b2') + os.symlink('a/b', 'source/f') # Create destination directory fs.mkdirp('dest') @@ -64,7 +67,6 @@ class TestCopy: fs.copy('source/1', 'dest/1') assert os.path.exists('dest/1') - assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode def test_dir_dest(self, stage): """Test using a directory as the destination.""" @@ -73,7 +75,14 @@ class TestCopy: fs.copy('source/1', 'dest') assert os.path.exists('dest/1') - assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode + + +def check_added_exe_permissions(src, dst): + src_mode = os.stat(src).st_mode + dst_mode = os.stat(dst).st_mode + for perm in [stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH]: + if src_mode & perm: + assert dst_mode & perm class TestInstall: @@ -86,7 +95,7 @@ class TestInstall: fs.install('source/1', 'dest/1') assert os.path.exists('dest/1') - assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode + check_added_exe_permissions('source/1', 'dest/1') def test_dir_dest(self, stage): """Test using a directory as the destination.""" @@ -95,7 +104,7 @@ class TestInstall: fs.install('source/1', 'dest') assert os.path.exists('dest/1') - assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode + check_added_exe_permissions('source/1', 'dest/1') class TestCopyTree: @@ -126,6 +135,24 @@ class TestCopyTree: assert os.path.exists('dest/2') assert os.path.islink('dest/2') + assert os.path.exists('dest/a/b2') + with fs.working_dir('dest/a'): + assert os.path.exists(os.readlink('b2')) + + assert (os.path.realpath('dest/f/2') == + os.path.abspath('dest/a/b/2')) + assert os.path.realpath('dest/2') == os.path.abspath('dest/1') + + def test_symlinks_true_ignore(self, stage): + """Test copying when specifying relative paths that should be ignored + """ + with fs.working_dir(str(stage)): + ignore = lambda p: p in ['c/d/e', 'a'] + fs.copy_tree('source', 'dest', symlinks=True, ignore=ignore) + assert not os.path.exists('dest/a') + assert os.path.exists('dest/c/d') + assert not os.path.exists('dest/c/d/e') + def test_symlinks_false(self, stage): """Test copying without symlink preservation.""" diff --git a/var/spack/repos/builtin/packages/foam-extend/package.py b/var/spack/repos/builtin/packages/foam-extend/package.py index 7de30232a3..2821c58eb9 100644 --- a/var/spack/repos/builtin/packages/foam-extend/package.py +++ b/var/spack/repos/builtin/packages/foam-extend/package.py @@ -54,7 +54,6 @@ ############################################################################## import glob import re -import shutil import os from spack import * @@ -382,7 +381,6 @@ class FoamExtend(Package): def install(self, spec, prefix): """Install under the projectdir""" - opts = str(self.foam_arch) # Fairly ugly since intermediate targets are scattered inside sources appdir = 'applications' @@ -419,19 +417,22 @@ class FoamExtend(Package): subitem = join_path(appdir, 'Allwmake') install(subitem, join_path(self.projectdir, subitem)) - ignored = [opts] # Ignore intermediate targets + foam_arch_str = str(self.foam_arch) + # Ignore intermediate targets + ignore = lambda p: os.path.basename(p) == foam_arch_str + for d in ['src', 'tutorials']: install_tree( d, join_path(self.projectdir, d), - ignore=shutil.ignore_patterns(*ignored), + ignore=ignore, symlinks=True) for d in ['solvers', 'utilities']: install_tree( join_path(appdir, d), join_path(self.projectdir, appdir, d), - ignore=shutil.ignore_patterns(*ignored), + ignore=ignore, symlinks=True) etc_dir = join_path(self.projectdir, 'etc') diff --git a/var/spack/repos/builtin/packages/openfoam-com/package.py b/var/spack/repos/builtin/packages/openfoam-com/package.py index 08d5dbcf8b..7613d688f7 100644 --- a/var/spack/repos/builtin/packages/openfoam-com/package.py +++ b/var/spack/repos/builtin/packages/openfoam-com/package.py @@ -60,7 +60,6 @@ ############################################################################## import glob import re -import shutil import os from spack import * @@ -692,12 +691,13 @@ class OpenfoamCom(Package): dirs.extend(['doc']) # Install platforms (and doc) skipping intermediate targets - ignored = ['src', 'applications', 'html', 'Guides'] + relative_ignore_paths = ['src', 'applications', 'html', 'Guides'] + ignore = lambda p: p in relative_ignore_paths for d in dirs: install_tree( d, join_path(self.projectdir, d), - ignore=shutil.ignore_patterns(*ignored), + ignore=ignore, symlinks=True) etc_dir = join_path(self.projectdir, 'etc') diff --git a/var/spack/repos/builtin/packages/openfoam-org/package.py b/var/spack/repos/builtin/packages/openfoam-org/package.py index dfb3af51f6..9db7597c49 100644 --- a/var/spack/repos/builtin/packages/openfoam-org/package.py +++ b/var/spack/repos/builtin/packages/openfoam-org/package.py @@ -55,7 +55,6 @@ ############################################################################## import glob import re -import shutil import os import llnl.util.tty as tty @@ -345,12 +344,13 @@ class OpenfoamOrg(Package): dirs.extend(['doc']) # Install platforms (and doc) skipping intermediate targets - ignored = ['src', 'applications', 'html', 'Guides'] + relative_ignore_paths = ['src', 'applications', 'html', 'Guides'] + ignore = lambda p: p in relative_ignore_paths for d in dirs: install_tree( d, join_path(self.projectdir, d), - ignore=shutil.ignore_patterns(*ignored), + ignore=ignore, symlinks=True) etc_dir = join_path(self.projectdir, 'etc') -- cgit v1.2.3-70-g09d2