From 73c978ddd99973e29f2ba42078b10455c1de5ca8 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 15 Aug 2018 11:30:09 -0500 Subject: install_tree, copy_tree can install into existing directory structures (#8289) Replace use of `shutil.copytree` with `copy_tree` and `install_tree` functions in `llnl.util.filesystem`. - `copy_tree` copies without setting permissions. It should be used to copy files around in the build directory. - `install_tree` copies files and sets permissions. It should be used to copy files into the installation directory. - `install` and `copy` are analogous single-file functions. - add more extensive tests for these functions - update packages to use these functions. --- lib/spack/llnl/util/filesystem.py | 97 +++++++++++-- lib/spack/spack/test/llnl/util/filesystem.py | 209 +++++++++++++++++++++++++++ lib/spack/spack/test/util/filesystem.py | 61 -------- 3 files changed, 294 insertions(+), 73 deletions(-) create mode 100644 lib/spack/spack/test/llnl/util/filesystem.py delete mode 100644 lib/spack/spack/test/util/filesystem.py (limited to 'lib') diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index ace6cfded6..40eb6a7aa8 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -60,7 +60,9 @@ __all__ = [ 'fix_darwin_install_name', 'force_remove', 'force_symlink', + 'copy', 'install', + 'copy_tree', 'install_tree', 'is_exe', 'join_path', @@ -264,27 +266,98 @@ def unset_executable_mode(path): os.chmod(path, mode) -def install(src, dest): - """Manually install a file to a particular location.""" - tty.debug("Installing %s to %s" % (src, dest)) +def copy(src, dest, _permissions=False): + """Copies the file *src* to the file or directory *dest*. + + If *dest* specifies a directory, the file will be copied into *dest* + using the base filename from *src*. + + Parameters: + src (str): the file to copy + dest (str): the destination file or directory + _permissions (bool): for internal use only + """ + if _permissions: + tty.debug('Installing {0} to {1}'.format(src, dest)) + else: + tty.debug('Copying {0} to {1}'.format(src, dest)) # Expand dest to its eventual full path if it is a directory. if os.path.isdir(dest): dest = join_path(dest, os.path.basename(src)) shutil.copy(src, dest) - set_install_permissions(dest) - copy_mode(src, dest) + + if _permissions: + set_install_permissions(dest) + copy_mode(src, dest) + + +def install(src, dest): + """Installs the file *src* to the file or directory *dest*. + + Same as :py:func:`copy` with the addition of setting proper + permissions on the installed file. + + Parameters: + src (str): the file to install + dest (str): the destination file or directory + """ + copy(src, dest, _permissions=True) -def install_tree(src, dest, **kwargs): - """Manually install a directory tree to a particular location.""" - tty.debug("Installing %s to %s" % (src, dest)) - shutil.copytree(src, dest, **kwargs) +def copy_tree(src, dest, symlinks=True, _permissions=False): + """Recursively copy an entire directory tree rooted at *src*. - for s, d in traverse_tree(src, dest, follow_nonexisting=False): - set_install_permissions(d) - copy_mode(s, d) + If the destination directory *dest* does not already exist, it will + be created as well as missing parent directories. + + If *symlinks* is true, symbolic links in the source tree are represented + as symbolic links in the new tree and the metadata of the original links + will be copied as far as the platform allows; if false, the contents and + metadata of the linked files are copied to the new tree. + + Parameters: + src (str): the directory to copy + dest (str): the destination directory + symlinks (bool): whether or not to preserve symlinks + _permissions (bool): for internal use only + """ + if _permissions: + tty.debug('Installing {0} to {1}'.format(src, dest)) + else: + tty.debug('Copying {0} to {1}'.format(src, dest)) + + mkdirp(dest) + + for s, d in traverse_tree(src, dest, order='pre', follow_nonexisting=True): + if symlinks and os.path.islink(s): + # Note that this won't rewrite absolute links into the old + # root to point at the new root. Should we handle that case? + target = os.readlink(s) + os.symlink(os.path.abspath(target), d) + elif os.path.isdir(s): + mkdirp(d) + else: + shutil.copyfile(s, d) + + if _permissions: + set_install_permissions(d) + copy_mode(s, d) + + +def install_tree(src, dest, symlinks=True): + """Recursively install an entire directory tree rooted at *src*. + + Same as :py:func:`copy_tree` with the addition of setting proper + permissions on the installed files and directories. + + Parameters: + src (str): the directory to install + dest (str): the destination directory + symlinks (bool): whether or not to preserve symlinks + """ + copy_tree(src, dest, symlinks, _permissions=True) def is_exe(path): diff --git a/lib/spack/spack/test/llnl/util/filesystem.py b/lib/spack/spack/test/llnl/util/filesystem.py new file mode 100644 index 0000000000..7701185dbe --- /dev/null +++ b/lib/spack/spack/test/llnl/util/filesystem.py @@ -0,0 +1,209 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Tests for ``llnl/util/filesystem.py``""" + +import llnl.util.filesystem as fs +import os +import pytest + + +@pytest.fixture() +def stage(tmpdir_factory): + """Creates a stage with the directory structure for the tests.""" + + s = tmpdir_factory.mktemp('filesystem_test') + + with s.as_cwd(): + # Create source file hierarchy + fs.touchp('source/1') + fs.touchp('source/a/b/2') + fs.touchp('source/a/b/3') + fs.touchp('source/c/4') + fs.touchp('source/c/d/5') + fs.touchp('source/c/d/6') + fs.touchp('source/c/d/e/7') + + # Create symlink + os.symlink(os.path.abspath('source/1'), 'source/2') + + # Create destination directory + fs.mkdirp('dest') + + yield s + + +class TestCopy: + """Tests for ``filesystem.copy``""" + + def test_file_dest(self, stage): + """Test using a filename as the destination.""" + + with fs.working_dir(str(stage)): + fs.copy('source/1', 'dest/1') + + assert os.path.exists('dest/1') + assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode + + def test_dir_dest(self, stage): + """Test using a directory as the destination.""" + + with fs.working_dir(str(stage)): + fs.copy('source/1', 'dest') + + assert os.path.exists('dest/1') + assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode + + +class TestInstall: + """Tests for ``filesystem.install``""" + + def test_file_dest(self, stage): + """Test using a filename as the destination.""" + + with fs.working_dir(str(stage)): + fs.install('source/1', 'dest/1') + + assert os.path.exists('dest/1') + assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode + + def test_dir_dest(self, stage): + """Test using a directory as the destination.""" + + with fs.working_dir(str(stage)): + fs.install('source/1', 'dest') + + assert os.path.exists('dest/1') + assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode + + +class TestCopyTree: + """Tests for ``filesystem.copy_tree``""" + + def test_existing_dir(self, stage): + """Test copying to an existing directory.""" + + with fs.working_dir(str(stage)): + fs.copy_tree('source', 'dest') + + assert os.path.exists('dest/a/b/2') + + def test_non_existing_dir(self, stage): + """Test copying to a non-existing directory.""" + + with fs.working_dir(str(stage)): + fs.copy_tree('source', 'dest/sub/directory') + + assert os.path.exists('dest/sub/directory/a/b/2') + + def test_symlinks_true(self, stage): + """Test copying with symlink preservation.""" + + with fs.working_dir(str(stage)): + fs.copy_tree('source', 'dest', symlinks=True) + + assert os.path.exists('dest/2') + assert os.path.islink('dest/2') + + def test_symlinks_false(self, stage): + """Test copying without symlink preservation.""" + + with fs.working_dir(str(stage)): + fs.copy_tree('source', 'dest', symlinks=False) + + assert os.path.exists('dest/2') + assert not os.path.islink('dest/2') + + +class TestInstallTree: + """Tests for ``filesystem.install_tree``""" + + def test_existing_dir(self, stage): + """Test installing to an existing directory.""" + + with fs.working_dir(str(stage)): + fs.install_tree('source', 'dest') + + assert os.path.exists('dest/a/b/2') + + def test_non_existing_dir(self, stage): + """Test installing to a non-existing directory.""" + + with fs.working_dir(str(stage)): + fs.install_tree('source', 'dest/sub/directory') + + assert os.path.exists('dest/sub/directory/a/b/2') + + def test_symlinks_true(self, stage): + """Test installing with symlink preservation.""" + + with fs.working_dir(str(stage)): + fs.install_tree('source', 'dest', symlinks=True) + + assert os.path.exists('dest/2') + assert os.path.islink('dest/2') + + def test_symlinks_false(self, stage): + """Test installing without symlink preservation.""" + + with fs.working_dir(str(stage)): + fs.install_tree('source', 'dest', symlinks=False) + + assert os.path.exists('dest/2') + assert not os.path.islink('dest/2') + + +def test_move_transaction_commit(tmpdir): + + fake_library = tmpdir.mkdir('lib').join('libfoo.so') + fake_library.write('Just some fake content.') + + old_md5 = fs.hash_directory(str(tmpdir)) + + with fs.replace_directory_transaction(str(tmpdir.join('lib'))): + fake_library = tmpdir.mkdir('lib').join('libfoo.so') + fake_library.write('Other content.') + new_md5 = fs.hash_directory(str(tmpdir)) + + assert old_md5 != fs.hash_directory(str(tmpdir)) + assert new_md5 == fs.hash_directory(str(tmpdir)) + + +def test_move_transaction_rollback(tmpdir): + + fake_library = tmpdir.mkdir('lib').join('libfoo.so') + fake_library.write('Just some fake content.') + + h = fs.hash_directory(str(tmpdir)) + + try: + with fs.replace_directory_transaction(str(tmpdir.join('lib'))): + assert h != fs.hash_directory(str(tmpdir)) + fake_library = tmpdir.mkdir('lib').join('libfoo.so') + fake_library.write('Other content.') + raise RuntimeError('') + except RuntimeError: + pass + + assert h == fs.hash_directory(str(tmpdir)) diff --git a/lib/spack/spack/test/util/filesystem.py b/lib/spack/spack/test/util/filesystem.py deleted file mode 100644 index 86c3332bb5..0000000000 --- a/lib/spack/spack/test/util/filesystem.py +++ /dev/null @@ -1,61 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/spack/spack -# Please also see the NOTICE and LICENSE files for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## - -import llnl.util.filesystem as fs - - -def test_move_transaction_commit(tmpdir): - - fake_library = tmpdir.mkdir('lib').join('libfoo.so') - fake_library.write('Just some fake content.') - - old_md5 = fs.hash_directory(str(tmpdir)) - - with fs.replace_directory_transaction(str(tmpdir.join('lib'))): - fake_library = tmpdir.mkdir('lib').join('libfoo.so') - fake_library.write('Other content.') - new_md5 = fs.hash_directory(str(tmpdir)) - - assert old_md5 != fs.hash_directory(str(tmpdir)) - assert new_md5 == fs.hash_directory(str(tmpdir)) - - -def test_move_transaction_rollback(tmpdir): - - fake_library = tmpdir.mkdir('lib').join('libfoo.so') - fake_library.write('Just some fake content.') - - h = fs.hash_directory(str(tmpdir)) - - try: - with fs.replace_directory_transaction(str(tmpdir.join('lib'))): - assert h != fs.hash_directory(str(tmpdir)) - fake_library = tmpdir.mkdir('lib').join('libfoo.so') - fake_library.write('Other content.') - raise RuntimeError('') - except RuntimeError: - pass - - assert h == fs.hash_directory(str(tmpdir)) -- cgit v1.2.3-60-g2f50