install_tree, copy_tree can install into existing directory structures (#8289)

Replace use of `shutil.copytree` with `copy_tree` and `install_tree` functions in `llnl.util.filesystem`.

- `copy_tree` copies without setting permissions.  It should be used to copy files around in the build directory.
- `install_tree` copies files and sets permissions.  It should be used to copy files into the installation directory.
- `install` and `copy` are analogous single-file functions.
- add more extensive tests for these functions
- update packages to use these functions.

3 files changed, 294 insertions, 73 deletions

diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index ace6cfded6..40eb6a7aa8 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -60,7 +60,9 @@ __all__ = [
     'fix_darwin_install_name',
     'force_remove',
     'force_symlink',
+    'copy',
     'install',
+    'copy_tree',
     'install_tree',
     'is_exe',
     'join_path',
@@ -264,27 +266,98 @@ def unset_executable_mode(path):
     os.chmod(path, mode)
 
 
-def install(src, dest):
-    """Manually install a file to a particular location."""
-    tty.debug("Installing %s to %s" % (src, dest))
+def copy(src, dest, _permissions=False):
+    """Copies the file *src* to the file or directory *dest*.
+
+    If *dest* specifies a directory, the file will be copied into *dest*
+    using the base filename from *src*.
+
+    Parameters:
+        src (str): the file to copy
+        dest (str): the destination file or directory
+        _permissions (bool): for internal use only
+    """
+    if _permissions:
+        tty.debug('Installing {0} to {1}'.format(src, dest))
+    else:
+        tty.debug('Copying {0} to {1}'.format(src, dest))
 
     # Expand dest to its eventual full path if it is a directory.
     if os.path.isdir(dest):
         dest = join_path(dest, os.path.basename(src))
 
     shutil.copy(src, dest)
-    set_install_permissions(dest)
-    copy_mode(src, dest)
+
+    if _permissions:
+        set_install_permissions(dest)
+        copy_mode(src, dest)
+
+
+def install(src, dest):
+    """Installs the file *src* to the file or directory *dest*.
+
+    Same as :py:func:`copy` with the addition of setting proper
+    permissions on the installed file.
+
+    Parameters:
+        src (str): the file to install
+        dest (str): the destination file or directory
+    """
+    copy(src, dest, _permissions=True)
 
 
-def install_tree(src, dest, **kwargs):
-    """Manually install a directory tree to a particular location."""
-    tty.debug("Installing %s to %s" % (src, dest))
-    shutil.copytree(src, dest, **kwargs)
+def copy_tree(src, dest, symlinks=True, _permissions=False):
+    """Recursively copy an entire directory tree rooted at *src*.
 
-    for s, d in traverse_tree(src, dest, follow_nonexisting=False):
-        set_install_permissions(d)
-        copy_mode(s, d)
+    If the destination directory *dest* does not already exist, it will
+    be created as well as missing parent directories.
+
+    If *symlinks* is true, symbolic links in the source tree are represented
+    as symbolic links in the new tree and the metadata of the original links
+    will be copied as far as the platform allows; if false, the contents and
+    metadata of the linked files are copied to the new tree.
+
+    Parameters:
+        src (str): the directory to copy
+        dest (str): the destination directory
+        symlinks (bool): whether or not to preserve symlinks
+        _permissions (bool): for internal use only
+    """
+    if _permissions:
+        tty.debug('Installing {0} to {1}'.format(src, dest))
+    else:
+        tty.debug('Copying {0} to {1}'.format(src, dest))
+
+    mkdirp(dest)
+
+    for s, d in traverse_tree(src, dest, order='pre', follow_nonexisting=True):
+        if symlinks and os.path.islink(s):
+            # Note that this won't rewrite absolute links into the old
+            # root to point at the new root. Should we handle that case?
+            target = os.readlink(s)
+            os.symlink(os.path.abspath(target), d)
+        elif os.path.isdir(s):
+            mkdirp(d)
+        else:
+            shutil.copyfile(s, d)
+
+        if _permissions:
+            set_install_permissions(d)
+            copy_mode(s, d)
+
+
+def install_tree(src, dest, symlinks=True):
+    """Recursively install an entire directory tree rooted at *src*.
+
+    Same as :py:func:`copy_tree` with the addition of setting proper
+    permissions on the installed files and directories.
+
+    Parameters:
+        src (str): the directory to install
+        dest (str): the destination directory
+        symlinks (bool): whether or not to preserve symlinks
+    """
+    copy_tree(src, dest, symlinks, _permissions=True)
 
 
 def is_exe(path):
diff --git a/lib/spack/spack/test/llnl/util/filesystem.py b/lib/spack/spack/test/llnl/util/filesystem.py
new file mode 100644
index 0000000000..7701185dbe
--- /dev/null
+++ b/lib/spack/spack/test/llnl/util/filesystem.py
@@ -0,0 +1,209 @@
+##############################################################################
+# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/spack/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""Tests for ``llnl/util/filesystem.py``"""
+
+import llnl.util.filesystem as fs
+import os
+import pytest
+
+
+@pytest.fixture()
+def stage(tmpdir_factory):
+    """Creates a stage with the directory structure for the tests."""
+
+    s = tmpdir_factory.mktemp('filesystem_test')
+
+    with s.as_cwd():
+        # Create source file hierarchy
+        fs.touchp('source/1')
+        fs.touchp('source/a/b/2')
+        fs.touchp('source/a/b/3')
+        fs.touchp('source/c/4')
+        fs.touchp('source/c/d/5')
+        fs.touchp('source/c/d/6')
+        fs.touchp('source/c/d/e/7')
+
+        # Create symlink
+        os.symlink(os.path.abspath('source/1'), 'source/2')
+
+        # Create destination directory
+        fs.mkdirp('dest')
+
+    yield s
+
+
+class TestCopy:
+    """Tests for ``filesystem.copy``"""
+
+    def test_file_dest(self, stage):
+        """Test using a filename as the destination."""
+
+        with fs.working_dir(str(stage)):
+            fs.copy('source/1', 'dest/1')
+
+            assert os.path.exists('dest/1')
+            assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode
+
+    def test_dir_dest(self, stage):
+        """Test using a directory as the destination."""
+
+        with fs.working_dir(str(stage)):
+            fs.copy('source/1', 'dest')
+
+            assert os.path.exists('dest/1')
+            assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode
+
+
+class TestInstall:
+    """Tests for ``filesystem.install``"""
+
+    def test_file_dest(self, stage):
+        """Test using a filename as the destination."""
+
+        with fs.working_dir(str(stage)):
+            fs.install('source/1', 'dest/1')
+
+            assert os.path.exists('dest/1')
+            assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode
+
+    def test_dir_dest(self, stage):
+        """Test using a directory as the destination."""
+
+        with fs.working_dir(str(stage)):
+            fs.install('source/1', 'dest')
+
+            assert os.path.exists('dest/1')
+            assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode
+
+
+class TestCopyTree:
+    """Tests for ``filesystem.copy_tree``"""
+
+    def test_existing_dir(self, stage):
+        """Test copying to an existing directory."""
+
+        with fs.working_dir(str(stage)):
+            fs.copy_tree('source', 'dest')
+
+            assert os.path.exists('dest/a/b/2')
+
+    def test_non_existing_dir(self, stage):
+        """Test copying to a non-existing directory."""
+
+        with fs.working_dir(str(stage)):
+            fs.copy_tree('source', 'dest/sub/directory')
+
+            assert os.path.exists('dest/sub/directory/a/b/2')
+
+    def test_symlinks_true(self, stage):
+        """Test copying with symlink preservation."""
+
+        with fs.working_dir(str(stage)):
+            fs.copy_tree('source', 'dest', symlinks=True)
+
+            assert os.path.exists('dest/2')
+            assert os.path.islink('dest/2')
+
+    def test_symlinks_false(self, stage):
+        """Test copying without symlink preservation."""
+
+        with fs.working_dir(str(stage)):
+            fs.copy_tree('source', 'dest', symlinks=False)
+
+            assert os.path.exists('dest/2')
+            assert not os.path.islink('dest/2')
+
+
+class TestInstallTree:
+    """Tests for ``filesystem.install_tree``"""
+
+    def test_existing_dir(self, stage):
+        """Test installing to an existing directory."""
+
+        with fs.working_dir(str(stage)):
+            fs.install_tree('source', 'dest')
+
+            assert os.path.exists('dest/a/b/2')
+
+    def test_non_existing_dir(self, stage):
+        """Test installing to a non-existing directory."""
+
+        with fs.working_dir(str(stage)):
+            fs.install_tree('source', 'dest/sub/directory')
+
+            assert os.path.exists('dest/sub/directory/a/b/2')
+
+    def test_symlinks_true(self, stage):
+        """Test installing with symlink preservation."""
+
+        with fs.working_dir(str(stage)):
+            fs.install_tree('source', 'dest', symlinks=True)
+
+            assert os.path.exists('dest/2')
+            assert os.path.islink('dest/2')
+
+    def test_symlinks_false(self, stage):
+        """Test installing without symlink preservation."""
+
+        with fs.working_dir(str(stage)):
+            fs.install_tree('source', 'dest', symlinks=False)
+
+            assert os.path.exists('dest/2')
+            assert not os.path.islink('dest/2')
+
+
+def test_move_transaction_commit(tmpdir):
+
+    fake_library = tmpdir.mkdir('lib').join('libfoo.so')
+    fake_library.write('Just some fake content.')
+
+    old_md5 = fs.hash_directory(str(tmpdir))
+
+    with fs.replace_directory_transaction(str(tmpdir.join('lib'))):
+        fake_library = tmpdir.mkdir('lib').join('libfoo.so')
+        fake_library.write('Other content.')
+        new_md5 = fs.hash_directory(str(tmpdir))
+
+    assert old_md5 != fs.hash_directory(str(tmpdir))
+    assert new_md5 == fs.hash_directory(str(tmpdir))
+
+
+def test_move_transaction_rollback(tmpdir):
+
+    fake_library = tmpdir.mkdir('lib').join('libfoo.so')
+    fake_library.write('Just some fake content.')
+
+    h = fs.hash_directory(str(tmpdir))
+
+    try:
+        with fs.replace_directory_transaction(str(tmpdir.join('lib'))):
+            assert h != fs.hash_directory(str(tmpdir))
+            fake_library = tmpdir.mkdir('lib').join('libfoo.so')
+            fake_library.write('Other content.')
+            raise RuntimeError('')
+    except RuntimeError:
+        pass
+
+    assert h == fs.hash_directory(str(tmpdir))
diff --git a/lib/spack/spack/test/util/filesystem.py b/lib/spack/spack/test/util/filesystem.py
deleted file mode 100644
index 86c3332bb5..0000000000
--- a/lib/spack/spack/test/util/filesystem.py
+++ /dev/null
@@ -1,61 +0,0 @@
-##############################################################################
-# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
-# Produced at the Lawrence Livermore National Laboratory.
-#
-# This file is part of Spack.
-# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
-# LLNL-CODE-647188
-#
-# For details, see https://github.com/spack/spack
-# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License (as
-# published by the Free Software Foundation) version 2.1, February 1999.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
-# conditions of the GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-##############################################################################
-
-import llnl.util.filesystem as fs
-
-
-def test_move_transaction_commit(tmpdir):
-
-    fake_library = tmpdir.mkdir('lib').join('libfoo.so')
-    fake_library.write('Just some fake content.')
-
-    old_md5 = fs.hash_directory(str(tmpdir))
-
-    with fs.replace_directory_transaction(str(tmpdir.join('lib'))):
-        fake_library = tmpdir.mkdir('lib').join('libfoo.so')
-        fake_library.write('Other content.')
-        new_md5 = fs.hash_directory(str(tmpdir))
-
-    assert old_md5 != fs.hash_directory(str(tmpdir))
-    assert new_md5 == fs.hash_directory(str(tmpdir))
-
-
-def test_move_transaction_rollback(tmpdir):
-
-    fake_library = tmpdir.mkdir('lib').join('libfoo.so')
-    fake_library.write('Just some fake content.')
-
-    h = fs.hash_directory(str(tmpdir))
-
-    try:
-        with fs.replace_directory_transaction(str(tmpdir.join('lib'))):
-            assert h != fs.hash_directory(str(tmpdir))
-            fake_library = tmpdir.mkdir('lib').join('libfoo.so')
-            fake_library.write('Other content.')
-            raise RuntimeError('')
-    except RuntimeError:
-        pass
-
-    assert h == fs.hash_directory(str(tmpdir))