install/install_tree: glob support (#18376)

* install/install_tree: glob support

* Add unit tests

* Update existing packages

* Raise error if glob finds no files, document function raises

2 files changed, 200 insertions, 64 deletions

diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index 5e08273677..763401b3aa 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -337,41 +337,63 @@ def unset_executable_mode(path):
 
 
 def copy(src, dest, _permissions=False):
-    """Copies the file *src* to the file or directory *dest*.
+    """Copy the file(s) *src* to the file or directory *dest*.
 
     If *dest* specifies a directory, the file will be copied into *dest*
     using the base filename from *src*.
 
+    *src* may contain glob characters.
+
     Parameters:
-        src (str): the file to copy
+        src (str): the file(s) to copy
         dest (str): the destination file or directory
         _permissions (bool): for internal use only
+
+    Raises:
+        IOError: if *src* does not match any files or directories
+        ValueError: if *src* matches multiple files but *dest* is
+            not a directory
     """
     if _permissions:
         tty.debug('Installing {0} to {1}'.format(src, dest))
     else:
         tty.debug('Copying {0} to {1}'.format(src, dest))
 
-    # Expand dest to its eventual full path if it is a directory.
-    if os.path.isdir(dest):
-        dest = join_path(dest, os.path.basename(src))
+    files = glob.glob(src)
+    if not files:
+        raise IOError("No such file or directory: '{0}'".format(src))
+    if len(files) > 1 and not os.path.isdir(dest):
+        raise ValueError(
+            "'{0}' matches multiple files but '{1}' is not a directory".format(
+                src, dest))
 
-    shutil.copy(src, dest)
+    for src in files:
+        # Expand dest to its eventual full path if it is a directory.
+        dst = dest
+        if os.path.isdir(dest):
+            dst = join_path(dest, os.path.basename(src))
 
-    if _permissions:
-        set_install_permissions(dest)
-        copy_mode(src, dest)
+        shutil.copy(src, dst)
+
+        if _permissions:
+            set_install_permissions(dst)
+            copy_mode(src, dst)
 
 
 def install(src, dest):
-    """Installs the file *src* to the file or directory *dest*.
+    """Install the file(s) *src* to the file or directory *dest*.
 
     Same as :py:func:`copy` with the addition of setting proper
     permissions on the installed file.
 
     Parameters:
-        src (str): the file to install
+        src (str): the file(s) to install
         dest (str): the destination file or directory
+
+    Raises:
+        IOError: if *src* does not match any files or directories
+        ValueError: if *src* matches multiple files but *dest* is
+            not a directory
     """
     copy(src, dest, _permissions=True)
 
@@ -396,6 +418,8 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
     If the destination directory *dest* does not already exist, it will
     be created as well as missing parent directories.
 
+    *src* may contain glob characters.
+
     If *symlinks* is true, symbolic links in the source tree are represented
     as symbolic links in the new tree and the metadata of the original links
     will be copied as far as the platform allows; if false, the contents and
@@ -410,56 +434,66 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
         symlinks (bool): whether or not to preserve symlinks
         ignore (function): function indicating which files to ignore
         _permissions (bool): for internal use only
+
+    Raises:
+        IOError: if *src* does not match any files or directories
+        ValueError: if *src* is a parent directory of *dest*
     """
     if _permissions:
         tty.debug('Installing {0} to {1}'.format(src, dest))
     else:
         tty.debug('Copying {0} to {1}'.format(src, dest))
 
-    abs_src = os.path.abspath(src)
-    if not abs_src.endswith(os.path.sep):
-        abs_src += os.path.sep
     abs_dest = os.path.abspath(dest)
     if not abs_dest.endswith(os.path.sep):
         abs_dest += os.path.sep
 
-    # Stop early to avoid unnecessary recursion if being asked to copy from a
-    # parent directory.
-    if abs_dest.startswith(abs_src):
-        raise ValueError('Cannot copy ancestor directory {0} into {1}'.
-                         format(abs_src, abs_dest))
-
-    mkdirp(dest)
-
-    for s, d in traverse_tree(abs_src, abs_dest, order='pre',
-                              follow_symlinks=not symlinks,
-                              ignore=ignore,
-                              follow_nonexisting=True):
-        if os.path.islink(s):
-            link_target = resolve_link_target_relative_to_the_link(s)
-            if symlinks:
-                target = os.readlink(s)
-                if os.path.isabs(target):
-                    new_target = re.sub(abs_src, abs_dest, target)
-                    if new_target != target:
-                        tty.debug("Redirecting link {0} to {1}"
-                                  .format(target, new_target))
-                        target = new_target
-
-                os.symlink(target, d)
-            elif os.path.isdir(link_target):
-                mkdirp(d)
-            else:
-                shutil.copyfile(s, d)
-        else:
-            if os.path.isdir(s):
-                mkdirp(d)
+    files = glob.glob(src)
+    if not files:
+        raise IOError("No such file or directory: '{0}'".format(src))
+
+    for src in files:
+        abs_src = os.path.abspath(src)
+        if not abs_src.endswith(os.path.sep):
+            abs_src += os.path.sep
+
+        # Stop early to avoid unnecessary recursion if being asked to copy
+        # from a parent directory.
+        if abs_dest.startswith(abs_src):
+            raise ValueError('Cannot copy ancestor directory {0} into {1}'.
+                             format(abs_src, abs_dest))
+
+        mkdirp(abs_dest)
+
+        for s, d in traverse_tree(abs_src, abs_dest, order='pre',
+                                  follow_symlinks=not symlinks,
+                                  ignore=ignore,
+                                  follow_nonexisting=True):
+            if os.path.islink(s):
+                link_target = resolve_link_target_relative_to_the_link(s)
+                if symlinks:
+                    target = os.readlink(s)
+                    if os.path.isabs(target):
+                        new_target = re.sub(abs_src, abs_dest, target)
+                        if new_target != target:
+                            tty.debug("Redirecting link {0} to {1}"
+                                      .format(target, new_target))
+                            target = new_target
+
+                    os.symlink(target, d)
+                elif os.path.isdir(link_target):
+                    mkdirp(d)
+                else:
+                    shutil.copyfile(s, d)
             else:
-                shutil.copy2(s, d)
+                if os.path.isdir(s):
+                    mkdirp(d)
+                else:
+                    shutil.copy2(s, d)
 
-        if _permissions:
-            set_install_permissions(d)
-            copy_mode(s, d)
+            if _permissions:
+                set_install_permissions(d)
+                copy_mode(s, d)
 
 
 def install_tree(src, dest, symlinks=True, ignore=None):
@@ -473,6 +507,10 @@ def install_tree(src, dest, symlinks=True, ignore=None):
         dest (str): the destination directory
         symlinks (bool): whether or not to preserve symlinks
         ignore (function): function indicating which files to ignore
+
+    Raises:
+        IOError: if *src* does not match any files or directories
+        ValueError: if *src* is a parent directory of *dest*
     """
     copy_tree(src, dest, symlinks=symlinks, ignore=ignore, _permissions=True)
 
diff --git a/lib/spack/spack/test/llnl/util/filesystem.py b/lib/spack/spack/test/llnl/util/filesystem.py
index b48abb4fc6..d56817e9bb 100644
--- a/lib/spack/spack/test/llnl/util/filesystem.py
+++ b/lib/spack/spack/test/llnl/util/filesystem.py
@@ -30,6 +30,9 @@ def stage(tmpdir_factory):
         fs.touchp('source/c/d/5')
         fs.touchp('source/c/d/6')
         fs.touchp('source/c/d/e/7')
+        fs.touchp('source/g/h/i/8')
+        fs.touchp('source/g/h/i/9')
+        fs.touchp('source/g/i/j/10')
 
         # Create symlinks
         os.symlink(os.path.abspath('source/1'), 'source/2')
@@ -61,6 +64,31 @@ class TestCopy:
 
             assert os.path.exists('dest/1')
 
+    def test_glob_src(self, stage):
+        """Test using a glob as the source."""
+
+        with fs.working_dir(str(stage)):
+            fs.copy('source/a/*/*', 'dest')
+
+            assert os.path.exists('dest/2')
+            assert os.path.exists('dest/3')
+
+    def test_non_existing_src(self, stage):
+        """Test using a non-existing source."""
+
+        with fs.working_dir(str(stage)):
+            with pytest.raises(IOError, match='No such file or directory'):
+                fs.copy('source/none', 'dest')
+
+    def test_multiple_src_file_dest(self, stage):
+        """Test a glob that matches multiple source files and a dest
+        that is not a directory."""
+
+        with fs.working_dir(str(stage)):
+            match = '.* matches multiple files but .* is not a directory'
+            with pytest.raises(ValueError, match=match):
+                fs.copy('source/a/*/*', 'dest/1')
+
 
 def check_added_exe_permissions(src, dst):
     src_mode = os.stat(src).st_mode
@@ -91,6 +119,33 @@ class TestInstall:
             assert os.path.exists('dest/1')
             check_added_exe_permissions('source/1', 'dest/1')
 
+    def test_glob_src(self, stage):
+        """Test using a glob as the source."""
+
+        with fs.working_dir(str(stage)):
+            fs.install('source/a/*/*', 'dest')
+
+            assert os.path.exists('dest/2')
+            assert os.path.exists('dest/3')
+            check_added_exe_permissions('source/a/b/2', 'dest/2')
+            check_added_exe_permissions('source/a/b/3', 'dest/3')
+
+    def test_non_existing_src(self, stage):
+        """Test using a non-existing source."""
+
+        with fs.working_dir(str(stage)):
+            with pytest.raises(IOError, match='No such file or directory'):
+                fs.install('source/none', 'dest')
+
+    def test_multiple_src_file_dest(self, stage):
+        """Test a glob that matches multiple source files and a dest
+        that is not a directory."""
+
+        with fs.working_dir(str(stage)):
+            match = '.* matches multiple files but .* is not a directory'
+            with pytest.raises(ValueError, match=match):
+                fs.install('source/a/*/*', 'dest/1')
+
 
 class TestCopyTree:
     """Tests for ``filesystem.copy_tree``"""
@@ -111,21 +166,6 @@ class TestCopyTree:
 
             assert os.path.exists('dest/sub/directory/a/b/2')
 
-    def test_parent_dir(self, stage):
-        """Test copying to from a parent directory."""
-
-        # Make sure we get the right error if we try to copy a parent into
-        # a descendent directory.
-        with pytest.raises(ValueError, match="Cannot copy"):
-            with fs.working_dir(str(stage)):
-                fs.copy_tree('source', 'source/sub/directory')
-
-        # Only point with this check is to make sure we don't try to perform
-        # the copy.
-        with pytest.raises(IOError, match="No such file or directory"):
-            with fs.working_dir(str(stage)):
-                fs.copy_tree('foo/ba', 'foo/bar')
-
     def test_symlinks_true(self, stage):
         """Test copying with symlink preservation."""
 
@@ -162,6 +202,31 @@ class TestCopyTree:
             assert os.path.exists('dest/2')
             assert not os.path.islink('dest/2')
 
+    def test_glob_src(self, stage):
+        """Test using a glob as the source."""
+
+        with fs.working_dir(str(stage)):
+            fs.copy_tree('source/g/*', 'dest')
+
+            assert os.path.exists('dest/i/8')
+            assert os.path.exists('dest/i/9')
+            assert os.path.exists('dest/j/10')
+
+    def test_non_existing_src(self, stage):
+        """Test using a non-existing source."""
+
+        with fs.working_dir(str(stage)):
+            with pytest.raises(IOError, match='No such file or directory'):
+                fs.copy_tree('source/none', 'dest')
+
+    def test_parent_dir(self, stage):
+        """Test source as a parent directory of destination."""
+
+        with fs.working_dir(str(stage)):
+            match = 'Cannot copy ancestor directory'
+            with pytest.raises(ValueError, match=match):
+                fs.copy_tree('source', 'source/sub/directory')
+
 
 class TestInstallTree:
     """Tests for ``filesystem.install_tree``"""
@@ -173,6 +238,7 @@ class TestInstallTree:
             fs.install_tree('source', 'dest')
 
             assert os.path.exists('dest/a/b/2')
+            check_added_exe_permissions('source/a/b/2', 'dest/a/b/2')
 
     def test_non_existing_dir(self, stage):
         """Test installing to a non-existing directory."""
@@ -181,6 +247,8 @@ class TestInstallTree:
             fs.install_tree('source', 'dest/sub/directory')
 
             assert os.path.exists('dest/sub/directory/a/b/2')
+            check_added_exe_permissions(
+                'source/a/b/2', 'dest/sub/directory/a/b/2')
 
     def test_symlinks_true(self, stage):
         """Test installing with symlink preservation."""
@@ -190,6 +258,7 @@ class TestInstallTree:
 
             assert os.path.exists('dest/2')
             assert os.path.islink('dest/2')
+            check_added_exe_permissions('source/2', 'dest/2')
 
     def test_symlinks_false(self, stage):
         """Test installing without symlink preservation."""
@@ -199,6 +268,35 @@ class TestInstallTree:
 
             assert os.path.exists('dest/2')
             assert not os.path.islink('dest/2')
+            check_added_exe_permissions('source/2', 'dest/2')
+
+    def test_glob_src(self, stage):
+        """Test using a glob as the source."""
+
+        with fs.working_dir(str(stage)):
+            fs.install_tree('source/g/*', 'dest')
+
+            assert os.path.exists('dest/i/8')
+            assert os.path.exists('dest/i/9')
+            assert os.path.exists('dest/j/10')
+            check_added_exe_permissions('source/g/h/i/8', 'dest/i/8')
+            check_added_exe_permissions('source/g/h/i/9', 'dest/i/9')
+            check_added_exe_permissions('source/g/i/j/10', 'dest/j/10')
+
+    def test_non_existing_src(self, stage):
+        """Test using a non-existing source."""
+
+        with fs.working_dir(str(stage)):
+            with pytest.raises(IOError, match='No such file or directory'):
+                fs.install_tree('source/none', 'dest')
+
+    def test_parent_dir(self, stage):
+        """Test source as a parent directory of destination."""
+
+        with fs.working_dir(str(stage)):
+            match = 'Cannot copy ancestor directory'
+            with pytest.raises(ValueError, match=match):
+                fs.install_tree('source', 'source/sub/directory')
 
 
 def test_paths_containing_libs(dirs_with_libfiles):