environment views: single pass view generation (#29443)

Reduces the number of stat calls to a bare minimum:
- Single pass over src prefixes
- Handle projection clashes in memory

Symlinked directories in the src prefixes are now conditionally
transformed into directories with symlinks in the dst dir. Notably
`intel-mkl`, `cuda` and `qt` has top-level symlinked directories that
previously resulted in empty directories in the view. We now avoid
cycles and possible exponential blowup by only expanding symlinks that:
- point to dirs deeper in the folder structure;
- are a fixed depth of 2.

4 files changed, 65 insertions, 1 deletions

diff --git a/var/spack/repos/builtin.mock/packages/view-dir-dir/package.py b/var/spack/repos/builtin.mock/packages/view-dir-dir/package.py
new file mode 100644
index 0000000000..b85a81caa4
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/view-dir-dir/package.py
@@ -0,0 +1,21 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+
+class ViewDirDir(Package):
+    """Installs a <prefix>/bin/x where x is a dir, in contrast to view-dir-file."""
+    homepage = "http://www.spack.org"
+    url = "http://www.spack.org/downloads/aml-1.0.tar.gz"
+    has_code = False
+
+    version('0.1.0', sha256='cc89a8768693f1f11539378b21cdca9f0ce3fc5cb564f9b3e4154a051dcea69b')
+
+    def install(self, spec, prefix):
+        os.mkdir(os.path.join(prefix, 'bin'))
+        os.mkdir(os.path.join(prefix, 'bin', 'x'))
+        with open(os.path.join(prefix, 'bin', 'x', 'file_in_dir'), 'wb') as f:
+            f.write(b'hello world')
diff --git a/var/spack/repos/builtin.mock/packages/view-dir-file/package.py b/var/spack/repos/builtin.mock/packages/view-dir-file/package.py
new file mode 100644
index 0000000000..a47e02ca24
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/view-dir-file/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+
+class ViewDirFile(Package):
+    """Installs a <prefix>/bin/x where x is a file, in contrast to view-dir-dir"""
+    homepage = "http://www.spack.org"
+    url = "http://www.spack.org/downloads/aml-1.0.tar.gz"
+    has_code = False
+
+    version('0.1.0', sha256='cc89a8768693f1f11539378b21cdca9f0ce3fc5cb564f9b3e4154a051dcea69b')
+
+    def install(self, spec, prefix):
+        os.mkdir(os.path.join(prefix, 'bin'))
+        with open(os.path.join(prefix, 'bin', 'x'), 'wb') as f:
+            f.write(b'file')
diff --git a/var/spack/repos/builtin.mock/packages/view-dir-symlinked-dir/package.py b/var/spack/repos/builtin.mock/packages/view-dir-symlinked-dir/package.py
new file mode 100644
index 0000000000..ae0d90541b
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/view-dir-symlinked-dir/package.py
@@ -0,0 +1,23 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+
+class ViewDirSymlinkedDir(Package):
+    """Installs <prefix>/bin/x/file_in_symlinked_dir where x -> y is a symlinked dir.
+    This should be mergeable with view-dir-dir, but not with view-dir-file."""
+    homepage = "http://www.spack.org"
+    url = "http://www.spack.org/downloads/aml-1.0.tar.gz"
+    has_code = False
+
+    version('0.1.0', sha256='cc89a8768693f1f11539378b21cdca9f0ce3fc5cb564f9b3e4154a051dcea69b')
+
+    def install(self, spec, prefix):
+        os.mkdir(os.path.join(prefix, 'bin'))
+        os.mkdir(os.path.join(prefix, 'bin', 'y'))
+        with open(os.path.join(prefix, 'bin', 'y', 'file_in_symlinked_dir'), 'wb') as f:
+            f.write(b'hello world')
+        os.symlink('y', os.path.join(prefix, 'bin', 'x'))
diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
index c97407d9fc..7f96898287 100644
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -1365,7 +1365,7 @@ config.update(get_paths())
                                             self.spec
                                         ))
 
-    def add_files_to_view(self, view, merge_map):
+    def add_files_to_view(self, view, merge_map, skip_if_exists=False):
         bin_dir = self.spec.prefix.bin if sys.platform != 'win32'\
             else self.spec.prefix
         for src, dst in merge_map.items():