python: always use a venv (#40773)

This commit adds a layer of indirection to improve build isolation with 
and without external Python, as well as usability of environment views.

It adds `python-venv` as a dependency to all packages that `extends("python")`, 
which has the following advantages:

1. Build isolation: only `PYTHONPATH` is considered in builds, not 
   user / system packages
2. Stable install layout: fixes the problem on Debian, RHEL and Fedora where 
   external / system python produces `bin/local` subdirs in Spack install prefixes. 
3. Environment views are Python virtual environments (and if you add 
   `py-pip` things like `pip list` work)

Views work whether they're symlink, hardlink or copy type.

This commit additionally makes `spec["python"].command` return 
`spec["python-venv"].command`. The rationale is that packages in repos we do 
not own do not pass the underlying python to the build system, which could still 
result in incorrectly computed install layouts.

Other attributes like `libs`, `headers` should be on `python` anyways and need no change.

43 files changed, 173 insertions, 204 deletions

diff --git a/var/spack/repos/builtin.mock/packages/python-venv/package.py b/var/spack/repos/builtin.mock/packages/python-venv/package.py
new file mode 100644
index 0000000000..741fc3c627
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/python-venv/package.py
@@ -0,0 +1,21 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PythonVenv(Package):
+    """A Spack managed Python virtual environment"""
+
+    homepage = "https://docs.python.org/3/library/venv.html"
+    has_code = False
+
+    version("1.0")
+
+    extends("python")
+
+    def install(self, spec, prefix):
+        pass
diff --git a/var/spack/repos/builtin/packages/ascent/package.py b/var/spack/repos/builtin/packages/ascent/package.py
index 5fb06916f3..17c5604033 100644
--- a/var/spack/repos/builtin/packages/ascent/package.py
+++ b/var/spack/repos/builtin/packages/ascent/package.py
@@ -495,7 +495,7 @@ class Ascent(CMakePackage, CudaPackage):
             cfg.write("# Enable python module builds\n")
             cfg.write(cmake_cache_entry("ENABLE_PYTHON", "ON"))
             cfg.write("# python from spack \n")
-            cfg.write(cmake_cache_entry("PYTHON_EXECUTABLE", spec["python"].command.path))
+            cfg.write(cmake_cache_entry("PYTHON_EXECUTABLE", python.path))
             try:
                 cfg.write("# python module install dir\n")
                 cfg.write(cmake_cache_entry("PYTHON_MODULE_INSTALL_PREFIX", python_platlib))
diff --git a/var/spack/repos/builtin/packages/bohrium/package.py b/var/spack/repos/builtin/packages/bohrium/package.py
index 64ad8c100e..fd9e665663 100644
--- a/var/spack/repos/builtin/packages/bohrium/package.py
+++ b/var/spack/repos/builtin/packages/bohrium/package.py
@@ -256,10 +256,6 @@ class Bohrium(CMakePackage, CudaPackage):
         cxx("-o", "test_cxxadd", file_cxxadd, *cxx_flags)
         test_cxxadd = Executable("./test_cxxadd")
 
-        # Build python test commandline
-        file_pyadd = join_path(os.path.dirname(self.module.__file__), "pyadd.py")
-        test_pyadd = Executable(spec["python"].command.path + " " + file_pyadd)
-
         # Run tests for each available stack
         for bh_stack in stacks:
             tty.info("Testing with bohrium stack '" + bh_stack + "'")
@@ -270,5 +266,6 @@ class Bohrium(CMakePackage, CudaPackage):
 
             # Python test (if +python)
             if "+python" in spec:
-                py_output = test_pyadd(output=str, env=test_env)
+                file_pyadd = join_path(os.path.dirname(self.module.__file__), "pyadd.py")
+                py_output = python(file_pyadd, output=str, env=test_env)
                 compare_output(py_output, "Success!\n")
diff --git a/var/spack/repos/builtin/packages/cantera/package.py b/var/spack/repos/builtin/packages/cantera/package.py
index efb9d7fcbe..ef057f955b 100644
--- a/var/spack/repos/builtin/packages/cantera/package.py
+++ b/var/spack/repos/builtin/packages/cantera/package.py
@@ -134,13 +134,9 @@ class Cantera(SConsPackage):
 
         # Python module
         if "+python" in spec:
-            args.extend(
-                ["python_package=full", "python_cmd={0}".format(spec["python"].command.path)]
-            )
+            args.extend(["python_package=full", "python_cmd={0}".format(python.path)])
             if spec["python"].satisfies("@3:"):
-                args.extend(
-                    ["python3_package=y", "python3_cmd={0}".format(spec["python"].command.path)]
-                )
+                args.extend(["python3_package=y", "python3_cmd={0}".format(python.path)])
             else:
                 args.append("python3_package=n")
         else:
diff --git a/var/spack/repos/builtin/packages/clingo-bootstrap/package.py b/var/spack/repos/builtin/packages/clingo-bootstrap/package.py
index 022cb7e6e3..abb5c53b3c 100644
--- a/var/spack/repos/builtin/packages/clingo-bootstrap/package.py
+++ b/var/spack/repos/builtin/packages/clingo-bootstrap/package.py
@@ -120,9 +120,7 @@ class ClingoBootstrap(Clingo):
         )
         python_runtime_env.unset("SPACK_ENV")
         python_runtime_env.unset("SPACK_PYTHON")
-        self.spec["python"].command(
-            spack.paths.spack_script, "solve", "--fresh", "hdf5", extra_env=python_runtime_env
-        )
+        python(spack.paths.spack_script, "solve", "--fresh", "hdf5", extra_env=python_runtime_env)
 
         # Clean the build dir.
         rmtree(self.build_directory, ignore_errors=True)
diff --git a/var/spack/repos/builtin/packages/clingo/package.py b/var/spack/repos/builtin/packages/clingo/package.py
index fe1a6f859f..7af9b213fb 100644
--- a/var/spack/repos/builtin/packages/clingo/package.py
+++ b/var/spack/repos/builtin/packages/clingo/package.py
@@ -3,8 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import os
-
 from spack.compiler import UnsupportedCompilerFlag
 from spack.package import *
 
@@ -123,7 +121,4 @@ class Clingo(CMakePackage):
         return args
 
     def win_add_library_dependent(self):
-        if "+python" in self.spec:
-            return [os.path.join(self.prefix, self.spec["python"].package.platlib)]
-        else:
-            return []
+        return [python_platlib] if "+python" in self.spec else []
diff --git a/var/spack/repos/builtin/packages/conduit/package.py b/var/spack/repos/builtin/packages/conduit/package.py
index c1066ce752..001b04953d 100644
--- a/var/spack/repos/builtin/packages/conduit/package.py
+++ b/var/spack/repos/builtin/packages/conduit/package.py
@@ -443,7 +443,7 @@ class Conduit(CMakePackage):
             cfg.write("# Enable python module builds\n")
             cfg.write(cmake_cache_entry("ENABLE_PYTHON", "ON"))
             cfg.write("# python from spack \n")
-            cfg.write(cmake_cache_entry("PYTHON_EXECUTABLE", spec["python"].command.path))
+            cfg.write(cmake_cache_entry("PYTHON_EXECUTABLE", python.path))
             try:
                 cfg.write("# python module install dir\n")
                 cfg.write(cmake_cache_entry("PYTHON_MODULE_INSTALL_PREFIX", python_platlib))
diff --git a/var/spack/repos/builtin/packages/dsqss/package.py b/var/spack/repos/builtin/packages/dsqss/package.py
index 72e5f5961f..44f76ab619 100644
--- a/var/spack/repos/builtin/packages/dsqss/package.py
+++ b/var/spack/repos/builtin/packages/dsqss/package.py
@@ -58,7 +58,6 @@ class Dsqss(CMakePackage):
         copy(join_path(test01, "std.toml"), ".")
 
         # prepare
-        python = self.spec["python"].command
         opts = [self.spec.prefix.bin.dla_pre, "std.toml"]
         with test_part(self, "test_dla_pre", purpose="prepare dla"):
             python(*opts)
diff --git a/var/spack/repos/builtin/packages/fenics/package.py b/var/spack/repos/builtin/packages/fenics/package.py
index ae8336aa39..cf56c14fd0 100644
--- a/var/spack/repos/builtin/packages/fenics/package.py
+++ b/var/spack/repos/builtin/packages/fenics/package.py
@@ -156,7 +156,7 @@ class Fenics(CMakePackage):
     depends_on("py-sphinx@1.0.1:", when="+doc", type="build")
 
     def cmake_args(self):
-        args = [
+        return [
             self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
             self.define("DOLFIN_SKIP_BUILD_TESTS", True),
             self.define_from_variant("DOLFIN_ENABLE_OPENMP", "openmp"),
@@ -180,11 +180,6 @@ class Fenics(CMakePackage):
             self.define_from_variant("DOLFIN_ENABLE_ZLIB", "zlib"),
         ]
 
-        if "+python" in self.spec:
-            args.append(self.define("PYTHON_EXECUTABLE", self.spec["python"].command.path))
-
-        return args
-
     # set environment for bulding python interface
     def setup_build_environment(self, env):
         env.set("DOLFIN_DIR", self.prefix)
diff --git a/var/spack/repos/builtin/packages/gurobi/package.py b/var/spack/repos/builtin/packages/gurobi/package.py
index c0fa33639d..d9dd35c2ed 100644
--- a/var/spack/repos/builtin/packages/gurobi/package.py
+++ b/var/spack/repos/builtin/packages/gurobi/package.py
@@ -57,5 +57,4 @@ class Gurobi(Package):
     @run_after("install")
     def gurobipy(self):
         with working_dir("linux64"):
-            python = which("python")
             python("setup.py", "install", "--prefix={0}".format(self.prefix))
diff --git a/var/spack/repos/builtin/packages/lbann/package.py b/var/spack/repos/builtin/packages/lbann/package.py
index 482d591a1c..e875832b92 100644
--- a/var/spack/repos/builtin/packages/lbann/package.py
+++ b/var/spack/repos/builtin/packages/lbann/package.py
@@ -402,12 +402,8 @@ class Lbann(CachedCMakePackage, CudaPackage, ROCmPackage):
         )
         entries.append(cmake_cache_option("protobuf_MODULE_COMPATIBLE", True))
 
-        if spec.satisfies("^python") and "+pfe" in spec:
-            entries.append(
-                cmake_cache_path(
-                    "LBANN_PFE_PYTHON_EXECUTABLE", "{0}/python3".format(spec["python"].prefix.bin)
-                )
-            )
+        if spec.satisfies("+pfe ^python"):
+            entries.append(cmake_cache_path("LBANN_PFE_PYTHON_EXECUTABLE", python.path))
             entries.append(
                 cmake_cache_string("LBANN_PFE_PYTHONPATH", env["PYTHONPATH"])
             )  # do NOT need to sub ; for : because
diff --git a/var/spack/repos/builtin/packages/libcap-ng/package.py b/var/spack/repos/builtin/packages/libcap-ng/package.py
index 373cf7c399..8256fc16a5 100644
--- a/var/spack/repos/builtin/packages/libcap-ng/package.py
+++ b/var/spack/repos/builtin/packages/libcap-ng/package.py
@@ -33,7 +33,7 @@ class LibcapNg(AutotoolsPackage):
 
     def setup_build_environment(self, env):
         if self.spec.satisfies("+python"):
-            env.set("PYTHON", self.spec["python"].command.path)
+            env.set("PYTHON", python.path)
 
     def configure_args(self):
         args = []
diff --git a/var/spack/repos/builtin/packages/mapserver/package.py b/var/spack/repos/builtin/packages/mapserver/package.py
index e26951531e..c55cba0fea 100644
--- a/var/spack/repos/builtin/packages/mapserver/package.py
+++ b/var/spack/repos/builtin/packages/mapserver/package.py
@@ -3,8 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import os
-
 from spack.package import *
 
 
@@ -60,9 +58,7 @@ class Mapserver(CMakePackage):
         # prefix. This hack patches the CMakeLists.txt for the Python
         # bindings and hard-wires in the right destination. A bit ugly,
         # sorry, but I don't speak cmake.
-        pyversiondir = "python{0}".format(self.spec["python"].version.up_to(2))
-        sitepackages = os.path.join(self.spec.prefix.lib, pyversiondir, "site-packages")
-        filter_file(r"\${PYTHON_SITE_PACKAGES}", sitepackages, "mapscript/python/CMakeLists.txt")
+        filter_file(r"\${PYTHON_SITE_PACKAGES}", python_platlib, "mapscript/python/CMakeLists.txt")
 
     def cmake_args(self):
         args = []
diff --git a/var/spack/repos/builtin/packages/omnitrace/package.py b/var/spack/repos/builtin/packages/omnitrace/package.py
index bc3ed31758..384f8ca2db 100644
--- a/var/spack/repos/builtin/packages/omnitrace/package.py
+++ b/var/spack/repos/builtin/packages/omnitrace/package.py
@@ -141,7 +141,3 @@ class Omnitrace(CMakePackage):
             files = glob.glob(pattern)
             if files:
                 env.set("TAU_MAKEFILE", files[0])
-
-    def setup_run_environment(self, env):
-        if "+python" in self.spec:
-            env.prepend_path("PYTHONPATH", join_path(self.prefix.lib, "python", "site-packages"))
diff --git a/var/spack/repos/builtin/packages/open3d/package.py b/var/spack/repos/builtin/packages/open3d/package.py
index 2a63de25ca..a8f1beee6b 100644
--- a/var/spack/repos/builtin/packages/open3d/package.py
+++ b/var/spack/repos/builtin/packages/open3d/package.py
@@ -119,7 +119,7 @@ class Open3d(CMakePackage, CudaPackage):
     def test(self):
         if "+python" in self.spec:
             self.run_test(
-                self.spec["python"].command.path,
+                python.path,
                 ["-c", "import open3d"],
                 purpose="checking import of open3d",
                 work_dir="spack-test",
diff --git a/var/spack/repos/builtin/packages/opencv/package.py b/var/spack/repos/builtin/packages/opencv/package.py
index 8e3f3236bb..64375458a3 100644
--- a/var/spack/repos/builtin/packages/opencv/package.py
+++ b/var/spack/repos/builtin/packages/opencv/package.py
@@ -1027,14 +1027,13 @@ class Opencv(CMakePackage, CudaPackage):
             )
 
         # Python
-        python_exe = spec["python"].command.path
         python_lib = spec["python"].libs[0]
         python_include_dir = spec["python"].headers.directories[0]
 
         if "+python3" in spec:
             args.extend(
                 [
-                    self.define("PYTHON3_EXECUTABLE", python_exe),
+                    self.define("PYTHON3_EXECUTABLE", python.path),
                     self.define("PYTHON3_LIBRARY", python_lib),
                     self.define("PYTHON3_INCLUDE_DIR", python_include_dir),
                     self.define("PYTHON2_EXECUTABLE", ""),
diff --git a/var/spack/repos/builtin/packages/openwsman/package.py b/var/spack/repos/builtin/packages/openwsman/package.py
index 4be921bad9..9d9870706b 100644
--- a/var/spack/repos/builtin/packages/openwsman/package.py
+++ b/var/spack/repos/builtin/packages/openwsman/package.py
@@ -31,13 +31,9 @@ class Openwsman(CMakePackage):
     def patch(self):
         """Change python install directory."""
         if self.spec.satisfies("+python"):
-            python_spec = self.spec["python"]
-            python_libdir = join_path(
-                self.spec.prefix.lib, "python" + str(python_spec.version.up_to(2)), "site-packages"
-            )
             filter_file(
                 "DESTINATION .*",
-                "DESTINATION {0} )".format(python_libdir),
+                "DESTINATION {0} )".format(python_platlib),
                 join_path("bindings", "python", "CMakeLists.txt"),
             )
 
diff --git a/var/spack/repos/builtin/packages/precice/package.py b/var/spack/repos/builtin/packages/precice/package.py
index 48484114fd..6426fa51f3 100644
--- a/var/spack/repos/builtin/packages/precice/package.py
+++ b/var/spack/repos/builtin/packages/precice/package.py
@@ -191,7 +191,7 @@ class Precice(CMakePackage):
             python_library = spec["python"].libs[0]
             python_include = spec["python"].headers.directories[0]
             numpy_include = join_path(
-                spec["py-numpy"].prefix, spec["python"].package.platlib, "numpy", "core", "include"
+                spec["py-numpy"].package.module.python_platlib, "numpy", "core", "include"
             )
             cmake_args.extend(
                 [
diff --git a/var/spack/repos/builtin/packages/py-alphafold/package.py b/var/spack/repos/builtin/packages/py-alphafold/package.py
index 674f3ebf10..0269a2abdf 100644
--- a/var/spack/repos/builtin/packages/py-alphafold/package.py
+++ b/var/spack/repos/builtin/packages/py-alphafold/package.py
@@ -77,7 +77,7 @@ class PyAlphafold(PythonPackage, CudaPackage):
     @run_after("install")
     def install_scripts(self):
         mkdirp(self.prefix.bin)
-        shebang = "#!{0}\n".format(self.spec["python"].command)
+        shebang = f"#!{python.path}\n"
         for fname in glob.glob("run_alphafold*.py"):
             destfile = join_path(self.prefix.bin, fname)
             with open(fname, "r") as src:
diff --git a/var/spack/repos/builtin/packages/py-chainer/package.py b/var/spack/repos/builtin/packages/py-chainer/package.py
index 1f78c76a30..a1ab6819a1 100644
--- a/var/spack/repos/builtin/packages/py-chainer/package.py
+++ b/var/spack/repos/builtin/packages/py-chainer/package.py
@@ -59,7 +59,7 @@ class PyChainer(PythonPackage):
 
         mnist_file = join_path(self.install_test_root.examples.chainermn.mnist, "train_mnist.py")
         mpirun = which(self.spec["mpi"].prefix.bin.mpirun)
-        opts = ["-n", "4", self.spec["python"].command.path, mnist_file, "-o", "."]
+        opts = ["-n", "4", python.path, mnist_file, "-o", "."]
         env["OMP_NUM_THREADS"] = "4"
 
         mpirun(*opts)
diff --git a/var/spack/repos/builtin/packages/py-eccodes/package.py b/var/spack/repos/builtin/packages/py-eccodes/package.py
index 107cddc03e..b53c545e4a 100644
--- a/var/spack/repos/builtin/packages/py-eccodes/package.py
+++ b/var/spack/repos/builtin/packages/py-eccodes/package.py
@@ -43,5 +43,4 @@ class PyEccodes(PythonPackage):
 
     def test_selfcheck(self):
         """checking system setup"""
-        python = self.spec["python"].command
         python("-m", "eccodes", "selfcheck")
diff --git a/var/spack/repos/builtin/packages/py-gmxapi/package.py b/var/spack/repos/builtin/packages/py-gmxapi/package.py
index 77a03d6fd4..5869d1ad4b 100644
--- a/var/spack/repos/builtin/packages/py-gmxapi/package.py
+++ b/var/spack/repos/builtin/packages/py-gmxapi/package.py
@@ -49,5 +49,4 @@ class PyGmxapi(PythonPackage):
     def install_test(self):
         with working_dir("spack-test", create=True):
             # test include helper points to right location
-            python = self.spec["python"].command
             python("-m", "pytest", "-x", os.path.join(self.build_directory, "test"))
diff --git a/var/spack/repos/builtin/packages/py-gpaw/package.py b/var/spack/repos/builtin/packages/py-gpaw/package.py
index 8a94a37b1d..6e4181afbd 100644
--- a/var/spack/repos/builtin/packages/py-gpaw/package.py
+++ b/var/spack/repos/builtin/packages/py-gpaw/package.py
@@ -59,7 +59,7 @@ class PyGpaw(PythonPackage):
 
         python_include = spec["python"].headers.directories[0]
         numpy_include = join_path(
-            spec["py-numpy"].prefix, spec["python"].package.platlib, "numpy", "core", "include"
+            spec["py-numpy"].package.module.python_platlib, "numpy", "core", "include"
         )
 
         libs = blas.libs + lapack.libs + libxc.libs
diff --git a/var/spack/repos/builtin/packages/py-installer/package.py b/var/spack/repos/builtin/packages/py-installer/package.py
index 01132b63bb..6ecf2c5b4e 100644
--- a/var/spack/repos/builtin/packages/py-installer/package.py
+++ b/var/spack/repos/builtin/packages/py-installer/package.py
@@ -36,6 +36,4 @@ class PyInstaller(Package, PythonExtension):
         python(*args)
 
     def setup_dependent_package(self, module, dependent_spec):
-        installer = dependent_spec["python"].command
-        installer.add_default_arg("-m", "installer")
-        setattr(module, "installer", installer)
+        setattr(module, "installer", python.with_default_args("-m", "installer"))
diff --git a/var/spack/repos/builtin/packages/py-ipykernel/package.py b/var/spack/repos/builtin/packages/py-ipykernel/package.py
index 2357fbb56d..80490e66f0 100644
--- a/var/spack/repos/builtin/packages/py-ipykernel/package.py
+++ b/var/spack/repos/builtin/packages/py-ipykernel/package.py
@@ -89,4 +89,4 @@ class PyIpykernel(PythonPackage):
     @run_after("install")
     def install_data(self):
         """install the Jupyter kernel spec"""
-        self.spec["python"].command("-m", "ipykernel", "install", "--prefix=" + self.prefix)
+        python("-m", "ipykernel", "install", "--prefix=" + self.prefix)
diff --git a/var/spack/repos/builtin/packages/py-libensemble/package.py b/var/spack/repos/builtin/packages/py-libensemble/package.py
index cf5cbd6b8d..5b553912ae 100644
--- a/var/spack/repos/builtin/packages/py-libensemble/package.py
+++ b/var/spack/repos/builtin/packages/py-libensemble/package.py
@@ -100,7 +100,6 @@ class PyLibensemble(PythonPackage):
         if not os.path.isfile(exe):
             raise SkipTest(f"{script} is missing")
 
-        python = self.spec["python"].command
         python(exe, "--comms", "local", "--nworkers", "2")
 
     def test_uniform_sampling(self):
diff --git a/var/spack/repos/builtin/packages/py-nanobind/package.py b/var/spack/repos/builtin/packages/py-nanobind/package.py
index 909175140b..6fb598acdc 100644
--- a/var/spack/repos/builtin/packages/py-nanobind/package.py
+++ b/var/spack/repos/builtin/packages/py-nanobind/package.py
@@ -60,5 +60,5 @@ class PyNanobind(PythonPackage):
 
     @property
     def cmake_prefix_paths(self):
-        paths = [join_path(self.prefix, self.spec["python"].package.platlib, "nanobind", "cmake")]
+        paths = [join_path(python_platlib, "nanobind", "cmake")]
         return paths
diff --git a/var/spack/repos/builtin/packages/py-pip/package.py b/var/spack/repos/builtin/packages/py-pip/package.py
index 023eab21ec..111e50911b 100644
--- a/var/spack/repos/builtin/packages/py-pip/package.py
+++ b/var/spack/repos/builtin/packages/py-pip/package.py
@@ -77,7 +77,5 @@ class PyPip(Package, PythonExtension):
             args.insert(0, os.path.join(whl, "pip"))
         python(*args)
 
-    def setup_dependent_package(self, module, dependent_spec):
-        pip = dependent_spec["python"].command
-        pip.add_default_arg("-m", "pip")
-        setattr(module, "pip", pip)
+    def setup_dependent_package(self, module, dependent_spec: Spec):
+        setattr(module, "pip", python.with_default_args("-m", "pip"))
diff --git a/var/spack/repos/builtin/packages/py-pybind11/package.py b/var/spack/repos/builtin/packages/py-pybind11/package.py
index 1fa9f7beee..f7c298309b 100644
--- a/var/spack/repos/builtin/packages/py-pybind11/package.py
+++ b/var/spack/repos/builtin/packages/py-pybind11/package.py
@@ -102,7 +102,6 @@ class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder):
 
         with working_dir("spack-test", create=True):
             # test include helper points to right location
-            python = self.spec["python"].command
             py_inc = python(
                 "-c", "import pybind11 as py; print(py.get_include())", output=str
             ).strip()
diff --git a/var/spack/repos/builtin/packages/py-pymol/package.py b/var/spack/repos/builtin/packages/py-pymol/package.py
index fe9bcce3a8..a4ab0b522d 100644
--- a/var/spack/repos/builtin/packages/py-pymol/package.py
+++ b/var/spack/repos/builtin/packages/py-pymol/package.py
@@ -58,7 +58,7 @@ class PyPymol(PythonPackage):
         script = join_path(python_platlib, "pymol", "__init__.py")
 
         shebang = "#!/bin/sh\n"
-        fdata = 'exec {0} {1} "$@"'.format(self.spec["python"].command, script)
+        fdata = f'exec {python.path} {script} "$@"'
         with open(fname, "w") as new:
             new.write(shebang + fdata)
         set_executable(fname)
diff --git a/var/spack/repos/builtin/packages/py-pyqt4/package.py b/var/spack/repos/builtin/packages/py-pyqt4/package.py
index 31e481d305..00e27c994e 100644
--- a/var/spack/repos/builtin/packages/py-pyqt4/package.py
+++ b/var/spack/repos/builtin/packages/py-pyqt4/package.py
@@ -48,7 +48,7 @@ class PyPyqt4(SIPPackage):
             "--destdir",
             python_platlib,
             "--pyuic4-interpreter",
-            self.spec["python"].command.path,
+            python.path,
             "--sipdir",
             self.prefix.share.sip.PyQt4,
             "--stubsdir",
diff --git a/var/spack/repos/builtin/packages/py-pyspark/package.py b/var/spack/repos/builtin/packages/py-pyspark/package.py
index 087378d753..058ac47bf7 100644
--- a/var/spack/repos/builtin/packages/py-pyspark/package.py
+++ b/var/spack/repos/builtin/packages/py-pyspark/package.py
@@ -25,5 +25,5 @@ class PyPyspark(PythonPackage):
     depends_on("py-py4j@0.10.9", when="@3.0.1:3.1.3", type=("build", "run"))
 
     def setup_run_environment(self, env):
-        env.set("PYSPARK_PYTHON", self.spec["python"].command.path)
-        env.set("PYSPARK_DRIVER_PYTHON", self.spec["python"].command.path)
+        env.set("PYSPARK_PYTHON", python.path)
+        env.set("PYSPARK_DRIVER_PYTHON", python.path)
diff --git a/var/spack/repos/builtin/packages/py-pythonsollya/package.py b/var/spack/repos/builtin/packages/py-pythonsollya/package.py
index 84a715c8d1..c8f427f873 100644
--- a/var/spack/repos/builtin/packages/py-pythonsollya/package.py
+++ b/var/spack/repos/builtin/packages/py-pythonsollya/package.py
@@ -34,9 +34,4 @@ class PyPythonsollya(PythonPackage):
 
     @run_before("install")
     def patch(self):
-        filter_file(
-            "PYTHON ?= python2",
-            "PYTHON ?= " + self.spec["python"].command.path,
-            "GNUmakefile",
-            string=True,
-        )
+        filter_file("PYTHON ?= python2", f"PYTHON ?= {python.path}", "GNUmakefile", string=True)
diff --git a/var/spack/repos/builtin/packages/py-tensorflow/package.py b/var/spack/repos/builtin/packages/py-tensorflow/package.py
index 9a25ea09d2..56d12ff231 100644
--- a/var/spack/repos/builtin/packages/py-tensorflow/package.py
+++ b/var/spack/repos/builtin/packages/py-tensorflow/package.py
@@ -410,7 +410,7 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
         spec = self.spec
 
         # Please specify the location of python
-        env.set("PYTHON_BIN_PATH", spec["python"].command.path)
+        env.set("PYTHON_BIN_PATH", python.path)
 
         # Please input the desired Python library path to use
         env.set("PYTHON_LIB_PATH", python_platlib)
diff --git a/var/spack/repos/builtin/packages/py-torch/package.py b/var/spack/repos/builtin/packages/py-torch/package.py
index 5de618494d..e7824283fd 100644
--- a/var/spack/repos/builtin/packages/py-torch/package.py
+++ b/var/spack/repos/builtin/packages/py-torch/package.py
@@ -688,7 +688,5 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
 
     @property
     def cmake_prefix_paths(self):
-        cmake_prefix_paths = [
-            join_path(self.prefix, self.spec["python"].package.platlib, "torch", "share", "cmake")
-        ]
+        cmake_prefix_paths = [join_path(python_platlib, "torch", "share", "cmake")]
         return cmake_prefix_paths
diff --git a/var/spack/repos/builtin/packages/py-xdot/package.py b/var/spack/repos/builtin/packages/py-xdot/package.py
index bf898a117e..73d703a287 100644
--- a/var/spack/repos/builtin/packages/py-xdot/package.py
+++ b/var/spack/repos/builtin/packages/py-xdot/package.py
@@ -50,8 +50,7 @@ class PyXdot(PythonPackage):
             dst,
         )
         # regenerate the byte-compiled __init__.py
-        python3 = spec["python"].command
-        python3("-m", "compileall", dst)
+        python("-m", "compileall", dst)
 
     def setup_run_environment(self, env):
         spec = self.spec
diff --git a/var/spack/repos/builtin/packages/python-venv/package.py b/var/spack/repos/builtin/packages/python-venv/package.py
new file mode 100644
index 0000000000..cd0fd84371
--- /dev/null
+++ b/var/spack/repos/builtin/packages/python-venv/package.py
@@ -0,0 +1,100 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+import shutil
+
+import llnl.util.filesystem as fs
+
+from spack.package import *
+
+
+class PythonVenv(Package):
+    """A Spack managed Python virtual environment"""
+
+    homepage = "https://docs.python.org/3/library/venv.html"
+    has_code = False
+
+    maintainers("haampie")
+
+    version("1.0")
+
+    extends("python")
+
+    def install(self, spec, prefix):
+        # Create a virtual environment
+        python("-m", "venv", "--without-pip", prefix)
+
+    def add_files_to_view(self, view, merge_map: Dict[str, str], skip_if_exists=True):
+        for src, dst in merge_map.items():
+            if skip_if_exists and os.path.lexists(dst):
+                continue
+
+            name = os.path.basename(dst)
+
+            # Replace the VIRTUAL_ENV variable in the activate scripts after copying
+            if name.lower().startswith("activate"):
+                shutil.copy(src, dst)
+                fs.filter_file(
+                    self.spec.prefix,
+                    os.path.abspath(view.get_projection_for_spec(self.spec)),
+                    dst,
+                    string=True,
+                )
+            else:
+                view.link(src, dst)
+
+    @property
+    def bindir(self):
+        windows = self.spec.satisfies("platform=windows")
+        return join_path(self.prefix, "Scripts" if windows else "bin")
+
+    @property
+    def command(self):
+        """Returns a python Executable instance"""
+        return which("python3", path=self.bindir)
+
+    def _get_path(self, name) -> str:
+        return self.command(
+            "-Ec", f"import sysconfig; print(sysconfig.get_path('{name}'))", output=str
+        ).strip()
+
+    @property
+    def platlib(self) -> str:
+        """Directory for site-specific, platform-specific files."""
+        relative_platlib = os.path.relpath(self._get_path("platlib"), self.prefix)
+        assert not relative_platlib.startswith("..")
+        return relative_platlib
+
+    @property
+    def purelib(self) -> str:
+        """Directory for site-specific, non-platform-specific files."""
+        relative_purelib = os.path.relpath(self._get_path("purelib"), self.prefix)
+        assert not relative_purelib.startswith("..")
+        return relative_purelib
+
+    @property
+    def headers(self):
+        return HeaderList([])
+
+    @property
+    def libs(self):
+        return LibraryList([])
+
+    def setup_dependent_run_environment(self, env, dependent_spec):
+        """Set PYTHONPATH to include the site-packages directory for the
+        extension and any other python extensions it depends on."""
+        # Packages may be installed in platform-specific or platform-independent site-packages
+        # directories
+        for directory in {self.platlib, self.purelib}:
+            path = os.path.join(dependent_spec.prefix, directory)
+            if os.path.isdir(path):
+                env.prepend_path("PYTHONPATH", path)
+
+    def setup_dependent_package(self, module, dependent_spec):
+        """Called before python modules' install() methods."""
+        module.python = self.command
+        module.python_platlib = join_path(dependent_spec.prefix, self.platlib)
+        module.python_purelib = join_path(dependent_spec.prefix, self.purelib)
diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
index ad494226ee..b87ee81305 100644
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -8,19 +8,16 @@ import json
 import os
 import platform
 import re
-import stat
 import subprocess
 import sys
 from shutil import copy
-from typing import Dict, List, Tuple
+from typing import Dict, List
 
 import llnl.util.tty as tty
-from llnl.util.filesystem import is_nonsymlink_exe_with_shebang, path_contains_subdirectory
 from llnl.util.lang import dedupe
 
 from spack.build_environment import dso_suffix, stat_suffix
 from spack.package import *
-from spack.util.environment import is_system_path
 from spack.util.prefix import Prefix
 
 
@@ -1115,7 +1112,7 @@ print(json.dumps(config))
         path = self.config_vars["platlib"]
         if path.startswith(prefix):
             return path.replace(prefix, "")
-        return os.path.join("lib64", "python{}".format(self.version.up_to(2)), "site-packages")
+        return os.path.join("lib64", f"python{self.version.up_to(2)}", "site-packages")
 
     @property
     def purelib(self):
@@ -1135,7 +1132,7 @@ print(json.dumps(config))
         path = self.config_vars["purelib"]
         if path.startswith(prefix):
             return path.replace(prefix, "")
-        return os.path.join("lib", "python{}".format(self.version.up_to(2)), "site-packages")
+        return os.path.join("lib", f"python{self.version.up_to(2)}", "site-packages")
 
     @property
     def include(self):
@@ -1163,39 +1160,6 @@ print(json.dumps(config))
         """Set PYTHONPATH to include the site-packages directory for the
         extension and any other python extensions it depends on.
         """
-        # Ensure the current Python is first in the PATH
-        path = os.path.dirname(self.command.path)
-        if not is_system_path(path):
-            env.prepend_path("PATH", path)
-
-        # Add installation prefix to PYTHONPATH, needed to run import tests
-        prefixes = set()
-        if dependent_spec.package.extends(self.spec):
-            prefixes.add(dependent_spec.prefix)
-
-        # Add direct build/run/test dependencies to PYTHONPATH,
-        # needed to build the package and to run import tests
-        for direct_dep in dependent_spec.dependencies(deptype=("build", "run", "test")):
-            if direct_dep.package.extends(self.spec):
-                prefixes.add(direct_dep.prefix)
-
-                # Add recursive run dependencies of all direct dependencies,
-                # needed by direct dependencies at run-time
-                for indirect_dep in direct_dep.traverse(deptype="run"):
-                    if indirect_dep.package.extends(self.spec):
-                        prefixes.add(indirect_dep.prefix)
-
-        for prefix in prefixes:
-            # Packages may be installed in platform-specific or platform-independent
-            # site-packages directories
-            for directory in {self.platlib, self.purelib}:
-                env.prepend_path("PYTHONPATH", os.path.join(prefix, directory))
-
-            if self.spec.satisfies("platform=windows"):
-                prefix_scripts_dir = prefix.Scripts
-                if os.path.exists(prefix_scripts_dir):
-                    env.prepend_path("PATH", prefix_scripts_dir)
-
         # We need to make sure that the extensions are compiled and linked with
         # the Spack wrapper. Paths to the executables that are used for these
         # operations are normally taken from the sysconfigdata file, which we
@@ -1241,9 +1205,7 @@ print(json.dumps(config))
                 # invoked directly (no change would be required in that case
                 # because Spack arranges for the Spack ld wrapper to be the
                 # first instance of "ld" in PATH).
-                new_link = config_link.replace(
-                    " {0} ".format(config_compile), " {0} ".format(new_compile)
-                )
+                new_link = config_link.replace(f" {config_compile} ", f" {new_compile} ")
 
             # There is logic in the sysconfig module that is sensitive to the
             # fact that LDSHARED is set in the environment, therefore we export
@@ -1256,66 +1218,23 @@ print(json.dumps(config))
         """Set PYTHONPATH to include the site-packages directory for the
         extension and any other python extensions it depends on.
         """
-        if dependent_spec.package.extends(self.spec):
-            # Packages may be installed in platform-specific or platform-independent
-            # site-packages directories
-            for directory in {self.platlib, self.purelib}:
-                env.prepend_path("PYTHONPATH", os.path.join(dependent_spec.prefix, directory))
+        if not dependent_spec.package.extends(self.spec) or dependent_spec.dependencies(
+            "python-venv"
+        ):
+            return
+
+        # Packages may be installed in platform-specific or platform-independent site-packages
+        # directories
+        for directory in {self.platlib, self.purelib}:
+            env.prepend_path("PYTHONPATH", os.path.join(dependent_spec.prefix, directory))
 
     def setup_dependent_package(self, module, dependent_spec):
         """Called before python modules' install() methods."""
-
         module.python = self.command
-
         module.python_include = join_path(dependent_spec.prefix, self.include)
         module.python_platlib = join_path(dependent_spec.prefix, self.platlib)
         module.python_purelib = join_path(dependent_spec.prefix, self.purelib)
 
-    def add_files_to_view(self, view, merge_map, skip_if_exists=True):
-        # The goal is to copy the `python` executable, so that its search paths are relative to the
-        # view instead of the install prefix. This is an obsolete way of creating something that
-        # resembles a virtual environnent. Also we copy scripts with shebang lines. Finally we need
-        # to re-target symlinks pointing to copied files.
-        bin_dir = self.spec.prefix.bin if sys.platform != "win32" else self.spec.prefix
-        copied_files: Dict[Tuple[int, int], str] = {}  # File identifier -> source
-        delayed_links: List[Tuple[str, str]] = []  # List of symlinks from merge map
-        for src, dst in merge_map.items():
-            if skip_if_exists and os.path.lexists(dst):
-                continue
-
-            # Files not in the bin dir are linked the default way.
-            if not path_contains_subdirectory(src, bin_dir):
-                view.link(src, dst, spec=self.spec)
-                continue
-
-            s = os.lstat(src)
-
-            # Symlink is delayed because we may need to re-target if its target is copied in view
-            if stat.S_ISLNK(s.st_mode):
-                delayed_links.append((src, dst))
-                continue
-
-            # Anything that's not a symlink gets copied. Scripts with shebangs are immediately
-            # updated when necessary.
-            copied_files[(s.st_dev, s.st_ino)] = dst
-            copy(src, dst)
-            if is_nonsymlink_exe_with_shebang(src):
-                filter_file(
-                    self.spec.prefix, os.path.abspath(view.get_projection_for_spec(self.spec)), dst
-                )
-
-        # Finally re-target the symlinks that point to copied files.
-        for src, dst in delayed_links:
-            try:
-                s = os.stat(src)
-                target = copied_files[(s.st_dev, s.st_ino)]
-            except (OSError, KeyError):
-                target = None
-            if target:
-                os.symlink(os.path.relpath(target, os.path.dirname(dst)), dst)
-            else:
-                view.link(src, dst, spec=self.spec)
-
     def test_hello_world(self):
         """run simple hello world program"""
         # do not use self.command because we are also testing the run env
diff --git a/var/spack/repos/builtin/packages/qgis/package.py b/var/spack/repos/builtin/packages/qgis/package.py
index 051a78660d..196a52622f 100644
--- a/var/spack/repos/builtin/packages/qgis/package.py
+++ b/var/spack/repos/builtin/packages/qgis/package.py
@@ -188,9 +188,7 @@ class Qgis(CMakePackage):
     @run_before("cmake", when="^py-pyqt5")
     def fix_pyqt5_cmake(self):
         cmfile = FileFilter(join_path("cmake", "FindPyQt5.cmake"))
-        pyqtpath = join_path(
-            self.spec["py-pyqt5"].prefix, self.spec["python"].package.platlib, "PyQt5"
-        )
+        pyqtpath = join_path(self.spec["py-pyqt5"].package.module.python_platlib, "PyQt5")
         cmfile.filter(
             'SET(PYQT5_MOD_DIR "${Python_SITEARCH}/PyQt5")',
             'SET(PYQT5_MOD_DIR "' + pyqtpath + '")',
@@ -210,7 +208,7 @@ class Qgis(CMakePackage):
             pyqtx = "PyQt6"
 
         sip_inc_dir = join_path(
-            self.spec["qscintilla"].prefix, self.spec["python"].package.platlib, pyqtx, "bindings"
+            self.spec["qscintilla"].package.module.python_platlib, pyqtx, "bindings"
         )
         with open(join_path("python", "gui", "pyproject.toml.in"), "a") as tomlfile:
             tomlfile.write(f'\n[tool.sip.project]\nsip-include-dirs = ["{sip_inc_dir}"]\n')
diff --git a/var/spack/repos/builtin/packages/qscintilla/package.py b/var/spack/repos/builtin/packages/qscintilla/package.py
index bbf0b5a309..ce39d02d45 100644
--- a/var/spack/repos/builtin/packages/qscintilla/package.py
+++ b/var/spack/repos/builtin/packages/qscintilla/package.py
@@ -101,7 +101,7 @@ class Qscintilla(QMakePackage):
         with working_dir(join_path(self.stage.source_path, "Python")):
             copy(ftoml, "pyproject.toml")
             sip_inc_dir = join_path(
-                self.spec[py_pyqtx].prefix, self.spec["python"].package.platlib, pyqtx, "bindings"
+                self.spec[py_pyqtx].package.module.python_platlib, pyqtx, "bindings"
             )
 
             with open("pyproject.toml", "a") as tomlfile:
diff --git a/var/spack/repos/builtin/packages/redland-bindings/package.py b/var/spack/repos/builtin/packages/redland-bindings/package.py
index 964f62108f..3c6bb42abc 100644
--- a/var/spack/repos/builtin/packages/redland-bindings/package.py
+++ b/var/spack/repos/builtin/packages/redland-bindings/package.py
@@ -3,8 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import os
-
 from spack.package import *
 
 
@@ -27,11 +25,4 @@ class RedlandBindings(AutotoolsPackage):
     extends("python")
 
     def configure_args(self):
-        plib = self.spec["python"].prefix.lib
-        plib64 = self.spec["python"].prefix.lib64
-        mybase = self.prefix.lib
-        if os.path.isdir(plib64) and not os.path.isdir(plib):
-            mybase = self.prefix.lib64
-        pver = "python{0}".format(self.spec["python"].version.up_to(2))
-        myplib = join_path(mybase, pver, "site-packages")
-        return ["--with-python", "PYTHON_LIB={0}".format(myplib)]
+        return ["--with-python", f"PYTHON_LIB={python_platlib}"]
diff --git a/var/spack/repos/builtin/packages/z3/package.py b/var/spack/repos/builtin/packages/z3/package.py
index e386e050c8..b42d46655a 100644
--- a/var/spack/repos/builtin/packages/z3/package.py
+++ b/var/spack/repos/builtin/packages/z3/package.py
@@ -53,13 +53,6 @@ class Z3(CMakePackage):
         ]
 
         if spec.satisfies("+python"):
-            args.append(
-                self.define(
-                    "CMAKE_INSTALL_PYTHON_PKG_DIR",
-                    join_path(
-                        prefix.lib, "python%s" % spec["python"].version.up_to(2), "site-packages"
-                    ),
-                )
-            )
+            args.append(self.define("CMAKE_INSTALL_PYTHON_PKG_DIR", python_platlib))
 
         return args
diff --git a/var/spack/repos/duplicates.test/packages/python-venv b/var/spack/repos/duplicates.test/packages/python-venv
new file mode 120000
index 0000000000..a9a1ce867a
--- /dev/null
+++ b/var/spack/repos/duplicates.test/packages/python-venv
@@ -0,0 +1 @@
+../../builtin.mock/packages/python-venv
\ No newline at end of file