Revert python build isolation & setuptools source install (#40796)

* Revert "Improve build isolation in PythonPipBuilder (#40224)"

This reverts commit 0f43074f3e93d13445ea662a13f5672f960947c2.

* Revert "py-setuptools: sdist + rpath patch backport (#40205)"

This reverts commit 512e41a84aa794ec0cc53872aaa6c228c36e0b49.

3 files changed, 208 insertions, 163 deletions

diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py
index 7f71cbae70..521994b1ec 100644
--- a/lib/spack/spack/build_systems/python.py
+++ b/lib/spack/spack/build_systems/python.py
@@ -6,7 +6,6 @@ import inspect
 import os
 import re
 import shutil
-import stat
 from typing import Optional
 
 import archspec
@@ -26,7 +25,6 @@ import spack.store
 from spack.directives import build_system, depends_on, extends, maintainers
 from spack.error import NoHeadersError, NoLibrariesError
 from spack.install_test import test_part
-from spack.util.executable import Executable
 
 from ._checks import BaseBuilder, execute_install_time_tests
 
@@ -369,51 +367,6 @@ class PythonPackage(PythonExtension):
         raise NoLibrariesError(msg.format(self.spec.name, root))
 
 
-def fixup_shebangs(path: str, old_interpreter: bytes, new_interpreter: bytes):
-    # Recurse into the install prefix and fixup shebangs
-    exe = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
-    dirs = [path]
-    hardlinks = set()
-
-    while dirs:
-        with os.scandir(dirs.pop()) as entries:
-            for entry in entries:
-                if entry.is_dir(follow_symlinks=False):
-                    dirs.append(entry.path)
-                    continue
-
-                # Only consider files, not symlinks
-                if not entry.is_file(follow_symlinks=False):
-                    continue
-
-                lstat = entry.stat(follow_symlinks=False)
-
-                # Skip over files that are not executable
-                if not (lstat.st_mode & exe):
-                    continue
-
-                # Don't modify hardlinks more than once
-                if lstat.st_nlink > 1:
-                    key = (lstat.st_ino, lstat.st_dev)
-                    if key in hardlinks:
-                        continue
-                    hardlinks.add(key)
-
-                # Finally replace shebangs if any.
-                with open(entry.path, "rb+") as f:
-                    contents = f.read(2)
-                    if contents != b"#!":
-                        continue
-                    contents += f.read()
-
-                    if old_interpreter not in contents:
-                        continue
-
-                    f.seek(0)
-                    f.write(contents.replace(old_interpreter, new_interpreter))
-                    f.truncate()
-
-
 @spack.builder.builder("python_pip")
 class PythonPipBuilder(BaseBuilder):
     phases = ("install",)
@@ -511,36 +464,8 @@ class PythonPipBuilder(BaseBuilder):
         """
         return []
 
-    @property
-    def _build_venv_path(self):
-        """Return the path to the virtual environment used for building when
-        python is external."""
-        return os.path.join(self.spec.package.stage.path, "build_env")
-
-    @property
-    def _build_venv_python(self) -> Executable:
-        """Return the Python executable in the build virtual environment when
-        python is external."""
-        return Executable(os.path.join(self._build_venv_path, "bin", "python"))
-
     def install(self, pkg, spec, prefix):
         """Install everything from build directory."""
-        python: Executable = spec["python"].command
-        # Since we invoke pip with --no-build-isolation, we have to make sure that pip cannot
-        # execute hooks from user and system site-packages.
-        if spec["python"].external:
-            # There are no environment variables to disable the system site-packages, so we use a
-            # virtual environment instead. The downside of this approach is that pip produces
-            # incorrect shebangs that refer to the virtual environment, which we have to fix up.
-            python("-m", "venv", "--without-pip", self._build_venv_path)
-            pip = self._build_venv_python
-        else:
-            # For a Spack managed Python, system site-packages is empty/unused by design, so it
-            # suffices to disable user site-packages, for which there is an environment variable.
-            pip = python
-            pip.add_default_env("PYTHONNOUSERSITE", "1")
-        pip.add_default_arg("-m")
-        pip.add_default_arg("pip")
 
         args = PythonPipBuilder.std_args(pkg) + [f"--prefix={prefix}"]
 
@@ -556,31 +481,15 @@ class PythonPipBuilder(BaseBuilder):
         else:
             args.append(".")
 
+        pip = spec["python"].command
+        # Hide user packages, since we don't have build isolation. This is
+        # necessary because pip / setuptools may run hooks from arbitrary
+        # packages during the build. There is no equivalent variable to hide
+        # system packages, so this is not reliable for external Python.
+        pip.add_default_env("PYTHONNOUSERSITE", "1")
+        pip.add_default_arg("-m")
+        pip.add_default_arg("pip")
         with fs.working_dir(self.build_directory):
             pip(*args)
 
-    @spack.builder.run_after("install")
-    def fixup_shebangs_pointing_to_build(self):
-        """When installing a package using an external python, we use a temporary virtual
-        environment which improves build isolation. The downside is that pip produces shebangs
-        that point to the temporary virtual environment. This method fixes them up to point to the
-        underlying Python."""
-        # No need to fixup shebangs if no build venv was used. (this post install function also
-        # runs when install was overridden in another package, so check existence of the venv path)
-        if not os.path.exists(self._build_venv_path):
-            return
-
-        # Use sys.executable, since that's what pip uses.
-        interpreter = (
-            lambda python: python("-c", "import sys; print(sys.executable)", output=str)
-            .strip()
-            .encode("utf-8")
-        )
-
-        fixup_shebangs(
-            path=self.spec.prefix,
-            old_interpreter=interpreter(self._build_venv_python),
-            new_interpreter=interpreter(self.spec["python"].command),
-        )
-
     spack.builder.run_after("install")(execute_install_time_tests)
diff --git a/var/spack/repos/builtin/packages/py-setuptools/package.py b/var/spack/repos/builtin/packages/py-setuptools/package.py
index 03487bfaa0..36cc6de4ee 100644
--- a/var/spack/repos/builtin/packages/py-setuptools/package.py
+++ b/var/spack/repos/builtin/packages/py-setuptools/package.py
@@ -6,60 +6,191 @@
 from spack.package import *
 
 
-class PySetuptools(PythonPackage):
+class PySetuptools(Package, PythonExtension):
     """A Python utility that aids in the process of downloading, building,
     upgrading, installing, and uninstalling Python packages."""
 
     homepage = "https://github.com/pypa/setuptools"
-    pypi = "setuptools/setuptools-62.3.2.tar.gz"
+    url = "https://files.pythonhosted.org/packages/py3/s/setuptools/setuptools-62.3.2-py3-none-any.whl"
+    list_url = "https://pypi.org/simple/setuptools/"
 
     tags = ["build-tools"]
 
-    version("68.0.0", sha256="baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235")
-    version("67.6.0", sha256="2ee892cd5f29f3373097f5a814697e397cf3ce313616df0af11231e2ad118077")
-    version("65.5.0", sha256="512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17")
-    version("65.0.0", sha256="d73f8cd714a1a6691f5eb5abeeacbf313242b7aa2f5eba93776542c1aad90c6f")
-    version("64.0.0", sha256="9b5d2cb8df48f005825654e0cb17217418317e4d996c035f0bca7cbaeb8acf51")
-    version("63.4.3", sha256="521c833d1e5e1ef0869940e7f486a83de7773b9f029010ad0c2fe35453a9dad9")
-    version("63.0.0", sha256="7388e17e72f5c0c7279f59da950a7925910e35bc1a84e19d3affbb40da248d1d")
-    version("62.6.0", sha256="990a4f7861b31532871ab72331e755b5f14efbe52d336ea7f6118144dd478741")
-    version("62.4.0", sha256="bf8a748ac98b09d32c9a64a995a6b25921c96cc5743c1efa82763ba80ff54e91")
-    version("62.3.2", sha256="a43bdedf853c670e5fed28e5623403bad2f73cf02f9a2774e91def6bda8265a7")
-    version("59.4.0", sha256="b4c634615a0cf5b02cf83c7bedffc8da0ca439f00e79452699454da6fbd4153d")
-    version("58.2.0", sha256="2c55bdb85d5bb460bd2e3b12052b677879cffcf46c0c688f2e5bf51d36001145")
-    version("57.4.0", sha256="6bac238ffdf24e8806c61440e755192470352850f3419a52f26ffe0a1a64f465")
-    version("57.1.0", sha256="cfca9c97e7eebbc8abe18d5e5e962a08dcad55bb63afddd82d681de4d22a597b")
-    version("51.0.0", sha256="029c49fd713e9230f6a41c0298e6e1f5839f2cde7104c0ad5e053a37777e7688")
-    version("50.3.2", sha256="ed0519d27a243843b05d82a5e9d01b0b083d9934eaa3d02779a23da18077bd3c")
-    version("50.1.0", sha256="4a7708dafd2d360ce5e2ac7577374da9fb65fc867bc4cdaf461f9f834dfa6ac3")
-    version("49.6.0", sha256="46bd862894ed22c2edff033c758c2dc026324788d758e96788e8f7c11f4e9707")
-    version("49.2.0", sha256="afe9e81fee0270d3f60d52608549cc8ec4c46dada8c95640c1a00160f577acf2")
-    version("46.1.3", sha256="795e0475ba6cd7fa082b1ee6e90d552209995627a2a227a47c6ea93282f4bfb1")
-    version("44.1.1", sha256="c67aa55db532a0dadc4d2e20ba9961cbd3ccc84d544e9029699822542b5a476b")
-    version("44.1.0", sha256="794a96b0c1dc6f182c36b72ab70d7e90f1d59f7a132e6919bb37b4fd4d424aca")
-    version("43.0.0", sha256="db45ebb4a4b3b95ff0aca3ce5fe1e820ce17be393caf8902c78aa36240e8c378")
-    version("41.4.0", sha256="7eae782ccf36b790c21bde7d86a4f303a441cd77036b25c559a602cf5186ce4d")
-    version("41.3.0", sha256="9f5c54b529b2156c6f288e837e625581bb31ff94d4cfd116b8f271c589749556")
-    version("41.0.1", sha256="a222d126f5471598053c9a77f4b5d4f26eaa1f150ad6e01dcf1a42e185d05613")
-    version("41.0.0", sha256="79d30254b6fe7a8e672e43cd85f13a9f3f2a50080bc81d851143e2219ef0dcb1")
-    version("40.8.0", sha256="6e4eec90337e849ade7103723b9a99631c1f0d19990d6e8412dc42f5ae8b304d")
-    version("40.4.3", sha256="acbc5740dd63f243f46c2b4b8e2c7fd92259c2ddb55a4115b16418a2ed371b15")
-    version("40.2.0", sha256="47881d54ede4da9c15273bac65f9340f8929d4f0213193fa7894be384f2dcfa6")
-    version("39.2.0", sha256="f7cddbb5f5c640311eb00eab6e849f7701fa70bf6a183fc8a2c33dd1d1672fb2")
-    version("39.0.1", sha256="bec7badf0f60e7fc8153fac47836edc41b74e5d541d7692e614e635720d6a7c7")
-    version("25.2.0", sha256="b2757ddac2c41173140b111e246d200768f6dd314110e1e40661d0ecf9b4d6a6")
-    version("20.7.0", sha256="505cdf282c5f6e3a056e79f0244b8945f3632257bba8469386c6b9b396400233")
-    version("20.6.7", sha256="d20152ee6337323d3b6d95cd733fb719d6b4f3fbc40f61f7a48e5a1bb96478b2")
-
-    def url_for_version(self, version):
-        url = self.url.rsplit("/", 1)[0]
-        if version.satisfies(ver("32.1.2:51.0.0")):
-            url += "/setuptools-{}.zip"
-        else:
-            url += "/setuptools-{}.tar.gz"
-        return url.format(version)
-
-    patch("rpath-compiler-flag.patch", when="@48:58.2")
+    version(
+        "68.0.0",
+        sha256="11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f",
+        expand=False,
+    )
+    version(
+        "67.6.0",
+        sha256="b78aaa36f6b90a074c1fa651168723acbf45d14cb1196b6f02c0fd07f17623b2",
+        expand=False,
+    )
+    version(
+        "65.5.0",
+        sha256="f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356",
+        expand=False,
+    )
+    version(
+        "65.0.0",
+        sha256="fe9a97f68b064a6ddd4bacfb0b4b93a4c65a556d97ce906255540439d0c35cef",
+        expand=False,
+    )
+    version(
+        "64.0.0",
+        sha256="63f463b90ff5e0a1422010100268fd688e15c44ae0798659013c8412963e15e4",
+        expand=False,
+    )
+    version(
+        "63.4.3",
+        sha256="7f61f7e82647f77d4118eeaf43d64cbcd4d87e38af9611694d4866eb070cd10d",
+        expand=False,
+    )
+    version(
+        "63.0.0",
+        sha256="045aec56a3eee5c82373a70e02db8b6da9a10f7faf61ff89a14ab66c738ed370",
+        expand=False,
+    )
+    version(
+        "62.6.0",
+        sha256="c1848f654aea2e3526d17fc3ce6aeaa5e7e24e66e645b5be2171f3f6b4e5a178",
+        expand=False,
+    )
+    version(
+        "62.4.0",
+        sha256="5a844ad6e190dccc67d6d7411d119c5152ce01f7c76be4d8a1eaa314501bba77",
+        expand=False,
+    )
+    version(
+        "62.3.2",
+        sha256="68e45d17c9281ba25dc0104eadd2647172b3472d9e01f911efa57965e8d51a36",
+        expand=False,
+    )
+    version(
+        "59.4.0",
+        sha256="feb5ff19b354cde9efd2344ef6d5e79880ce4be643037641b49508bbb850d060",
+        expand=False,
+    )
+    version(
+        "58.2.0",
+        sha256="2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11",
+        expand=False,
+    )
+    version(
+        "57.4.0",
+        sha256="a49230977aa6cfb9d933614d2f7b79036e9945c4cdd7583163f4e920b83418d6",
+        expand=False,
+    )
+    version(
+        "57.1.0",
+        sha256="ddae4c1b9220daf1e32ba9d4e3714df6019c5b583755559be84ff8199f7e1fe3",
+        expand=False,
+    )
+    version(
+        "51.0.0",
+        sha256="8c177936215945c9a37ef809ada0fab365191952f7a123618432bbfac353c529",
+        expand=False,
+    )
+    version(
+        "50.3.2",
+        sha256="2c242a0856fbad7efbe560df4a7add9324f340cf48df43651e9604924466794a",
+        expand=False,
+    )
+    version(
+        "50.1.0",
+        sha256="4537c77e6e7dc170081f8547564551d4ff4e4999717434e1257600bbd3a23296",
+        expand=False,
+    )
+    version(
+        "49.6.0",
+        sha256="4dd5bb0a0a0cff77b46ca5dd3a84857ee48c83e8223886b556613c724994073f",
+        expand=False,
+    )
+    version(
+        "49.2.0",
+        sha256="272c7f48f5cddc5af5901f4265274c421c7eede5c8bc454ac2903d3f8fc365e9",
+        expand=False,
+    )
+    version(
+        "46.1.3",
+        sha256="4fe404eec2738c20ab5841fa2d791902d2a645f32318a7850ef26f8d7215a8ee",
+        expand=False,
+    )
+    version(
+        "44.1.1",
+        sha256="27a714c09253134e60a6fa68130f78c7037e5562c4f21f8f318f2ae900d152d5",
+        expand=False,
+    )
+    version(
+        "44.1.0",
+        sha256="992728077ca19db6598072414fb83e0a284aca1253aaf2e24bb1e55ee6db1a30",
+        expand=False,
+    )
+    version(
+        "43.0.0",
+        sha256="a67faa51519ef28cd8261aff0e221b6e4c370f8fb8bada8aa3e7ad8945199963",
+        expand=False,
+    )
+    version(
+        "41.4.0",
+        sha256="8d01f7ee4191d9fdcd9cc5796f75199deccb25b154eba82d44d6a042cf873670",
+        expand=False,
+    )
+    version(
+        "41.3.0",
+        sha256="e9832acd9be6f3174f4c34b40e7d913a146727920cbef6465c1c1bd2d21a4ec4",
+        expand=False,
+    )
+    version(
+        "41.0.1",
+        sha256="c7769ce668c7a333d84e17fe8b524b1c45e7ee9f7908ad0a73e1eda7e6a5aebf",
+        expand=False,
+    )
+    version(
+        "41.0.0",
+        sha256="e67486071cd5cdeba783bd0b64f5f30784ff855b35071c8670551fd7fc52d4a1",
+        expand=False,
+    )
+    version(
+        "40.8.0",
+        sha256="e8496c0079f3ac30052ffe69b679bd876c5265686127a3159cfa415669b7f9ab",
+        expand=False,
+    )
+    version(
+        "40.4.3",
+        sha256="ce4137d58b444bac11a31d4e0c1805c69d89e8ed4e91fde1999674ecc2f6f9ff",
+        expand=False,
+    )
+    version(
+        "40.2.0",
+        sha256="ea3796a48a207b46ea36a9d26de4d0cc87c953a683a7b314ea65d666930ea8e6",
+        expand=False,
+    )
+    version(
+        "39.2.0",
+        sha256="8fca9275c89964f13da985c3656cb00ba029d7f3916b37990927ffdf264e7926",
+        expand=False,
+    )
+    version(
+        "39.0.1",
+        sha256="8010754433e3211b9cdbbf784b50f30e80bf40fc6b05eb5f865fab83300599b8",
+        expand=False,
+    )
+    version(
+        "25.2.0",
+        sha256="2845247c359bb91097ccf8f6be8a69edfa44847f3d2d5def39aa43c3d7f615ca",
+        expand=False,
+    )
+    version(
+        "20.7.0",
+        sha256="8917a52aa3a389893221b173a89dae0471022d32bff3ebc31a1072988aa8039d",
+        expand=False,
+    )
+    version(
+        "20.6.7",
+        sha256="9982ee4d279a2541dc1a7efee994ff9c535cfc05315e121e09df7f93da48c442",
+        expand=False,
+    )
 
     extends("python")
 
@@ -69,10 +200,6 @@ class PySetuptools(PythonPackage):
     depends_on("python@2.7:2.8,3.5:", when="@44", type=("build", "run"))
     depends_on("python@2.7:2.8,3.4:", when="@:43", type=("build", "run"))
 
-    # Newer pip requires setuptools to be installed, before building
-    # setuptools. This issue was fixed or worked around in setuptools 54+
-    depends_on("py-pip@:18", when="@:53", type="build")
-
     # Uses HTMLParser.unescape
     depends_on("python@:3.8", when="@:41.0", type=("build", "run"))
 
@@ -81,3 +208,25 @@ class PySetuptools(PythonPackage):
 
     # https://github.com/pypa/setuptools/issues/3661
     depends_on("python@:3.11", when="@:67", type=("build", "run"))
+
+    depends_on("py-pip", type="build")
+
+    def url_for_version(self, version):
+        url = "https://files.pythonhosted.org/packages/{0}/s/setuptools/setuptools-{1}-{0}-none-any.whl"
+
+        if version >= Version("45.1.0"):
+            python_tag = "py3"
+        else:
+            python_tag = "py2.py3"
+        return url.format(python_tag, version)
+
+    def install(self, spec, prefix):
+        # When setuptools changes its entry point we might get weird
+        # incompatibilities if building from sources in a non-isolated environment.
+        #
+        # https://github.com/pypa/setuptools/issues/980#issuecomment-1154471423
+        #
+        # We work around this issue by installing setuptools from wheels
+        whl = self.stage.archive_file
+        args = ["-m", "pip"] + std_pip_args + ["--prefix=" + prefix, whl]
+        python(*args)
diff --git a/var/spack/repos/builtin/packages/py-setuptools/rpath-compiler-flag.patch b/var/spack/repos/builtin/packages/py-setuptools/rpath-compiler-flag.patch
deleted file mode 100644
index 6b37d62323..0000000000
--- a/var/spack/repos/builtin/packages/py-setuptools/rpath-compiler-flag.patch
+++ /dev/null
@@ -1,13 +0,0 @@
-diff --git a/setuptools/_distutils/unixccompiler.py b/setuptools/_distutils/unixccompiler.py
---- a/setuptools/_distutils/unixccompiler.py
-+++ b/setuptools/_distutils/unixccompiler.py
-@@ -257,7 +257,7 @@ class UnixCCompiler(CCompiler):
-                 # No idea how --enable-new-dtags would be passed on to
-                 # ld if this system was using GNU ld.  Don't know if a
-                 # system like this even exists.
--                return "-R" + dir
-+                return "-Wl,-rpath," + dir
- 
-     def library_option(self, lib):
-         return "-l" + lib
-