Bootstrap most of Spack dependencies using environments (#34029)

This commit reworks the bootstrapping procedure to use Spack environments 
as much as possible.

The `spack.bootstrap` module has also been reorganized into a Python package. 
A distinction is made among "core" Spack dependencies (clingo, GnuPG, patchelf)
and other dependencies. For a number of reasons, explained in the `spack.bootstrap.core`
module docstring, "core" dependencies are bootstrapped with the current ad-hoc
method. 

All the other dependencies are instead bootstrapped using a Spack environment
that lives in a directory specific to the interpreter and the architecture being used. 

5 files changed, 69 insertions, 3 deletions

diff --git a/var/spack/repos/builtin/packages/py-black/package.py b/var/spack/repos/builtin/packages/py-black/package.py
index d2d3cd286f..21c114aa0b 100644
--- a/var/spack/repos/builtin/packages/py-black/package.py
+++ b/var/spack/repos/builtin/packages/py-black/package.py
@@ -52,6 +52,9 @@ class PyBlack(PythonPackage):
     depends_on("py-ipython@7.8:", when="+jupyter", type=("build", "run"))
     depends_on("py-tokenize-rt@3.2:", when="+jupyter", type=("build", "run"))
 
+    # Needed because this package is used to bootstrap Spack (Spack supports Python 3.6+)
+    depends_on("py-dataclasses@0.6:", when="^python@:3.6", type=("build", "run"))
+
     # see: https://github.com/psf/black/issues/2964
     # note that pip doesn't know this constraint.
     depends_on("py-click@:8.0", when="@:22.2", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-dataclasses/package.py b/var/spack/repos/builtin/packages/py-dataclasses/package.py
new file mode 100644
index 0000000000..a3de7b2313
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-dataclasses/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyDataclasses(PythonPackage):
+    """A backport of the dataclasses module for Python 3.6"""
+
+    homepage = "https://github.com/ericvsmith/dataclasses"
+    pypi = "dataclasses/dataclasses-0.7.tar.gz"
+
+    version("0.8", sha256="8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97")
+    version("0.7", sha256="494a6dcae3b8bcf80848eea2ef64c0cc5cd307ffc263e17cdf42f3e5420808e6")
+
+    depends_on("python@3.6.00:3.6", type=("build", "run"))
+    depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-platformdirs/package.py b/var/spack/repos/builtin/packages/py-platformdirs/package.py
index f226506db5..ee5a832315 100644
--- a/var/spack/repos/builtin/packages/py-platformdirs/package.py
+++ b/var/spack/repos/builtin/packages/py-platformdirs/package.py
@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import spack.build_systems.python
 from spack.package import *
 
 
@@ -18,9 +19,24 @@ class PyPlatformdirs(PythonPackage):
     version("2.4.0", sha256="367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2")
     version("2.3.0", sha256="15b056538719b1c94bdaccb29e5f81879c7f7f0f4a153f46086d155dffcd4f0f")
 
+    variant(
+        "wheel",
+        default=False,
+        sticky=True,
+        description="Install from wheel (required for bootstrapping Spack)",
+    )
+
     depends_on("python@3.7:", when="@2.4.1:", type=("build", "run"))
     depends_on("python@3.6:", type=("build", "run"))
     depends_on("py-setuptools@44:", when="@:2.5.1", type="build")
     depends_on("py-setuptools-scm@5:+toml", when="@:2.5.1", type="build")
     depends_on("py-hatchling@0.22.0:", when="@2.5.2:", type="build")
     depends_on("py-hatch-vcs", when="@2.5.2:", type="build")
+
+
+class PythonPipBuilder(spack.build_systems.python.PythonPipBuilder):
+    @when("+wheel")
+    def install(self, pkg, spec, prefix):
+        args = list(filter(lambda x: x != "--no-index", self.std_args(self.pkg)))
+        args += [f"--prefix={prefix}", self.spec.format("platformdirs=={version}")]
+        pip(*args)
diff --git a/var/spack/repos/builtin/packages/py-typed-ast/package.py b/var/spack/repos/builtin/packages/py-typed-ast/package.py
index 4dc338e06a..16f6fc03ab 100644
--- a/var/spack/repos/builtin/packages/py-typed-ast/package.py
+++ b/var/spack/repos/builtin/packages/py-typed-ast/package.py
@@ -2,7 +2,7 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
+import spack.build_systems.python
 from spack.package import *
 
 
@@ -23,7 +23,22 @@ class PyTypedAst(PythonPackage):
         url="https://files.pythonhosted.org/packages/source/t/typed-ast/typed-ast-1.3.5.tar.gz",
     )
 
+    variant(
+        "wheel",
+        default=False,
+        sticky=True,
+        description="Install from wheel (required for bootstrapping Spack)",
+    )
+
     depends_on("python@3.3:", type=("build", "link", "run"))
     depends_on("python@3.6:", when="@1.5.4:", type=("build", "link", "run"))
     depends_on("python@:3.8", when="@:1.4.0")  # build errors with 3.9 until 1.4.1
     depends_on("py-setuptools", type="build")
+
+
+class PythonPipBuilder(spack.build_systems.python.PythonPipBuilder):
+    @when("+wheel")
+    def install(self, pkg, spec, prefix):
+        args = list(filter(lambda x: x != "--no-index", self.std_args(self.pkg)))
+        args += [f"--prefix={prefix}", self.spec.format("typed-ast=={version}")]
+        pip(*args)
diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
index 513364e05e..3d3e955024 100644
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -42,7 +42,7 @@ class Python(Package):
 
     #: phase
     install_targets = ["install"]
-    build_targets = []  # type: List[str]
+    build_targets: List[str] = []
 
     version("3.11.0", sha256="64424e96e2457abbac899b90f9530985b51eef2905951febd935f0e73414caeb")
     version(
@@ -107,6 +107,13 @@ class Python(Package):
     version("3.7.1", sha256="36c1b81ac29d0f8341f727ef40864d99d8206897be96be73dc34d4739c9c9f06")
     version("3.7.0", sha256="85bb9feb6863e04fb1700b018d9d42d1caac178559ffa453d7e6a436e259fd0d")
 
+    # Python 3.6.15 has been added back only to allow bootstrapping Spack on Python 3.6
+    version(
+        "3.6.15",
+        sha256="54570b7e339e2cfd72b29c7e2fdb47c0b7b18b7412e61de5b463fc087c13b043",
+        deprecated=True,
+    )
+
     extendable = True
 
     # Variants to avoid cyclical dependencies for concretizer
@@ -226,7 +233,7 @@ class Python(Package):
     conflicts("%nvhpc")
 
     # Used to cache various attributes that are expensive to compute
-    _config_vars = {}  # type: Dict[str, Dict[str, str]]
+    _config_vars: Dict[str, Dict[str, str]] = {}
 
     # An in-source build with --enable-optimizations fails for python@3.X
     build_directory = "spack-build"
@@ -727,6 +734,12 @@ class Python(Package):
                 return Executable(path)
 
         else:
+            # Give a last try at rhel8 platform python
+            if self.spec.external and self.prefix == "/usr" and self.spec.satisfies("os=rhel8"):
+                path = os.path.join(self.prefix, "libexec", "platform-python")
+                if os.path.exists(path):
+                    return Executable(path)
+
             msg = "Unable to locate {0} command in {1}"
             raise RuntimeError(msg.format(self.name, self.prefix.bin))