go: refactor bootstrapping process (#35823)

* Refactor go bootstrapping to include binary or gcc bootstrap

4 files changed, 81 insertions, 126 deletions

diff --git a/etc/spack/defaults/packages.yaml b/etc/spack/defaults/packages.yaml
index f0596b0fe1..792e8f14a6 100644
--- a/etc/spack/defaults/packages.yaml
+++ b/etc/spack/defaults/packages.yaml
@@ -28,7 +28,7 @@ packages:
       gl: [glx, osmesa]
       glu: [mesa-glu, openglu]
       golang: [go, gcc]
-      go-external-or-gccgo-bootstrap: [go-bootstrap, gcc]
+      go-or-gccgo-bootstrap: [go-bootstrap, gcc]
       iconv: [libiconv]
       ipp: [intel-ipp]
       java: [openjdk, jdk, ibm-java]
diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py
index 15826825ec..e70d0bea9e 100644
--- a/var/spack/repos/builtin/packages/gcc/package.py
+++ b/var/spack/repos/builtin/packages/gcc/package.py
@@ -197,36 +197,33 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage):
     depends_on("autogen@5.5.4:", type="test")
     depends_on("guile@1.4.1:", type="test")
 
-    # See https://golang.org/doc/install/gccgo#Releases
+    # See https://go.dev/doc/install/gccgo#Releases
     with when("languages=go"):
-        provides("golang", when="@4.6:")
-        provides("golang@:1", when="@4.7.1:")
-        provides("golang@:1.1", when="@4.8:")
-        provides("golang@:1.1.2", when="@4.8.2:")
+        provides("go-or-gccgo-bootstrap@:1.0", when="@4.7.1:")
+        provides("go-or-gccgo-bootstrap@:1.2", when="@4.9:")
+        provides("go-or-gccgo-bootstrap@:1.4", when="@5:")
+        provides("go-or-gccgo-bootstrap@:1.6.1", when="@6:")
+        provides("go-or-gccgo-bootstrap@:1.8.1", when="@7:")
+        provides("go-or-gccgo-bootstrap@:1.10.1", when="@8:")
+        provides("go-or-gccgo-bootstrap@:1.12.2", when="@9:")
+        provides("go-or-gccgo-bootstrap@:1.14.6", when="@10:")
+        provides("go-or-gccgo-bootstrap@1.16.3:1.16.5", when="@11:")
+
+        provides("golang@:1.0", when="@4.7.1:")
         provides("golang@:1.2", when="@4.9:")
         provides("golang@:1.4", when="@5:")
         provides("golang@:1.6.1", when="@6:")
-        provides("golang@:1.8", when="@7:")
-        provides("golang@:1.10", when="@8:")
-        provides("golang@:1.12", when="@9:")
-        provides("golang@:1.14", when="@10:")
-        provides("golang@:1.16", when="@11:")
-        provides("golang@:1.18", when="@11:")
-        # GCC 4.6 added support for the Go programming language.
-        # See https://gcc.gnu.org/gcc-4.6/changes.html
-        conflicts("@:4.5", msg="support for Go has been added in GCC 4.6")
-        # aarch64 machines (including Macs with Apple silicon) can't use
-        # go-bootstrap because it pre-dates aarch64 support in Go. When not
-        # using an external go bootstrap go, These machines have to rely on
-        # Go support in gcc (which may require compiling a version of gcc
-        # with Go support just to satisfy this requirement).  However,
-        # there's also a bug in some versions of GCC's Go front-end that prevents
-        # these versions from properly bootstrapping Go.  (See issue #47771
-        # https://github.com/golang/go/issues/47771 )  On the 10.x branch, we need
-        # at least 10.4.  On the 11.x branch, we need at least 11.3:
-        provides("go-external-or-gccgo-bootstrap", when="gcc@10.4.0:10,11.3.0:target=aarch64:")
+        provides("golang@:1.8.1", when="@7:")
+        provides("golang@:1.10.1", when="@8:")
+        provides("golang@:1.12.2", when="@9:")
+        provides("golang@:1.14.6", when="@10:")
+        provides("golang@1.16.3:1.16.5", when="@11:")
+
+        # GCC 4.7.1 added full support for the Go 1.x programming language.
+        conflicts("@:4.7.0")
+
         # Go is not supported on macOS
-        conflicts("platform=darwin", msg="Go not supported on MacOS")
+        conflicts("platform=darwin", msg="GCC cannot build Go support on MacOS")
 
     # For a list of valid languages for a specific release,
     # run the following command in the GCC source directory:
diff --git a/var/spack/repos/builtin/packages/go-bootstrap/package.py b/var/spack/repos/builtin/packages/go-bootstrap/package.py
index a787e7de36..047f9f3353 100644
--- a/var/spack/repos/builtin/packages/go-bootstrap/package.py
+++ b/var/spack/repos/builtin/packages/go-bootstrap/package.py
@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import platform
 import re
 
 from spack.package import *
@@ -18,50 +19,54 @@ class GoBootstrap(Package):
     """Old C-bootstrapped go to bootstrap real go"""
 
     homepage = "https://golang.org"
+    url = "https://go.dev/dl/go1.20.1.darwin-amd64.tar.gz"
 
     extendable = True
 
-    # NOTE: Go@1.4.x is the only supported bootstrapping compiler because all
-    # later versions require a Go compiler to build.
-    # See: https://golang.org/doc/install/source#go14 and
-    # https://github.com/golang/go/issues/17545 and
-    # https://github.com/golang/go/issues/16352
-    version(
-        "1.4-bootstrap-20171003",
-        sha256="f4ff5b5eb3a3cae1c993723f3eab519c5bae18866b5e5f96fe1102f0cb5c3e52",
-        url="https://dl.google.com/go/go1.4-bootstrap-20171003.tar.gz",
-    )
-    version(
-        "1.4-bootstrap-20170531",
-        sha256="49f806f66762077861b7de7081f586995940772d29d4c45068c134441a743fa2",
-        url="https://storage.googleapis.com/golang/go1.4-bootstrap-20170531.tar.gz",
-    )
-    version(
-        "1.4-bootstrap-20161024",
-        sha256="398c70d9d10541ba9352974cc585c43220b6d8dbcd804ba2c9bd2fbf35fab286",
-        url="https://storage.googleapis.com/golang/go1.4-bootstrap-20161024.tar.gz",
-    )
-
-    provides("golang@:1.4-bootstrap-20171003")
+    maintainers("alecbcs")
 
     depends_on("git", type=("build", "link", "run"))
 
-    conflicts(
-        "os=monterey",
-        msg="go-bootstrap won't build on MacOS Monterey: "
-        "try `brew install go` and `spack external find go`",
-    )
-    conflicts("target=aarch64:", msg="Go bootstrap doesn't support aarch64 architectures")
-
-    # This virtual package allows a fallback to gccgo for aarch64,
-    # where go-bootstrap cannot be built(aarch64 was added with Go 1.5)
-    provides("go-external-or-gccgo-bootstrap")
-
-    # Support for aarch64 was added in Go 1.5, use an external package or gccgo instead:
-    conflicts("@:1.4", when="target=aarch64:")
-
     executables = ["^go$"]
 
+    # List binary go releases for multiple operating systems and architectures.
+    # These binary versions are not intended to stay up-to-date. Instead we
+    # should update these binary releases on a yearly schedule as
+    # bootstrapping requirements are modified by new releases of go.
+    go_releases = {
+        "1.17.13": {
+            "darwin": {
+                "amd64": "c101beaa232e0f448fab692dc036cd6b4677091ff89c4889cc8754b1b29c6608",
+                "arm64": "e4ccc9c082d91eaa0b866078b591fc97d24b91495f12deb3dd2d8eda4e55a6ea",
+            },
+            "linux": {
+                "amd64": "4cdd2bc664724dc7db94ad51b503512c5ae7220951cac568120f64f8e94399fc",
+                "arm64": "914daad3f011cc2014dea799bb7490442677e4ad6de0b2ac3ded6cee7e3f493d",
+                "ppc64le": "bd0763fb130f8412672ffe1e4a8e65888ebe2419e5caa9a67ac21e8c298aa254",
+            },
+        }
+    }
+
+    # Normalize architectures returned by platform to those used by the
+    # Go project.
+    go_targets = {
+        "aarch64": "arm64",
+        "arm64": "arm64",
+        "ppc64le": "ppc64le",
+        "amd64": "amd64",
+        "x86_64": "amd64",
+    }
+
+    # determine system os and architecture/target
+    os = platform.system().lower()
+    target = go_targets[platform.machine().lower()]
+
+    # construct releases for current system configuration
+    for release in go_releases:
+        if os in go_releases[release] and target in go_releases[release][os]:
+            version(release, sha256=go_releases[release][os][target])
+            provides(f"go-or-gccgo-bootstrap@{release}", when=f"@{release}")
+
     # When the user adds a go compiler using ``spack external find go-bootstrap``,
     # this lets us get the version for packages.yaml. Then, the solver can avoid
     # to build the bootstrap go compiler(for aarch64, it's only gccgo) from source:
@@ -72,25 +77,22 @@ class GoBootstrap(Package):
         match = re.search(r"go version go(\S+)", output)
         return match.group(1) if match else None
 
-    def patch(self):
-        if self.spec.satisfies("@:1.4.3"):
-            # NOTE: Older versions of Go attempt to download external files that have
-            # since been moved while running the test suite.  This patch modifies the
-            # test files so that these tests don't cause false failures.
-            # See: https://github.com/golang/go/issues/15694
-            test_suite_file = FileFilter(join_path("src", "run.bash"))
-            test_suite_file.filter(r"^(.*)(\$GOROOT/src/cmd/api/run.go)(.*)$", r"# \1\2\3")
+    def url_for_version(self, version):
+        # allow maintainers to checksum multiple architectures via
+        # `spack checksum go-bootstrap@1.18.9-darwin-arm64`
+        match = re.search(r"(\S+)-(\S+)-(\S+)", str(version))
+        if match:
+            version = match.group(1)
+            os = match.group(2)
+            target = match.group(3)
+        else:
+            os = self.os
+            target = self.target
 
-        # Go uses a hardcoded limit of 4096 bytes for its printf functions.
-        # This can cause environment variables to be truncated.
-        filter_file("char buf[4096];", "char buf[131072];", "src/cmd/dist/unix.c", string=True)
+        url = "https://go.dev/dl/go{0}.{1}-{2}.tar.gz"
+        return url.format(version, os, target)
 
     def install(self, spec, prefix):
-        env["CGO_ENABLED"] = "0"
-        bash = which("bash")
-        with working_dir("src"):
-            bash("{0}.bash".format("all" if self.run_tests else "make"))
-
         install_tree(".", prefix)
 
     def setup_dependent_build_environment(self, env, dependent_spec):
@@ -101,6 +103,3 @@ class GoBootstrap(Package):
         else:
             goroot = self.spec.prefix
         env.set("GOROOT_BOOTSTRAP", goroot)
-
-    def setup_build_environment(self, env):
-        env.set("GOROOT_FINAL", self.spec.prefix)
diff --git a/var/spack/repos/builtin/packages/go/package.py b/var/spack/repos/builtin/packages/go/package.py
index 9d798a7d8c..1844acc7ad 100644
--- a/var/spack/repos/builtin/packages/go/package.py
+++ b/var/spack/repos/builtin/packages/go/package.py
@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import os
-import platform
 import re
 
 import llnl.util.tty as tty
@@ -26,10 +25,6 @@ from spack.package import *
 # - on CentOS 7 systems (and possibly others) you need to have the
 #   glibc package installed or various static cgo tests fail.
 #
-# - When building on a *large* machine (144 cores, 1.5TB RAM) I need
-#   to run `ulimit -u 8192` to bump up the max number of user processes.
-#   Failure to do so results in an explosion in one of the tests and an
-#   epic stack trace....
 
 
 class Go(Package):
@@ -57,29 +52,8 @@ class Go(Package):
     provides("golang")
 
     depends_on("git", type=("build", "link", "run"))
-
-    # aarch64 machines (including Macs with Apple silicon) can't use
-    # go-bootstrap because it pre-dates aarch64 support in Go.  These machines
-    # have to rely on Go support in gcc (which may require compiling a version
-    # of gcc with Go support just to satisfy this requirement) or external go:
-
-    # #27769: On M1/MacOS, platform.machine() may return arm64:
-    if platform.machine() in ["arm64", "aarch64"]:
-        # Use an external go compiler from packages.yaml/`spack external find go-bootstrap`,
-        # but fallback to build go-bootstrap@1.4 or to gcc with languages=go (for aarch64):
-        depends_on("go-external-or-gccgo-bootstrap", type="build")
-    else:
-        depends_on("go-bootstrap", type="build")
-
-    # https://github.com/golang/go/issues/17545
-    patch("time_test.patch", when="@1.6.4:1.7.4")
-
-    # https://github.com/golang/go/issues/17986
-    # The fix for this issue has been merged into the 1.8 tree.
-    patch("misc-cgo-testcshared.patch", level=0, when="@1.6.4:1.7.5")
-
-    # Unrecognized option '-fno-lto'
-    conflicts("%gcc@:4", when="@1.17:")
+    depends_on("go-or-gccgo-bootstrap", type="build")
+    depends_on("go-or-gccgo-bootstrap@1.17.13:", type="build", when="@1.20:")
 
     @classmethod
     def determine_version(cls, exe):
@@ -87,28 +61,13 @@ class Go(Package):
         match = re.search(r"go version go(\S+)", output)
         return match.group(1) if match else None
 
-    # NOTE: Older versions of Go attempt to download external files that have
-    # since been moved while running the test suite.  This patch modifies the
-    # test files so that these tests don't cause false failures.
-    # See: https://github.com/golang/go/issues/15694
-    @when("@:1.4.3")
-    def patch(self):
-        test_suite_file = FileFilter(join_path("src", "run.bash"))
-        test_suite_file.filter(r"^(.*)(\$GOROOT/src/cmd/api/run.go)(.*)$", r"# \1\2\3")
-
     def install(self, spec, prefix):
         bash = which("bash")
 
-        wd = "."
-
-        # 1.11.5 directory structure is slightly different
-        if self.version == Version("1.11.5"):
-            wd = "go"
-
-        with working_dir(join_path(wd, "src")):
+        with working_dir("src"):
             bash("{0}.bash".format("all" if self.run_tests else "make"))
 
-        install_tree(wd, prefix)
+        install_tree(".", prefix)
 
     def setup_build_environment(self, env):
         env.set("GOROOT_FINAL", self.spec.prefix)