From f5228cf59c301c61c83d8139d9599eded5939d23 Mon Sep 17 00:00:00 2001 From: Alec Scott Date: Fri, 10 Mar 2023 16:27:49 -0800 Subject: go: refactor bootstrapping process (#35823) * Refactor go bootstrapping to include binary or gcc bootstrap --- etc/spack/defaults/packages.yaml | 2 +- var/spack/repos/builtin/packages/gcc/package.py | 47 +++++---- .../repos/builtin/packages/go-bootstrap/package.py | 109 ++++++++++----------- var/spack/repos/builtin/packages/go/package.py | 49 +-------- 4 files changed, 81 insertions(+), 126 deletions(-) diff --git a/etc/spack/defaults/packages.yaml b/etc/spack/defaults/packages.yaml index f0596b0fe1..792e8f14a6 100644 --- a/etc/spack/defaults/packages.yaml +++ b/etc/spack/defaults/packages.yaml @@ -28,7 +28,7 @@ packages: gl: [glx, osmesa] glu: [mesa-glu, openglu] golang: [go, gcc] - go-external-or-gccgo-bootstrap: [go-bootstrap, gcc] + go-or-gccgo-bootstrap: [go-bootstrap, gcc] iconv: [libiconv] ipp: [intel-ipp] java: [openjdk, jdk, ibm-java] diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 15826825ec..e70d0bea9e 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -197,36 +197,33 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage): depends_on("autogen@5.5.4:", type="test") depends_on("guile@1.4.1:", type="test") - # See https://golang.org/doc/install/gccgo#Releases + # See https://go.dev/doc/install/gccgo#Releases with when("languages=go"): - provides("golang", when="@4.6:") - provides("golang@:1", when="@4.7.1:") - provides("golang@:1.1", when="@4.8:") - provides("golang@:1.1.2", when="@4.8.2:") + provides("go-or-gccgo-bootstrap@:1.0", when="@4.7.1:") + provides("go-or-gccgo-bootstrap@:1.2", when="@4.9:") + provides("go-or-gccgo-bootstrap@:1.4", when="@5:") + provides("go-or-gccgo-bootstrap@:1.6.1", when="@6:") + provides("go-or-gccgo-bootstrap@:1.8.1", when="@7:") + provides("go-or-gccgo-bootstrap@:1.10.1", when="@8:") + provides("go-or-gccgo-bootstrap@:1.12.2", when="@9:") + provides("go-or-gccgo-bootstrap@:1.14.6", when="@10:") + provides("go-or-gccgo-bootstrap@1.16.3:1.16.5", when="@11:") + + provides("golang@:1.0", when="@4.7.1:") provides("golang@:1.2", when="@4.9:") provides("golang@:1.4", when="@5:") provides("golang@:1.6.1", when="@6:") - provides("golang@:1.8", when="@7:") - provides("golang@:1.10", when="@8:") - provides("golang@:1.12", when="@9:") - provides("golang@:1.14", when="@10:") - provides("golang@:1.16", when="@11:") - provides("golang@:1.18", when="@11:") - # GCC 4.6 added support for the Go programming language. - # See https://gcc.gnu.org/gcc-4.6/changes.html - conflicts("@:4.5", msg="support for Go has been added in GCC 4.6") - # aarch64 machines (including Macs with Apple silicon) can't use - # go-bootstrap because it pre-dates aarch64 support in Go. When not - # using an external go bootstrap go, These machines have to rely on - # Go support in gcc (which may require compiling a version of gcc - # with Go support just to satisfy this requirement). However, - # there's also a bug in some versions of GCC's Go front-end that prevents - # these versions from properly bootstrapping Go. (See issue #47771 - # https://github.com/golang/go/issues/47771 ) On the 10.x branch, we need - # at least 10.4. On the 11.x branch, we need at least 11.3: - provides("go-external-or-gccgo-bootstrap", when="gcc@10.4.0:10,11.3.0:target=aarch64:") + provides("golang@:1.8.1", when="@7:") + provides("golang@:1.10.1", when="@8:") + provides("golang@:1.12.2", when="@9:") + provides("golang@:1.14.6", when="@10:") + provides("golang@1.16.3:1.16.5", when="@11:") + + # GCC 4.7.1 added full support for the Go 1.x programming language. + conflicts("@:4.7.0") + # Go is not supported on macOS - conflicts("platform=darwin", msg="Go not supported on MacOS") + conflicts("platform=darwin", msg="GCC cannot build Go support on MacOS") # For a list of valid languages for a specific release, # run the following command in the GCC source directory: diff --git a/var/spack/repos/builtin/packages/go-bootstrap/package.py b/var/spack/repos/builtin/packages/go-bootstrap/package.py index a787e7de36..047f9f3353 100644 --- a/var/spack/repos/builtin/packages/go-bootstrap/package.py +++ b/var/spack/repos/builtin/packages/go-bootstrap/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import platform import re from spack.package import * @@ -18,50 +19,54 @@ class GoBootstrap(Package): """Old C-bootstrapped go to bootstrap real go""" homepage = "https://golang.org" + url = "https://go.dev/dl/go1.20.1.darwin-amd64.tar.gz" extendable = True - # NOTE: Go@1.4.x is the only supported bootstrapping compiler because all - # later versions require a Go compiler to build. - # See: https://golang.org/doc/install/source#go14 and - # https://github.com/golang/go/issues/17545 and - # https://github.com/golang/go/issues/16352 - version( - "1.4-bootstrap-20171003", - sha256="f4ff5b5eb3a3cae1c993723f3eab519c5bae18866b5e5f96fe1102f0cb5c3e52", - url="https://dl.google.com/go/go1.4-bootstrap-20171003.tar.gz", - ) - version( - "1.4-bootstrap-20170531", - sha256="49f806f66762077861b7de7081f586995940772d29d4c45068c134441a743fa2", - url="https://storage.googleapis.com/golang/go1.4-bootstrap-20170531.tar.gz", - ) - version( - "1.4-bootstrap-20161024", - sha256="398c70d9d10541ba9352974cc585c43220b6d8dbcd804ba2c9bd2fbf35fab286", - url="https://storage.googleapis.com/golang/go1.4-bootstrap-20161024.tar.gz", - ) - - provides("golang@:1.4-bootstrap-20171003") + maintainers("alecbcs") depends_on("git", type=("build", "link", "run")) - conflicts( - "os=monterey", - msg="go-bootstrap won't build on MacOS Monterey: " - "try `brew install go` and `spack external find go`", - ) - conflicts("target=aarch64:", msg="Go bootstrap doesn't support aarch64 architectures") - - # This virtual package allows a fallback to gccgo for aarch64, - # where go-bootstrap cannot be built(aarch64 was added with Go 1.5) - provides("go-external-or-gccgo-bootstrap") - - # Support for aarch64 was added in Go 1.5, use an external package or gccgo instead: - conflicts("@:1.4", when="target=aarch64:") - executables = ["^go$"] + # List binary go releases for multiple operating systems and architectures. + # These binary versions are not intended to stay up-to-date. Instead we + # should update these binary releases on a yearly schedule as + # bootstrapping requirements are modified by new releases of go. + go_releases = { + "1.17.13": { + "darwin": { + "amd64": "c101beaa232e0f448fab692dc036cd6b4677091ff89c4889cc8754b1b29c6608", + "arm64": "e4ccc9c082d91eaa0b866078b591fc97d24b91495f12deb3dd2d8eda4e55a6ea", + }, + "linux": { + "amd64": "4cdd2bc664724dc7db94ad51b503512c5ae7220951cac568120f64f8e94399fc", + "arm64": "914daad3f011cc2014dea799bb7490442677e4ad6de0b2ac3ded6cee7e3f493d", + "ppc64le": "bd0763fb130f8412672ffe1e4a8e65888ebe2419e5caa9a67ac21e8c298aa254", + }, + } + } + + # Normalize architectures returned by platform to those used by the + # Go project. + go_targets = { + "aarch64": "arm64", + "arm64": "arm64", + "ppc64le": "ppc64le", + "amd64": "amd64", + "x86_64": "amd64", + } + + # determine system os and architecture/target + os = platform.system().lower() + target = go_targets[platform.machine().lower()] + + # construct releases for current system configuration + for release in go_releases: + if os in go_releases[release] and target in go_releases[release][os]: + version(release, sha256=go_releases[release][os][target]) + provides(f"go-or-gccgo-bootstrap@{release}", when=f"@{release}") + # When the user adds a go compiler using ``spack external find go-bootstrap``, # this lets us get the version for packages.yaml. Then, the solver can avoid # to build the bootstrap go compiler(for aarch64, it's only gccgo) from source: @@ -72,25 +77,22 @@ class GoBootstrap(Package): match = re.search(r"go version go(\S+)", output) return match.group(1) if match else None - def patch(self): - if self.spec.satisfies("@:1.4.3"): - # NOTE: Older versions of Go attempt to download external files that have - # since been moved while running the test suite. This patch modifies the - # test files so that these tests don't cause false failures. - # See: https://github.com/golang/go/issues/15694 - test_suite_file = FileFilter(join_path("src", "run.bash")) - test_suite_file.filter(r"^(.*)(\$GOROOT/src/cmd/api/run.go)(.*)$", r"# \1\2\3") + def url_for_version(self, version): + # allow maintainers to checksum multiple architectures via + # `spack checksum go-bootstrap@1.18.9-darwin-arm64` + match = re.search(r"(\S+)-(\S+)-(\S+)", str(version)) + if match: + version = match.group(1) + os = match.group(2) + target = match.group(3) + else: + os = self.os + target = self.target - # Go uses a hardcoded limit of 4096 bytes for its printf functions. - # This can cause environment variables to be truncated. - filter_file("char buf[4096];", "char buf[131072];", "src/cmd/dist/unix.c", string=True) + url = "https://go.dev/dl/go{0}.{1}-{2}.tar.gz" + return url.format(version, os, target) def install(self, spec, prefix): - env["CGO_ENABLED"] = "0" - bash = which("bash") - with working_dir("src"): - bash("{0}.bash".format("all" if self.run_tests else "make")) - install_tree(".", prefix) def setup_dependent_build_environment(self, env, dependent_spec): @@ -101,6 +103,3 @@ class GoBootstrap(Package): else: goroot = self.spec.prefix env.set("GOROOT_BOOTSTRAP", goroot) - - def setup_build_environment(self, env): - env.set("GOROOT_FINAL", self.spec.prefix) diff --git a/var/spack/repos/builtin/packages/go/package.py b/var/spack/repos/builtin/packages/go/package.py index 9d798a7d8c..1844acc7ad 100644 --- a/var/spack/repos/builtin/packages/go/package.py +++ b/var/spack/repos/builtin/packages/go/package.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os -import platform import re import llnl.util.tty as tty @@ -26,10 +25,6 @@ from spack.package import * # - on CentOS 7 systems (and possibly others) you need to have the # glibc package installed or various static cgo tests fail. # -# - When building on a *large* machine (144 cores, 1.5TB RAM) I need -# to run `ulimit -u 8192` to bump up the max number of user processes. -# Failure to do so results in an explosion in one of the tests and an -# epic stack trace.... class Go(Package): @@ -57,29 +52,8 @@ class Go(Package): provides("golang") depends_on("git", type=("build", "link", "run")) - - # aarch64 machines (including Macs with Apple silicon) can't use - # go-bootstrap because it pre-dates aarch64 support in Go. These machines - # have to rely on Go support in gcc (which may require compiling a version - # of gcc with Go support just to satisfy this requirement) or external go: - - # #27769: On M1/MacOS, platform.machine() may return arm64: - if platform.machine() in ["arm64", "aarch64"]: - # Use an external go compiler from packages.yaml/`spack external find go-bootstrap`, - # but fallback to build go-bootstrap@1.4 or to gcc with languages=go (for aarch64): - depends_on("go-external-or-gccgo-bootstrap", type="build") - else: - depends_on("go-bootstrap", type="build") - - # https://github.com/golang/go/issues/17545 - patch("time_test.patch", when="@1.6.4:1.7.4") - - # https://github.com/golang/go/issues/17986 - # The fix for this issue has been merged into the 1.8 tree. - patch("misc-cgo-testcshared.patch", level=0, when="@1.6.4:1.7.5") - - # Unrecognized option '-fno-lto' - conflicts("%gcc@:4", when="@1.17:") + depends_on("go-or-gccgo-bootstrap", type="build") + depends_on("go-or-gccgo-bootstrap@1.17.13:", type="build", when="@1.20:") @classmethod def determine_version(cls, exe): @@ -87,28 +61,13 @@ class Go(Package): match = re.search(r"go version go(\S+)", output) return match.group(1) if match else None - # NOTE: Older versions of Go attempt to download external files that have - # since been moved while running the test suite. This patch modifies the - # test files so that these tests don't cause false failures. - # See: https://github.com/golang/go/issues/15694 - @when("@:1.4.3") - def patch(self): - test_suite_file = FileFilter(join_path("src", "run.bash")) - test_suite_file.filter(r"^(.*)(\$GOROOT/src/cmd/api/run.go)(.*)$", r"# \1\2\3") - def install(self, spec, prefix): bash = which("bash") - wd = "." - - # 1.11.5 directory structure is slightly different - if self.version == Version("1.11.5"): - wd = "go" - - with working_dir(join_path(wd, "src")): + with working_dir("src"): bash("{0}.bash".format("all" if self.run_tests else "make")) - install_tree(wd, prefix) + install_tree(".", prefix) def setup_build_environment(self, env): env.set("GOROOT_FINAL", self.spec.prefix) -- cgit v1.2.3-60-g2f50