diff options
| -rw-r--r-- | lib/spack/spack/main.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/package_base.py | 18 | ||||
| -rw-r--r-- | lib/spack/spack/solver/asp.py | 182 | ||||
| -rw-r--r-- | lib/spack/spack/test/concretize_compiler_runtimes.py | 42 | ||||
| -rw-r--r-- | lib/spack/spack/test/conftest.py | 6 | ||||
| -rw-r--r-- | var/spack/repos/builtin/packages/gcc-runtime/package.py | 72 | ||||
| -rw-r--r-- | var/spack/repos/builtin/packages/gcc/package.py | 29 | ||||
| -rw-r--r-- | var/spack/repos/compiler_runtime.test/packages/a/package.py | 13 | ||||
| -rw-r--r-- | var/spack/repos/compiler_runtime.test/packages/b/package.py | 12 | ||||
| -rw-r--r-- | var/spack/repos/compiler_runtime.test/packages/gcc-runtime/package.py | 13 | ||||
| -rw-r--r-- | var/spack/repos/compiler_runtime.test/packages/gcc/package.py | 32 | ||||
| -rw-r--r-- | var/spack/repos/compiler_runtime.test/repo.yaml | 2 |
12 files changed, 328 insertions, 95 deletions
diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index bcdc7d7599..56a4dc0e33 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -36,7 +36,6 @@ import spack.cmd import spack.config import spack.environment as ev import spack.modules -import spack.package_base import spack.paths import spack.platforms import spack.repo @@ -608,7 +607,6 @@ def setup_main_options(args): [(key, [spack.paths.mock_packages_path])] ) spack.repo.PATH = spack.repo.create(spack.config.CONFIG) - spack.package_base.WITH_GCC_RUNTIME = False # If the user asked for it, don't check ssl certs. if args.insecure: diff --git a/lib/spack/spack/package_base.py b/lib/spack/spack/package_base.py index bf8ed56d95..7d8f7104df 100644 --- a/lib/spack/spack/package_base.py +++ b/lib/spack/spack/package_base.py @@ -53,7 +53,6 @@ import spack.url import spack.util.environment import spack.util.path import spack.util.web -from spack.directives import _depends_on from spack.filesystem_view import YamlFilesystemView from spack.install_test import ( PackageTest, @@ -77,7 +76,6 @@ FLAG_HANDLER_TYPE = Callable[[str, Iterable[str]], FLAG_HANDLER_RETURN_TYPE] """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] -WITH_GCC_RUNTIME = True #: Filename for the Spack build/install log. _spack_build_logfile = "spack-build-out.txt" @@ -373,20 +371,6 @@ def on_package_attributes(**attr_dict): return _execute_under_condition -class BinaryPackage: - """This adds a universal dependency on gcc-runtime.""" - - def maybe_depend_on_gcc_runtime(self): - # Do not depend on itself, and allow tests to disable this universal dep - if self.name == "gcc-runtime" or not WITH_GCC_RUNTIME: - return - for v in ["13", "12", "11", "10", "9", "8", "7", "6", "5", "4"]: - _depends_on(self, f"gcc-runtime@{v}:", type="link", when=f"%gcc@{v} platform=linux") - _depends_on(self, f"gcc-runtime@{v}:", type="link", when=f"%gcc@{v} platform=cray") - - _directives_to_be_executed = [maybe_depend_on_gcc_runtime] - - class PackageViewMixin: """This collects all functionality related to adding installed Spack package to views. Packages can customize how they are added to views by @@ -449,7 +433,7 @@ class PackageViewMixin: Pb = TypeVar("Pb", bound="PackageBase") -class PackageBase(WindowsRPath, PackageViewMixin, BinaryPackage, metaclass=PackageMeta): +class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta): """This is the superclass for all spack packages. ***The Package class*** diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index b00982b4fb..ad62371c40 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -11,6 +11,7 @@ import os import pathlib import pprint import re +import sys import types import warnings from typing import Callable, Dict, List, NamedTuple, Optional, Sequence, Set, Tuple, Union @@ -61,6 +62,8 @@ GitOrStandardVersion = Union[spack.version.GitVersion, spack.version.StandardVer ASTType = None parse_files = None +#: Enable the addition of a runtime node +WITH_RUNTIME = sys.platform != "win32" #: Data class that contain configuration on what a #: clingo solve should output. @@ -122,6 +125,8 @@ class Provenance(enum.IntEnum): PACKAGE_PY = enum.auto() # An installed spec INSTALLED = enum.auto() + # A runtime injected from another package (e.g. a compiler) + RUNTIME = enum.auto() def __str__(self): return f"{self._name_.lower()}" @@ -2023,7 +2028,9 @@ class SpackSolverSetup: f.node_compiler_version(spec.name, spec.compiler.name, spec.compiler.version) ) - elif spec.compiler.versions: + elif spec.compiler.versions and spec.compiler.versions != vn.any_version: + # The condition above emits a facts only if we have an actual constraint + # on the compiler version, and avoids emitting them if any version is fine clauses.append( fn.attr( "node_compiler_version_satisfies", @@ -2578,6 +2585,9 @@ class SpackSolverSetup: self.possible_virtuals = node_counter.possible_virtuals() self.pkgs = node_counter.possible_dependencies() + runtimes = spack.repo.PATH.packages_with_tags("runtime") + self.pkgs.update(set(runtimes)) + # Fail if we already know an unreachable node is requested for spec in specs: missing_deps = [ @@ -2678,6 +2688,10 @@ class SpackSolverSetup: self.gen.h1("Variant Values defined in specs") self.define_variant_values() + if WITH_RUNTIME: + self.gen.h1("Runtimes") + self.define_runtime_constraints() + self.gen.h1("Version Constraints") self.collect_virtual_constraints() self.define_version_constraints() @@ -2688,6 +2702,21 @@ class SpackSolverSetup: self.gen.h1("Target Constraints") self.define_target_constraints() + def define_runtime_constraints(self): + """Define the constraints to be imposed on the runtimes""" + recorder = RuntimePropertyRecorder(self) + for compiler in self.possible_compilers: + if compiler.name != "gcc": + continue + try: + compiler_cls = spack.repo.PATH.get_pkg_class(compiler.name) + except spack.repo.UnknownPackageError: + continue + if hasattr(compiler_cls, "runtime_constraints"): + compiler_cls.runtime_constraints(compiler=compiler, pkg=recorder) + + recorder.consume_facts() + def literal_specs(self, specs): for spec in specs: self.gen.h2("Spec: %s" % str(spec)) @@ -2796,6 +2825,157 @@ class SpackSolverSetup: yield _spec_with_default_name(s, pkg_name) +class RuntimePropertyRecorder: + """An object of this class is injected in callbacks to compilers, to let them declare + properties of the runtimes they support and of the runtimes they provide, and to add + runtime dependencies to the nodes using said compiler. + + The usage of the object is the following. First, a runtime package name or the wildcard + "*" are passed as an argument to __call__, to set which kind of package we are referring to. + Then we can call one method with a directive-like API. + + Examples: + >>> pkg = RuntimePropertyRecorder(setup) + >>> # Every package compiled with %gcc has a link dependency on 'gcc-runtime' + >>> pkg("*").depends_on( + ... "gcc-runtime", + ... when="%gcc", + ... type="link", + ... description="If any package uses %gcc, it depends on gcc-runtime" + ... ) + >>> # The version of gcc-runtime is the same as the %gcc used to "compile" it + >>> pkg("gcc-runtime").requires("@=9.4.0", when="%gcc@=9.4.0") + """ + + def __init__(self, setup): + self._setup = setup + self.rules = [] + self.runtime_conditions = set() + # State of this object set in the __call__ method, and reset after + # each directive-like method + self.current_package = None + + def __call__(self, package_name: str) -> "RuntimePropertyRecorder": + """Sets a package name for the next directive-like method call""" + assert self.current_package is None, f"state was already set to '{self.current_package}'" + self.current_package = package_name + return self + + def reset(self): + """Resets the current state.""" + self.current_package = None + + def depends_on(self, dependency_str: str, *, when: str, type: str, description: str) -> None: + """Injects conditional dependencies on packages. + + Args: + dependency_str: the dependency spec to inject + when: anonymous condition to be met on a package to have the dependency + type: dependency type + description: human-readable description of the rule for adding the dependency + """ + # TODO: The API for this function is not final, and is still subject to change. At + # TODO: the moment, we implemented only the features strictly needed for the + # TODO: functionality currently provided by Spack, and we assert nothing else is required. + msg = "the 'depends_on' method can be called only with pkg('*')" + assert self.current_package == "*", msg + + when_spec = spack.spec.Spec(when) + assert when_spec.name is None, "only anonymous when specs are accepted" + + dependency_spec = spack.spec.Spec(dependency_str) + if dependency_spec.versions != vn.any_version: + self._setup.version_constraints.add((dependency_spec.name, dependency_spec.versions)) + + placeholder = "XXX" + node_variable = "node(ID, Package)" + when_spec.name = placeholder + + body_clauses = self._setup.spec_clauses(when_spec, body=True) + body_str = ( + f" {f',{os.linesep} '.join(str(x) for x in body_clauses)},\n" + f" not runtime(Package)" + ).replace(f'"{placeholder}"', f"{node_variable}") + head_clauses = self._setup.spec_clauses(dependency_spec, body=False) + + runtime_pkg = dependency_spec.name + main_rule = ( + f"% {description}\n" + f'1 {{ attr("depends_on", {node_variable}, node(0..X-1, "{runtime_pkg}"), "{type}") :' + f' max_dupes("gcc-runtime", X)}} 1:-\n' + f"{body_str}.\n\n" + ) + self.rules.append(main_rule) + for clause in head_clauses: + if clause.args[0] == "node": + continue + runtime_node = f'node(RuntimeID, "{runtime_pkg}")' + head_str = str(clause).replace(f'"{runtime_pkg}"', runtime_node) + rule = ( + f"{head_str} :-\n" + f' attr("depends_on", {node_variable}, {runtime_node}, "{type}"),\n' + f"{body_str}.\n\n" + ) + self.rules.append(rule) + + self.reset() + + def requires(self, impose: str, *, when: str): + """Injects conditional requirements on a given package. + + Args: + impose: constraint to be imposed + when: condition triggering the constraint + """ + msg = "the 'requires' method cannot be called with pkg('*') or without setting the package" + assert self.current_package is not None and self.current_package != "*", msg + + imposed_spec = spack.spec.Spec(f"{self.current_package}{impose}") + when_spec = spack.spec.Spec(f"{self.current_package}{when}") + + assert imposed_spec.versions.concrete, f"{impose} must have a concrete version" + assert when_spec.compiler.concrete, f"{when} must have a concrete compiler" + + # Add versions to possible versions + for s in (imposed_spec, when_spec): + if not s.versions.concrete: + continue + self._setup.possible_versions[s.name].add(s.version) + self._setup.declared_versions[s.name].append( + DeclaredVersion(version=s.version, idx=0, origin=Provenance.RUNTIME) + ) + + self.runtime_conditions.add((imposed_spec, when_spec)) + self.reset() + + def consume_facts(self): + """Consume the facts collected by this object, and emits rules and + facts for the runtimes. + """ + self._setup.gen.h2("Runtimes: rules") + self._setup.gen.newline() + for rule in self.rules: + if not isinstance(self._setup.gen.out, llnl.util.lang.Devnull): + self._setup.gen.out.write(rule) + self._setup.gen.control.add("base", [], rule) + + self._setup.gen.h2("Runtimes: conditions") + for runtime_pkg in spack.repo.PATH.packages_with_tags("runtime"): + self._setup.gen.fact(fn.runtime(runtime_pkg)) + self._setup.gen.fact(fn.possible_in_link_run(runtime_pkg)) + self._setup.gen.newline() + # Inject version rules for runtimes (versions are declared based + # on the available compilers) + self._setup.pkg_version_rules(runtime_pkg) + + for imposed_spec, when_spec in self.runtime_conditions: + msg = f"{when_spec} requires {imposed_spec} at runtime" + _ = self._setup.condition(when_spec, imposed_spec=imposed_spec, msg=msg) + + self._setup.trigger_rules() + self._setup.effect_rules() + + class SpecBuilder: """Class with actions to rebuild a spec from ASP results.""" diff --git a/lib/spack/spack/test/concretize_compiler_runtimes.py b/lib/spack/spack/test/concretize_compiler_runtimes.py new file mode 100644 index 0000000000..089ad28788 --- /dev/null +++ b/lib/spack/spack/test/concretize_compiler_runtimes.py @@ -0,0 +1,42 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os + +import pytest + +import spack.paths +import spack.repo +import spack.solver.asp +import spack.spec +from spack.version import Version + +pytestmark = [pytest.mark.only_clingo("Original concretizer does not support compiler runtimes")] + + +@pytest.fixture +def runtime_repo(config): + repo = os.path.join(spack.paths.repos_path, "compiler_runtime.test") + with spack.repo.use_repositories(repo) as mock_repo: + yield mock_repo + + +@pytest.fixture +def enable_runtimes(): + original = spack.solver.asp.WITH_RUNTIME + spack.solver.asp.WITH_RUNTIME = True + yield + spack.solver.asp.WITH_RUNTIME = original + + +def test_correct_gcc_runtime_is_injected_as_dependency(runtime_repo, enable_runtimes): + s = spack.spec.Spec("a%gcc@10.2.1 ^b%gcc@4.5.0").concretized() + a, b = s["a"], s["b"] + + # Both a and b should depend on the same gcc-runtime directly + assert a.dependencies("gcc-runtime") == b.dependencies("gcc-runtime") + + # And the gcc-runtime version should be that of the newest gcc used in the dag. + assert a["gcc-runtime"].version == Version("10.2.1") diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index df9a43a123..7b396a0358 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -44,6 +44,7 @@ import spack.package_prefs import spack.paths import spack.platforms import spack.repo +import spack.solver.asp import spack.stage import spack.store import spack.subprocess_context @@ -57,11 +58,6 @@ from spack.fetch_strategy import URLFetchStrategy from spack.util.pattern import Bunch -@pytest.fixture(scope="session", autouse=True) -def drop_gcc_runtime(): - spack.package_base.WITH_GCC_RUNTIME = False - - def ensure_configuration_fixture_run_before(request): """Ensure that fixture mutating the configuration run before the one where the function is called. diff --git a/var/spack/repos/builtin/packages/gcc-runtime/package.py b/var/spack/repos/builtin/packages/gcc-runtime/package.py index f0cd13dd14..ff13e05476 100644 --- a/var/spack/repos/builtin/packages/gcc-runtime/package.py +++ b/var/spack/repos/builtin/packages/gcc-runtime/package.py @@ -20,6 +20,8 @@ class GccRuntime(Package): homepage = "https://gcc.gnu.org" has_code = False + tags = ["runtime"] + maintainers("haampie") license("GPL-3.0-or-later WITH GCC-exception-3.1") @@ -42,76 +44,6 @@ class GccRuntime(Package): "ubsan", ] - for v in [ - "13.2", - "13.1", - "12.3", - "12.2", - "12.1", - "11.4", - "11.3", - "11.2", - "11.1", - "10.5", - "10.4", - "10.3", - "10.2", - "10.1", - "9.5", - "9.4", - "9.3", - "9.2", - "9.1", - "8.5", - "8.4", - "8.3", - "8.2", - "8.1", - "7.5", - "7.4", - "7.3", - "7.2", - "7.1", - "6.5", - "6.4", - "6.3", - "6.2", - "6.1", - "5.5", - "5.4", - "5.3", - "5.2", - "5.1", - "4.9.4", - "4.9.3", - "4.9.2", - "4.9.1", - "4.9.0", - "4.8.5", - "4.8.4", - "4.8.3", - "4.8.2", - "4.8.1", - "4.8.0", - "4.7.4", - "4.7.3", - "4.7.2", - "4.7.1", - "4.7.0", - "4.6.4", - "4.6.3", - "4.6.2", - "4.6.1", - "4.6.0", - "4.5.4", - "4.5.3", - "4.5.2", - "4.5.1", - "4.5.0", - ]: - version(v) - requires(f"%gcc@{v}", when=f"@{v}") - def install(self, spec, prefix): if spec.platform in ["linux", "cray", "freebsd"]: libraries = self._get_libraries_elf() diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index f6dce2ea00..2fb8d23aed 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -1111,3 +1111,32 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage): ), ), ) + + @classmethod + def runtime_constraints(cls, *, compiler, pkg): + """Callback function to inject runtime-related rules into the solver. + + Rule-injection is obtained through method calls of the ``pkg`` argument. + + Documentation for this function is temporary. When the API will be in its final state, + we'll document the behavior at https://spack.readthedocs.io/en/latest/ + + Args: + compiler: compiler object (node attribute) currently considered + pkg: object used to forward information to the solver + """ + pkg("*").depends_on( + "gcc-runtime", + when="%gcc", + type="link", + description="If any package uses %gcc, it depends on gcc-runtime", + ) + pkg("*").depends_on( + f"gcc-runtime@{str(compiler.version)}:", + when=f"%{str(compiler.spec)}", + type="link", + description=f"If any package uses %{str(compiler.spec)}, " + f"it depends on gcc-runtime@{str(compiler.version)}:", + ) + # The version of gcc-runtime is the same as the %gcc used to "compile" it + pkg("gcc-runtime").requires(f"@={str(compiler.version)}", when=f"%{str(compiler.spec)}") diff --git a/var/spack/repos/compiler_runtime.test/packages/a/package.py b/var/spack/repos/compiler_runtime.test/packages/a/package.py new file mode 100644 index 0000000000..1f68aadc2f --- /dev/null +++ b/var/spack/repos/compiler_runtime.test/packages/a/package.py @@ -0,0 +1,13 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class A(Package): + homepage = "http://www.example.com" + has_code = False + + version("1.0") + depends_on("b") diff --git a/var/spack/repos/compiler_runtime.test/packages/b/package.py b/var/spack/repos/compiler_runtime.test/packages/b/package.py new file mode 100644 index 0000000000..ab9f3150e6 --- /dev/null +++ b/var/spack/repos/compiler_runtime.test/packages/b/package.py @@ -0,0 +1,12 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class B(Package): + homepage = "http://www.example.com" + has_code = False + + version("1.0") diff --git a/var/spack/repos/compiler_runtime.test/packages/gcc-runtime/package.py b/var/spack/repos/compiler_runtime.test/packages/gcc-runtime/package.py new file mode 100644 index 0000000000..61d40a2df6 --- /dev/null +++ b/var/spack/repos/compiler_runtime.test/packages/gcc-runtime/package.py @@ -0,0 +1,13 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * + + +class GccRuntime(Package): + homepage = "https://example.com" + has_code = False + tags = ["runtime"] + requires("%gcc") diff --git a/var/spack/repos/compiler_runtime.test/packages/gcc/package.py b/var/spack/repos/compiler_runtime.test/packages/gcc/package.py new file mode 100644 index 0000000000..db81066599 --- /dev/null +++ b/var/spack/repos/compiler_runtime.test/packages/gcc/package.py @@ -0,0 +1,32 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * + + +class Gcc(Package): + homepage = "http://www.example.com/" + has_code = False + + version("13.2.0") + version("12.3.0") + + @classmethod + def runtime_constraints(cls, *, compiler, pkg): + pkg("*").depends_on( + "gcc-runtime", + when="%gcc", + type="link", + description="If any package uses %gcc, it depends on gcc-runtime", + ) + pkg("*").depends_on( + f"gcc-runtime@{str(compiler.version)}:", + when=f"%{str(compiler.spec)}", + type="link", + description=f"If any package uses %{str(compiler.spec)}, " + f"it depends on gcc-runtime@{str(compiler.version)}:", + ) + # The version of gcc-runtime is the same as the %gcc used to "compile" it + pkg("gcc-runtime").requires(f"@={str(compiler.version)}", when=f"%{str(compiler.spec)}") diff --git a/var/spack/repos/compiler_runtime.test/repo.yaml b/var/spack/repos/compiler_runtime.test/repo.yaml new file mode 100644 index 0000000000..1be826a641 --- /dev/null +++ b/var/spack/repos/compiler_runtime.test/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: compiler_runtime.test |