summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorMassimiliano Culpo <massimiliano.culpo@gmail.com>2023-11-30 18:36:24 +0100
committerTodd Gamblin <tgamblin@llnl.gov>2023-12-21 12:22:58 -0800
commitea7e3e4f9fdf98abd22f530dc61048ad0ce23d04 (patch)
treea8205a86f9b4bc15487eec436a47d96a1a73d279 /lib
parent8371bb4e192f876978fd77ea81c728bcd3475768 (diff)
downloadspack-ea7e3e4f9fdf98abd22f530dc61048ad0ce23d04.tar.gz
spack-ea7e3e4f9fdf98abd22f530dc61048ad0ce23d04.tar.bz2
spack-ea7e3e4f9fdf98abd22f530dc61048ad0ce23d04.tar.xz
spack-ea7e3e4f9fdf98abd22f530dc61048ad0ce23d04.zip
Compilers can inject first order rules into the solver
* Restore PackageBase class, and modify only ASP This prevents a noticeable slowdown in concretization due to the number of directives involved. * Fix issue with 'clang' being preferred to 'gcc', due to runtime version weights * Constraints on runtimes are declared by compilers The declaration of available runtime versions, and of their compatibility constraints are in the associated compiler class. Co-authored-by: Harmen Stoppels <harmenstoppels@gmail.com>
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/spack/main.py2
-rw-r--r--lib/spack/spack/package_base.py18
-rw-r--r--lib/spack/spack/solver/asp.py182
-rw-r--r--lib/spack/spack/test/concretize_compiler_runtimes.py42
-rw-r--r--lib/spack/spack/test/conftest.py6
5 files changed, 225 insertions, 25 deletions
diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py
index bcdc7d7599..56a4dc0e33 100644
--- a/lib/spack/spack/main.py
+++ b/lib/spack/spack/main.py
@@ -36,7 +36,6 @@ import spack.cmd
import spack.config
import spack.environment as ev
import spack.modules
-import spack.package_base
import spack.paths
import spack.platforms
import spack.repo
@@ -608,7 +607,6 @@ def setup_main_options(args):
[(key, [spack.paths.mock_packages_path])]
)
spack.repo.PATH = spack.repo.create(spack.config.CONFIG)
- spack.package_base.WITH_GCC_RUNTIME = False
# If the user asked for it, don't check ssl certs.
if args.insecure:
diff --git a/lib/spack/spack/package_base.py b/lib/spack/spack/package_base.py
index bf8ed56d95..7d8f7104df 100644
--- a/lib/spack/spack/package_base.py
+++ b/lib/spack/spack/package_base.py
@@ -53,7 +53,6 @@ import spack.url
import spack.util.environment
import spack.util.path
import spack.util.web
-from spack.directives import _depends_on
from spack.filesystem_view import YamlFilesystemView
from spack.install_test import (
PackageTest,
@@ -77,7 +76,6 @@ FLAG_HANDLER_TYPE = Callable[[str, Iterable[str]], FLAG_HANDLER_RETURN_TYPE]
"""Allowed URL schemes for spack packages."""
_ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
-WITH_GCC_RUNTIME = True
#: Filename for the Spack build/install log.
_spack_build_logfile = "spack-build-out.txt"
@@ -373,20 +371,6 @@ def on_package_attributes(**attr_dict):
return _execute_under_condition
-class BinaryPackage:
- """This adds a universal dependency on gcc-runtime."""
-
- def maybe_depend_on_gcc_runtime(self):
- # Do not depend on itself, and allow tests to disable this universal dep
- if self.name == "gcc-runtime" or not WITH_GCC_RUNTIME:
- return
- for v in ["13", "12", "11", "10", "9", "8", "7", "6", "5", "4"]:
- _depends_on(self, f"gcc-runtime@{v}:", type="link", when=f"%gcc@{v} platform=linux")
- _depends_on(self, f"gcc-runtime@{v}:", type="link", when=f"%gcc@{v} platform=cray")
-
- _directives_to_be_executed = [maybe_depend_on_gcc_runtime]
-
-
class PackageViewMixin:
"""This collects all functionality related to adding installed Spack
package to views. Packages can customize how they are added to views by
@@ -449,7 +433,7 @@ class PackageViewMixin:
Pb = TypeVar("Pb", bound="PackageBase")
-class PackageBase(WindowsRPath, PackageViewMixin, BinaryPackage, metaclass=PackageMeta):
+class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta):
"""This is the superclass for all spack packages.
***The Package class***
diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py
index b00982b4fb..ad62371c40 100644
--- a/lib/spack/spack/solver/asp.py
+++ b/lib/spack/spack/solver/asp.py
@@ -11,6 +11,7 @@ import os
import pathlib
import pprint
import re
+import sys
import types
import warnings
from typing import Callable, Dict, List, NamedTuple, Optional, Sequence, Set, Tuple, Union
@@ -61,6 +62,8 @@ GitOrStandardVersion = Union[spack.version.GitVersion, spack.version.StandardVer
ASTType = None
parse_files = None
+#: Enable the addition of a runtime node
+WITH_RUNTIME = sys.platform != "win32"
#: Data class that contain configuration on what a
#: clingo solve should output.
@@ -122,6 +125,8 @@ class Provenance(enum.IntEnum):
PACKAGE_PY = enum.auto()
# An installed spec
INSTALLED = enum.auto()
+ # A runtime injected from another package (e.g. a compiler)
+ RUNTIME = enum.auto()
def __str__(self):
return f"{self._name_.lower()}"
@@ -2023,7 +2028,9 @@ class SpackSolverSetup:
f.node_compiler_version(spec.name, spec.compiler.name, spec.compiler.version)
)
- elif spec.compiler.versions:
+ elif spec.compiler.versions and spec.compiler.versions != vn.any_version:
+ # The condition above emits a facts only if we have an actual constraint
+ # on the compiler version, and avoids emitting them if any version is fine
clauses.append(
fn.attr(
"node_compiler_version_satisfies",
@@ -2578,6 +2585,9 @@ class SpackSolverSetup:
self.possible_virtuals = node_counter.possible_virtuals()
self.pkgs = node_counter.possible_dependencies()
+ runtimes = spack.repo.PATH.packages_with_tags("runtime")
+ self.pkgs.update(set(runtimes))
+
# Fail if we already know an unreachable node is requested
for spec in specs:
missing_deps = [
@@ -2678,6 +2688,10 @@ class SpackSolverSetup:
self.gen.h1("Variant Values defined in specs")
self.define_variant_values()
+ if WITH_RUNTIME:
+ self.gen.h1("Runtimes")
+ self.define_runtime_constraints()
+
self.gen.h1("Version Constraints")
self.collect_virtual_constraints()
self.define_version_constraints()
@@ -2688,6 +2702,21 @@ class SpackSolverSetup:
self.gen.h1("Target Constraints")
self.define_target_constraints()
+ def define_runtime_constraints(self):
+ """Define the constraints to be imposed on the runtimes"""
+ recorder = RuntimePropertyRecorder(self)
+ for compiler in self.possible_compilers:
+ if compiler.name != "gcc":
+ continue
+ try:
+ compiler_cls = spack.repo.PATH.get_pkg_class(compiler.name)
+ except spack.repo.UnknownPackageError:
+ continue
+ if hasattr(compiler_cls, "runtime_constraints"):
+ compiler_cls.runtime_constraints(compiler=compiler, pkg=recorder)
+
+ recorder.consume_facts()
+
def literal_specs(self, specs):
for spec in specs:
self.gen.h2("Spec: %s" % str(spec))
@@ -2796,6 +2825,157 @@ class SpackSolverSetup:
yield _spec_with_default_name(s, pkg_name)
+class RuntimePropertyRecorder:
+ """An object of this class is injected in callbacks to compilers, to let them declare
+ properties of the runtimes they support and of the runtimes they provide, and to add
+ runtime dependencies to the nodes using said compiler.
+
+ The usage of the object is the following. First, a runtime package name or the wildcard
+ "*" are passed as an argument to __call__, to set which kind of package we are referring to.
+ Then we can call one method with a directive-like API.
+
+ Examples:
+ >>> pkg = RuntimePropertyRecorder(setup)
+ >>> # Every package compiled with %gcc has a link dependency on 'gcc-runtime'
+ >>> pkg("*").depends_on(
+ ... "gcc-runtime",
+ ... when="%gcc",
+ ... type="link",
+ ... description="If any package uses %gcc, it depends on gcc-runtime"
+ ... )
+ >>> # The version of gcc-runtime is the same as the %gcc used to "compile" it
+ >>> pkg("gcc-runtime").requires("@=9.4.0", when="%gcc@=9.4.0")
+ """
+
+ def __init__(self, setup):
+ self._setup = setup
+ self.rules = []
+ self.runtime_conditions = set()
+ # State of this object set in the __call__ method, and reset after
+ # each directive-like method
+ self.current_package = None
+
+ def __call__(self, package_name: str) -> "RuntimePropertyRecorder":
+ """Sets a package name for the next directive-like method call"""
+ assert self.current_package is None, f"state was already set to '{self.current_package}'"
+ self.current_package = package_name
+ return self
+
+ def reset(self):
+ """Resets the current state."""
+ self.current_package = None
+
+ def depends_on(self, dependency_str: str, *, when: str, type: str, description: str) -> None:
+ """Injects conditional dependencies on packages.
+
+ Args:
+ dependency_str: the dependency spec to inject
+ when: anonymous condition to be met on a package to have the dependency
+ type: dependency type
+ description: human-readable description of the rule for adding the dependency
+ """
+ # TODO: The API for this function is not final, and is still subject to change. At
+ # TODO: the moment, we implemented only the features strictly needed for the
+ # TODO: functionality currently provided by Spack, and we assert nothing else is required.
+ msg = "the 'depends_on' method can be called only with pkg('*')"
+ assert self.current_package == "*", msg
+
+ when_spec = spack.spec.Spec(when)
+ assert when_spec.name is None, "only anonymous when specs are accepted"
+
+ dependency_spec = spack.spec.Spec(dependency_str)
+ if dependency_spec.versions != vn.any_version:
+ self._setup.version_constraints.add((dependency_spec.name, dependency_spec.versions))
+
+ placeholder = "XXX"
+ node_variable = "node(ID, Package)"
+ when_spec.name = placeholder
+
+ body_clauses = self._setup.spec_clauses(when_spec, body=True)
+ body_str = (
+ f" {f',{os.linesep} '.join(str(x) for x in body_clauses)},\n"
+ f" not runtime(Package)"
+ ).replace(f'"{placeholder}"', f"{node_variable}")
+ head_clauses = self._setup.spec_clauses(dependency_spec, body=False)
+
+ runtime_pkg = dependency_spec.name
+ main_rule = (
+ f"% {description}\n"
+ f'1 {{ attr("depends_on", {node_variable}, node(0..X-1, "{runtime_pkg}"), "{type}") :'
+ f' max_dupes("gcc-runtime", X)}} 1:-\n'
+ f"{body_str}.\n\n"
+ )
+ self.rules.append(main_rule)
+ for clause in head_clauses:
+ if clause.args[0] == "node":
+ continue
+ runtime_node = f'node(RuntimeID, "{runtime_pkg}")'
+ head_str = str(clause).replace(f'"{runtime_pkg}"', runtime_node)
+ rule = (
+ f"{head_str} :-\n"
+ f' attr("depends_on", {node_variable}, {runtime_node}, "{type}"),\n'
+ f"{body_str}.\n\n"
+ )
+ self.rules.append(rule)
+
+ self.reset()
+
+ def requires(self, impose: str, *, when: str):
+ """Injects conditional requirements on a given package.
+
+ Args:
+ impose: constraint to be imposed
+ when: condition triggering the constraint
+ """
+ msg = "the 'requires' method cannot be called with pkg('*') or without setting the package"
+ assert self.current_package is not None and self.current_package != "*", msg
+
+ imposed_spec = spack.spec.Spec(f"{self.current_package}{impose}")
+ when_spec = spack.spec.Spec(f"{self.current_package}{when}")
+
+ assert imposed_spec.versions.concrete, f"{impose} must have a concrete version"
+ assert when_spec.compiler.concrete, f"{when} must have a concrete compiler"
+
+ # Add versions to possible versions
+ for s in (imposed_spec, when_spec):
+ if not s.versions.concrete:
+ continue
+ self._setup.possible_versions[s.name].add(s.version)
+ self._setup.declared_versions[s.name].append(
+ DeclaredVersion(version=s.version, idx=0, origin=Provenance.RUNTIME)
+ )
+
+ self.runtime_conditions.add((imposed_spec, when_spec))
+ self.reset()
+
+ def consume_facts(self):
+ """Consume the facts collected by this object, and emits rules and
+ facts for the runtimes.
+ """
+ self._setup.gen.h2("Runtimes: rules")
+ self._setup.gen.newline()
+ for rule in self.rules:
+ if not isinstance(self._setup.gen.out, llnl.util.lang.Devnull):
+ self._setup.gen.out.write(rule)
+ self._setup.gen.control.add("base", [], rule)
+
+ self._setup.gen.h2("Runtimes: conditions")
+ for runtime_pkg in spack.repo.PATH.packages_with_tags("runtime"):
+ self._setup.gen.fact(fn.runtime(runtime_pkg))
+ self._setup.gen.fact(fn.possible_in_link_run(runtime_pkg))
+ self._setup.gen.newline()
+ # Inject version rules for runtimes (versions are declared based
+ # on the available compilers)
+ self._setup.pkg_version_rules(runtime_pkg)
+
+ for imposed_spec, when_spec in self.runtime_conditions:
+ msg = f"{when_spec} requires {imposed_spec} at runtime"
+ _ = self._setup.condition(when_spec, imposed_spec=imposed_spec, msg=msg)
+
+ self._setup.trigger_rules()
+ self._setup.effect_rules()
+
+
class SpecBuilder:
"""Class with actions to rebuild a spec from ASP results."""
diff --git a/lib/spack/spack/test/concretize_compiler_runtimes.py b/lib/spack/spack/test/concretize_compiler_runtimes.py
new file mode 100644
index 0000000000..089ad28788
--- /dev/null
+++ b/lib/spack/spack/test/concretize_compiler_runtimes.py
@@ -0,0 +1,42 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+import pytest
+
+import spack.paths
+import spack.repo
+import spack.solver.asp
+import spack.spec
+from spack.version import Version
+
+pytestmark = [pytest.mark.only_clingo("Original concretizer does not support compiler runtimes")]
+
+
+@pytest.fixture
+def runtime_repo(config):
+ repo = os.path.join(spack.paths.repos_path, "compiler_runtime.test")
+ with spack.repo.use_repositories(repo) as mock_repo:
+ yield mock_repo
+
+
+@pytest.fixture
+def enable_runtimes():
+ original = spack.solver.asp.WITH_RUNTIME
+ spack.solver.asp.WITH_RUNTIME = True
+ yield
+ spack.solver.asp.WITH_RUNTIME = original
+
+
+def test_correct_gcc_runtime_is_injected_as_dependency(runtime_repo, enable_runtimes):
+ s = spack.spec.Spec("a%gcc@10.2.1 ^b%gcc@4.5.0").concretized()
+ a, b = s["a"], s["b"]
+
+ # Both a and b should depend on the same gcc-runtime directly
+ assert a.dependencies("gcc-runtime") == b.dependencies("gcc-runtime")
+
+ # And the gcc-runtime version should be that of the newest gcc used in the dag.
+ assert a["gcc-runtime"].version == Version("10.2.1")
diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py
index df9a43a123..7b396a0358 100644
--- a/lib/spack/spack/test/conftest.py
+++ b/lib/spack/spack/test/conftest.py
@@ -44,6 +44,7 @@ import spack.package_prefs
import spack.paths
import spack.platforms
import spack.repo
+import spack.solver.asp
import spack.stage
import spack.store
import spack.subprocess_context
@@ -57,11 +58,6 @@ from spack.fetch_strategy import URLFetchStrategy
from spack.util.pattern import Bunch
-@pytest.fixture(scope="session", autouse=True)
-def drop_gcc_runtime():
- spack.package_base.WITH_GCC_RUNTIME = False
-
-
def ensure_configuration_fixture_run_before(request):
"""Ensure that fixture mutating the configuration run before the one where
the function is called.