summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/spack/solver/asp.py114
-rw-r--r--lib/spack/spack/test/concretize.py45
2 files changed, 127 insertions, 32 deletions
diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py
index af0ddd67ae..eba1d8a3eb 100644
--- a/lib/spack/spack/solver/asp.py
+++ b/lib/spack/spack/solver/asp.py
@@ -13,7 +13,7 @@ import pprint
import re
import types
import warnings
-from typing import List, NamedTuple, Optional, Sequence, Tuple, Union
+from typing import Dict, List, NamedTuple, Optional, Sequence, Tuple, Union
import archspec.cpu
@@ -971,6 +971,70 @@ class PyclingoDriver:
return cycle_result.unsatisfiable
+class ConcreteSpecsByHash(collections.abc.Mapping):
+ """Mapping containing concrete specs keyed by DAG hash.
+
+ The mapping is ensured to be consistent, i.e. if a spec in the mapping has a dependency with
+ hash X, it is ensured to be the same object in memory as the spec keyed by X.
+ """
+
+ def __init__(self) -> None:
+ self.data: Dict[str, spack.spec.Spec] = {}
+
+ def __getitem__(self, dag_hash: str) -> spack.spec.Spec:
+ return self.data[dag_hash]
+
+ def add(self, spec: spack.spec.Spec) -> bool:
+ """Adds a new concrete spec to the mapping. Returns True if the spec was just added,
+ False if the spec was already in the mapping.
+
+ Args:
+ spec: spec to be added
+
+ Raises:
+ ValueError: if the spec is not concrete
+ """
+ if not spec.concrete:
+ msg = (
+ f"trying to store the non-concrete spec '{spec}' in a container "
+ f"that only accepts concrete"
+ )
+ raise ValueError(msg)
+
+ dag_hash = spec.dag_hash()
+ if dag_hash in self.data:
+ return False
+
+ # Here we need to iterate on the input and rewire the copy.
+ self.data[spec.dag_hash()] = spec.copy(deps=False)
+ nodes_to_reconstruct = [spec]
+
+ while nodes_to_reconstruct:
+ input_parent = nodes_to_reconstruct.pop()
+ container_parent = self.data[input_parent.dag_hash()]
+
+ for edge in input_parent.edges_to_dependencies():
+ input_child = edge.spec
+ container_child = self.data.get(input_child.dag_hash())
+ # Copy children that don't exist yet
+ if container_child is None:
+ container_child = input_child.copy(deps=False)
+ self.data[input_child.dag_hash()] = container_child
+ nodes_to_reconstruct.append(input_child)
+
+ # Rewire edges
+ container_parent.add_dependency_edge(
+ dependency_spec=container_child, depflag=edge.depflag, virtuals=edge.virtuals
+ )
+ return True
+
+ def __len__(self) -> int:
+ return len(self.data)
+
+ def __iter__(self):
+ return iter(self.data)
+
+
class SpackSolverSetup:
"""Class to set up and run a Spack concretization solve."""
@@ -994,9 +1058,7 @@ class SpackSolverSetup:
# (ID, CompilerSpec) -> dictionary of attributes
self.compiler_info = collections.defaultdict(dict)
- # hashes we've already added facts for
- self.seen_hashes = set()
- self.reusable_and_possible = {}
+ self.reusable_and_possible = ConcreteSpecsByHash()
# id for dummy variables
self._condition_id_counter = itertools.count()
@@ -2318,25 +2380,29 @@ class SpackSolverSetup:
for pkg, variant, value in self.variant_values_from_specs:
self.gen.fact(fn.pkg_fact(pkg, fn.variant_possible_value(variant, value)))
- def _facts_from_concrete_spec(self, spec, possible):
+ def register_concrete_spec(self, spec, possible):
# tell the solver about any installed packages that could
# be dependencies (don't tell it about the others)
- h = spec.dag_hash()
- if spec.name in possible and h not in self.seen_hashes:
- self.reusable_and_possible[h] = spec
- try:
- # Only consider installed packages for repo we know
- spack.repo.PATH.get(spec)
- except (spack.repo.UnknownNamespaceError, spack.repo.UnknownPackageError):
- return
+ if spec.name not in possible:
+ return
+
+ try:
+ # Only consider installed packages for repo we know
+ spack.repo.PATH.get(spec)
+ except (spack.repo.UnknownNamespaceError, spack.repo.UnknownPackageError) as e:
+ tty.debug(f"[REUSE] Issues when trying to reuse {spec.short_spec}: {str(e)}")
+ return
+
+ self.reusable_and_possible.add(spec)
+ def concrete_specs(self):
+ """Emit facts for reusable specs"""
+ for h, spec in self.reusable_and_possible.items():
# this indicates that there is a spec like this installed
self.gen.fact(fn.installed_hash(spec.name, h))
-
# this describes what constraints it imposes on the solve
self.impose(h, spec, body=True)
self.gen.newline()
-
# Declare as possible parts of specs that are not in package.py
# - Add versions to possible versions
# - Add OS to possible OS's
@@ -2347,15 +2413,12 @@ class SpackSolverSetup:
)
self.possible_oses.add(dep.os)
- # add the hash to the one seen so far
- self.seen_hashes.add(h)
-
def define_concrete_input_specs(self, specs, possible):
# any concrete specs in the input spec list
for input_spec in specs:
for spec in input_spec.traverse():
if spec.concrete:
- self._facts_from_concrete_spec(spec, possible)
+ self.register_concrete_spec(spec, possible)
def setup(
self,
@@ -2422,14 +2485,13 @@ class SpackSolverSetup:
# get possible compilers
self.possible_compilers = self.generate_possible_compilers(specs)
- self.gen.h1("Concrete input spec definitions")
+ self.gen.h1("Reusable concrete specs")
self.define_concrete_input_specs(specs, self.pkgs)
-
if reuse:
- self.gen.h1("Reusable specs")
self.gen.fact(fn.optimize_for_reuse())
for reusable_spec in reuse:
- self._facts_from_concrete_spec(reusable_spec, self.pkgs)
+ self.register_concrete_spec(reusable_spec, self.pkgs)
+ self.concrete_specs()
self.gen.h1("Generic statements on possible packages")
node_counter.possible_packages_facts(self.gen, fn)
@@ -2620,7 +2682,6 @@ class SpecBuilder:
self._specs = {}
self._result = None
self._command_line_specs = specs
- self._hash_specs = []
self._flag_sources = collections.defaultdict(lambda: set())
self._flag_compiler_defaults = set()
@@ -2631,7 +2692,6 @@ class SpecBuilder:
def hash(self, node, h):
if node not in self._specs:
self._specs[node] = self._hash_lookup[h]
- self._hash_specs.append(node)
def node(self, node):
if node not in self._specs:
@@ -2869,12 +2929,10 @@ class SpecBuilder:
# fix flags after all specs are constructed
self.reorder_flags()
- # cycle detection
- roots = [spec.root for spec in self._specs.values() if not spec.root.installed]
-
# inject patches -- note that we' can't use set() to unique the
# roots here, because the specs aren't complete, and the hash
# function will loop forever.
+ roots = [spec.root for spec in self._specs.values() if not spec.root.installed]
roots = dict((id(r), r) for r in roots)
for root in roots.values():
spack.spec.Spec.inject_patches_variant(root)
diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py
index 53f9c64d5e..48334e70a1 100644
--- a/lib/spack/spack/test/concretize.py
+++ b/lib/spack/spack/test/concretize.py
@@ -1170,7 +1170,7 @@ class TestConcretize:
)
@pytest.mark.parametrize("mock_db", [True, False])
def test_reuse_does_not_overwrite_dev_specs(
- self, dev_first, spec, mock_db, tmpdir, monkeypatch
+ self, dev_first, spec, mock_db, tmpdir, temporary_store, monkeypatch
):
"""Test that reuse does not mix dev specs with non-dev specs.
@@ -1182,8 +1182,7 @@ class TestConcretize:
# dev and non-dev specs that are otherwise identical
spec = Spec(spec)
dev_spec = spec.copy()
- dev_constraint = "dev_path=%s" % tmpdir.strpath
- dev_spec["dev-build-test-install"].constrain(dev_constraint)
+ dev_spec["dev-build-test-install"].constrain(f"dev_path={tmpdir.strpath}")
# run the test in both orders
first_spec = dev_spec if dev_first else spec
@@ -1196,7 +1195,7 @@ class TestConcretize:
return [first_spec]
if mock_db:
- monkeypatch.setattr(spack.store.STORE.db, "query", mock_fn)
+ temporary_store.db.add(first_spec, None)
else:
monkeypatch.setattr(spack.binary_distribution, "update_cache_and_get_specs", mock_fn)
@@ -2112,6 +2111,24 @@ class TestConcretize:
# when checksums are required
Spec("a@=3.0").concretized()
+ @pytest.mark.regression("39570")
+ @pytest.mark.db
+ def test_reuse_python_from_cli_and_extension_from_db(self, mutable_database):
+ """Tests that reusing python with and explicit request on the command line, when the spec
+ also reuses a python extension from the DB, doesn't fail.
+ """
+ s = Spec("py-extension1").concretized()
+ python_hash = s["python"].dag_hash()
+ s.package.do_install(fake=True, explicit=True)
+
+ with spack.config.override("concretizer:reuse", True):
+ with_reuse = Spec(f"py-extension2 ^/{python_hash}").concretized()
+
+ with spack.config.override("concretizer:reuse", False):
+ without_reuse = Spec("py-extension2").concretized()
+
+ assert with_reuse.dag_hash() == without_reuse.dag_hash()
+
@pytest.fixture()
def duplicates_test_repository():
@@ -2246,3 +2263,23 @@ class TestConcretizeSeparately:
def test_drop_moving_targets(v_str, v_opts, checksummed):
v = Version(v_str)
assert spack.solver.asp._is_checksummed_version((v, v_opts)) == checksummed
+
+
+class TestConcreteSpecsByHash:
+ """Tests the container of concrete specs"""
+
+ @pytest.mark.parametrize("input_specs", [["a"], ["a foobar=bar", "b"], ["a foobar=baz", "b"]])
+ def test_adding_specs(self, input_specs, default_mock_concretization):
+ """Tests that concrete specs in the container are equivalent, but stored as different
+ objects in memory.
+ """
+ container = spack.solver.asp.ConcreteSpecsByHash()
+ input_specs = [Spec(s).concretized() for s in input_specs]
+ for s in input_specs:
+ container.add(s)
+
+ for root in input_specs:
+ for node in root.traverse(root=True):
+ assert node == container[node.dag_hash()]
+ assert node.dag_hash() in container
+ assert node is not container[node.dag_hash()]