diff options
| -rw-r--r-- | lib/spack/spack/solver/asp.py | 114 | ||||
| -rw-r--r-- | lib/spack/spack/test/concretize.py | 45 |
2 files changed, 127 insertions, 32 deletions
diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index af0ddd67ae..eba1d8a3eb 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -13,7 +13,7 @@ import pprint import re import types import warnings -from typing import List, NamedTuple, Optional, Sequence, Tuple, Union +from typing import Dict, List, NamedTuple, Optional, Sequence, Tuple, Union import archspec.cpu @@ -971,6 +971,70 @@ class PyclingoDriver: return cycle_result.unsatisfiable +class ConcreteSpecsByHash(collections.abc.Mapping): + """Mapping containing concrete specs keyed by DAG hash. + + The mapping is ensured to be consistent, i.e. if a spec in the mapping has a dependency with + hash X, it is ensured to be the same object in memory as the spec keyed by X. + """ + + def __init__(self) -> None: + self.data: Dict[str, spack.spec.Spec] = {} + + def __getitem__(self, dag_hash: str) -> spack.spec.Spec: + return self.data[dag_hash] + + def add(self, spec: spack.spec.Spec) -> bool: + """Adds a new concrete spec to the mapping. Returns True if the spec was just added, + False if the spec was already in the mapping. + + Args: + spec: spec to be added + + Raises: + ValueError: if the spec is not concrete + """ + if not spec.concrete: + msg = ( + f"trying to store the non-concrete spec '{spec}' in a container " + f"that only accepts concrete" + ) + raise ValueError(msg) + + dag_hash = spec.dag_hash() + if dag_hash in self.data: + return False + + # Here we need to iterate on the input and rewire the copy. + self.data[spec.dag_hash()] = spec.copy(deps=False) + nodes_to_reconstruct = [spec] + + while nodes_to_reconstruct: + input_parent = nodes_to_reconstruct.pop() + container_parent = self.data[input_parent.dag_hash()] + + for edge in input_parent.edges_to_dependencies(): + input_child = edge.spec + container_child = self.data.get(input_child.dag_hash()) + # Copy children that don't exist yet + if container_child is None: + container_child = input_child.copy(deps=False) + self.data[input_child.dag_hash()] = container_child + nodes_to_reconstruct.append(input_child) + + # Rewire edges + container_parent.add_dependency_edge( + dependency_spec=container_child, depflag=edge.depflag, virtuals=edge.virtuals + ) + return True + + def __len__(self) -> int: + return len(self.data) + + def __iter__(self): + return iter(self.data) + + class SpackSolverSetup: """Class to set up and run a Spack concretization solve.""" @@ -994,9 +1058,7 @@ class SpackSolverSetup: # (ID, CompilerSpec) -> dictionary of attributes self.compiler_info = collections.defaultdict(dict) - # hashes we've already added facts for - self.seen_hashes = set() - self.reusable_and_possible = {} + self.reusable_and_possible = ConcreteSpecsByHash() # id for dummy variables self._condition_id_counter = itertools.count() @@ -2318,25 +2380,29 @@ class SpackSolverSetup: for pkg, variant, value in self.variant_values_from_specs: self.gen.fact(fn.pkg_fact(pkg, fn.variant_possible_value(variant, value))) - def _facts_from_concrete_spec(self, spec, possible): + def register_concrete_spec(self, spec, possible): # tell the solver about any installed packages that could # be dependencies (don't tell it about the others) - h = spec.dag_hash() - if spec.name in possible and h not in self.seen_hashes: - self.reusable_and_possible[h] = spec - try: - # Only consider installed packages for repo we know - spack.repo.PATH.get(spec) - except (spack.repo.UnknownNamespaceError, spack.repo.UnknownPackageError): - return + if spec.name not in possible: + return + + try: + # Only consider installed packages for repo we know + spack.repo.PATH.get(spec) + except (spack.repo.UnknownNamespaceError, spack.repo.UnknownPackageError) as e: + tty.debug(f"[REUSE] Issues when trying to reuse {spec.short_spec}: {str(e)}") + return + + self.reusable_and_possible.add(spec) + def concrete_specs(self): + """Emit facts for reusable specs""" + for h, spec in self.reusable_and_possible.items(): # this indicates that there is a spec like this installed self.gen.fact(fn.installed_hash(spec.name, h)) - # this describes what constraints it imposes on the solve self.impose(h, spec, body=True) self.gen.newline() - # Declare as possible parts of specs that are not in package.py # - Add versions to possible versions # - Add OS to possible OS's @@ -2347,15 +2413,12 @@ class SpackSolverSetup: ) self.possible_oses.add(dep.os) - # add the hash to the one seen so far - self.seen_hashes.add(h) - def define_concrete_input_specs(self, specs, possible): # any concrete specs in the input spec list for input_spec in specs: for spec in input_spec.traverse(): if spec.concrete: - self._facts_from_concrete_spec(spec, possible) + self.register_concrete_spec(spec, possible) def setup( self, @@ -2422,14 +2485,13 @@ class SpackSolverSetup: # get possible compilers self.possible_compilers = self.generate_possible_compilers(specs) - self.gen.h1("Concrete input spec definitions") + self.gen.h1("Reusable concrete specs") self.define_concrete_input_specs(specs, self.pkgs) - if reuse: - self.gen.h1("Reusable specs") self.gen.fact(fn.optimize_for_reuse()) for reusable_spec in reuse: - self._facts_from_concrete_spec(reusable_spec, self.pkgs) + self.register_concrete_spec(reusable_spec, self.pkgs) + self.concrete_specs() self.gen.h1("Generic statements on possible packages") node_counter.possible_packages_facts(self.gen, fn) @@ -2620,7 +2682,6 @@ class SpecBuilder: self._specs = {} self._result = None self._command_line_specs = specs - self._hash_specs = [] self._flag_sources = collections.defaultdict(lambda: set()) self._flag_compiler_defaults = set() @@ -2631,7 +2692,6 @@ class SpecBuilder: def hash(self, node, h): if node not in self._specs: self._specs[node] = self._hash_lookup[h] - self._hash_specs.append(node) def node(self, node): if node not in self._specs: @@ -2869,12 +2929,10 @@ class SpecBuilder: # fix flags after all specs are constructed self.reorder_flags() - # cycle detection - roots = [spec.root for spec in self._specs.values() if not spec.root.installed] - # inject patches -- note that we' can't use set() to unique the # roots here, because the specs aren't complete, and the hash # function will loop forever. + roots = [spec.root for spec in self._specs.values() if not spec.root.installed] roots = dict((id(r), r) for r in roots) for root in roots.values(): spack.spec.Spec.inject_patches_variant(root) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 53f9c64d5e..48334e70a1 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -1170,7 +1170,7 @@ class TestConcretize: ) @pytest.mark.parametrize("mock_db", [True, False]) def test_reuse_does_not_overwrite_dev_specs( - self, dev_first, spec, mock_db, tmpdir, monkeypatch + self, dev_first, spec, mock_db, tmpdir, temporary_store, monkeypatch ): """Test that reuse does not mix dev specs with non-dev specs. @@ -1182,8 +1182,7 @@ class TestConcretize: # dev and non-dev specs that are otherwise identical spec = Spec(spec) dev_spec = spec.copy() - dev_constraint = "dev_path=%s" % tmpdir.strpath - dev_spec["dev-build-test-install"].constrain(dev_constraint) + dev_spec["dev-build-test-install"].constrain(f"dev_path={tmpdir.strpath}") # run the test in both orders first_spec = dev_spec if dev_first else spec @@ -1196,7 +1195,7 @@ class TestConcretize: return [first_spec] if mock_db: - monkeypatch.setattr(spack.store.STORE.db, "query", mock_fn) + temporary_store.db.add(first_spec, None) else: monkeypatch.setattr(spack.binary_distribution, "update_cache_and_get_specs", mock_fn) @@ -2112,6 +2111,24 @@ class TestConcretize: # when checksums are required Spec("a@=3.0").concretized() + @pytest.mark.regression("39570") + @pytest.mark.db + def test_reuse_python_from_cli_and_extension_from_db(self, mutable_database): + """Tests that reusing python with and explicit request on the command line, when the spec + also reuses a python extension from the DB, doesn't fail. + """ + s = Spec("py-extension1").concretized() + python_hash = s["python"].dag_hash() + s.package.do_install(fake=True, explicit=True) + + with spack.config.override("concretizer:reuse", True): + with_reuse = Spec(f"py-extension2 ^/{python_hash}").concretized() + + with spack.config.override("concretizer:reuse", False): + without_reuse = Spec("py-extension2").concretized() + + assert with_reuse.dag_hash() == without_reuse.dag_hash() + @pytest.fixture() def duplicates_test_repository(): @@ -2246,3 +2263,23 @@ class TestConcretizeSeparately: def test_drop_moving_targets(v_str, v_opts, checksummed): v = Version(v_str) assert spack.solver.asp._is_checksummed_version((v, v_opts)) == checksummed + + +class TestConcreteSpecsByHash: + """Tests the container of concrete specs""" + + @pytest.mark.parametrize("input_specs", [["a"], ["a foobar=bar", "b"], ["a foobar=baz", "b"]]) + def test_adding_specs(self, input_specs, default_mock_concretization): + """Tests that concrete specs in the container are equivalent, but stored as different + objects in memory. + """ + container = spack.solver.asp.ConcreteSpecsByHash() + input_specs = [Spec(s).concretized() for s in input_specs] + for s in input_specs: + container.add(s) + + for root in input_specs: + for node in root.traverse(root=True): + assert node == container[node.dag_hash()] + assert node.dag_hash() in container + assert node is not container[node.dag_hash()] |