diff options
| -rw-r--r-- | lib/spack/spack/hash_types.py | 3 | ||||
| -rw-r--r-- | lib/spack/spack/spec.py | 103 | ||||
| -rw-r--r-- | lib/spack/spack/test/spec_semantics.py | 68 | ||||
| -rw-r--r-- | var/spack/repos/builtin.mock/packages/splice-h/package.py | 22 | ||||
| -rw-r--r-- | var/spack/repos/builtin.mock/packages/splice-t/package.py | 18 | ||||
| -rw-r--r-- | var/spack/repos/builtin.mock/packages/splice-z/package.py | 18 |
6 files changed, 232 insertions, 0 deletions
diff --git a/lib/spack/spack/hash_types.py b/lib/spack/spack/hash_types.py index 0ad321dec6..554665e24a 100644 --- a/lib/spack/spack/hash_types.py +++ b/lib/spack/spack/hash_types.py @@ -18,6 +18,9 @@ class SpecHashDescriptor(object): We currently use different hashes for different use cases. """ + + hash_types = ('_dag_hash', '_build_hash', '_full_hash') + def __init__(self, deptype=('link', 'run'), package_hash=False, attr=None): self.deptype = dp.canonical_deptype(deptype) self.package_hash = package_hash diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 5986cf8e18..42be765a71 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1088,6 +1088,12 @@ class Spec(object): # external specs. None signal that it was not set yet. self.extra_attributes = None + # This attribute holds the original build copy of the spec if it is + # deployed differently than it was built. None signals that the spec + # is deployed "as built." + # Build spec should be the actual build spec unless marked dirty. + self._build_spec = None + if isinstance(spec_like, six.string_types): spec_list = SpecParser(self).parse(spec_like) if len(spec_list) > 1: @@ -1302,6 +1308,13 @@ class Spec(object): """ return self._concrete + @property + def spliced(self): + """Returns whether or not this Spec is being deployed as built i.e. + whether or not this Spec has ever been spliced. + """ + return any(s.build_spec is not s for s in self.traverse(root=True)) + def traverse(self, **kwargs): direction = kwargs.get('direction', 'children') depth = kwargs.get('depth', False) @@ -2551,7 +2564,13 @@ class Spec(object): Only for internal use -- client code should use "concretize" unless there is a need to force a spec to be concrete. """ + # if set to false, clear out all hashes (set to None or remove attr) + # may need to change references to respect None for s in self.traverse(): + if (not value) and s.concrete and s.package.installed: + continue + elif not value: + s.clear_cached_hashes() s._mark_root_concrete(value) def concretized(self, tests=False): @@ -3365,6 +3384,7 @@ class Spec(object): self.compiler_flags = other.compiler_flags.copy() self.compiler_flags.spec = self self.variants = other.variants.copy() + self._build_spec = other._build_spec # FIXME: we manage _patches_in_order_of_appearance specially here # to keep it from leaking out of spec.py, but we should figure @@ -4225,6 +4245,89 @@ class Spec(object): # to give to the attribute the appropriate comparison semantic return self.architecture.target.microarchitecture + @property + def build_spec(self): + return self._build_spec or self + + @build_spec.setter + def build_spec(self, value): + self._build_spec = value + + def splice(self, other, transitive): + """Splices dependency "other" into this ("target") Spec, and return the + result as a concrete Spec. + If transitive, then other and its dependencies will be extrapolated to + a list of Specs and spliced in accordingly. + For example, let there exist a dependency graph as follows: + T + | \ + Z<-H + In this example, Spec T depends on H and Z, and H also depends on Z. + Suppose, however, that we wish to use a differently-built H, known as + H'. This function will splice in the new H' in one of two ways: + 1. transitively, where H' depends on the Z' it was built with, and the + new T* also directly depends on this new Z', or + 2. intransitively, where the new T* and H' both depend on the original + Z. + Since the Spec returned by this splicing function is no longer deployed + the same way it was built, any such changes are tracked by setting the + build_spec to point to the corresponding dependency from the original + Spec. + TODO: Extend this for non-concrete Specs. + """ + assert self.concrete + assert other.concrete + assert other.name in self + + # Multiple unique specs with the same name will collide, so the + # _dependents of these specs should not be trusted. + # Variants may also be ignored here for now... + + if transitive: + self_nodes = dict((s.name, s.copy(deps=False)) + for s in self.traverse(root=True) + if s.name not in other) + other_nodes = dict((s.name, s.copy(deps=False)) + for s in other.traverse(root=True)) + else: + # If we're not doing a transitive splice, then we only want the + # root of other. + self_nodes = dict((s.name, s.copy(deps=False)) + for s in self.traverse(root=True) + if s.name != other.name) + other_nodes = {other.name: other.copy(deps=False)} + + nodes = other_nodes.copy() + nodes.update(self_nodes) + + for name in nodes: + if name in self_nodes: + dependencies = self[name]._dependencies + for dep in dependencies: + nodes[name]._add_dependency(nodes[dep], + dependencies[dep].deptypes) + if any(dep not in self_nodes for dep in dependencies): + nodes[name].build_spec = self[name].build_spec + else: + dependencies = other[name]._dependencies + for dep in dependencies: + nodes[name]._add_dependency(nodes[dep], + dependencies[dep].deptypes) + if any(dep not in other_nodes for dep in dependencies): + nodes[name].build_spec = other[name].build_spec + + # Clear cached hashes + nodes[self.name].clear_cached_hashes() + return nodes[self.name] + + def clear_cached_hashes(self): + """ + Clears all cached hashes in a Spec, while preserving other properties. + """ + for attr in ht.SpecHashDescriptor.hash_types: + if hasattr(self, attr): + setattr(self, attr, None) + class LazySpecCache(collections.defaultdict): """Cache for Specs that uses a spec_like as key, and computes lazily diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 6aa3068da0..9fb5eaf2b6 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -985,6 +985,74 @@ class TestSpecSematics(object): assert 'avx512' not in spec.target assert spec.target < 'broadwell' + @pytest.mark.parametrize('transitive', [True, False]) + def test_splice(self, transitive): + # Tests the new splice function in Spec using a somewhat simple case + # with a variant with a conditional dependency. + # TODO: Test being able to splice in different provider for a virtual. + # Example: mvapich for mpich. + spec = Spec('splice-t') + dep = Spec('splice-h+foo') + spec.concretize() + dep.concretize() + # Sanity checking that these are not the same thing. + assert dep.dag_hash() != spec['splice-h'].dag_hash() + assert dep.build_hash() != spec['splice-h'].build_hash() + # Do the splice. + out = spec.splice(dep, transitive) + # Returned spec should still be concrete. + assert out.concrete + # Traverse the spec and assert that all dependencies are accounted for. + for node in spec.traverse(): + assert node.name in out + # If the splice worked, then the full hash of the spliced dep should + # now match the full hash of the build spec of the dependency from the + # returned spec. + out_h_build = out['splice-h'].build_spec + assert out_h_build.full_hash() == dep.full_hash() + # Transitivity should determine whether the transitive dependency was + # changed. + expected_z = dep['splice-z'] if transitive else spec['splice-z'] + assert out['splice-z'].full_hash() == expected_z.full_hash() + # Sanity check build spec of out should be the original spec. + assert (out['splice-t'].build_spec.full_hash() == + spec['splice-t'].full_hash()) + # Finally, the spec should know it's been spliced: + assert out.spliced + + @pytest.mark.parametrize('transitive', [True, False]) + def test_splice_input_unchanged(self, transitive): + spec = Spec('splice-t').concretized() + dep = Spec('splice-h+foo').concretized() + orig_spec_hash = spec.full_hash() + orig_dep_hash = dep.full_hash() + spec.splice(dep, transitive) + # Post-splice, dag hash should still be different; no changes should be + # made to these specs. + assert spec.full_hash() == orig_spec_hash + assert dep.full_hash() == orig_dep_hash + + @pytest.mark.parametrize('transitive', [True, False]) + def test_splice_subsequent(self, transitive): + spec = Spec('splice-t') + dep = Spec('splice-h+foo') + spec.concretize() + dep.concretize() + out = spec.splice(dep, transitive) + # Now we attempt a second splice. + dep = Spec('splice-z+bar') + dep.concretize() + # Transitivity shouldn't matter since Splice Z has no dependencies. + out2 = out.splice(dep, transitive) + assert out2.concrete + assert out2['splice-z'].build_hash() != spec['splice-z'].build_hash() + assert out2['splice-z'].build_hash() != out['splice-z'].build_hash() + assert out2['splice-z'].full_hash() != spec['splice-z'].full_hash() + assert out2['splice-z'].full_hash() != out['splice-z'].full_hash() + assert (out2['splice-t'].build_spec.full_hash() == + spec['splice-t'].full_hash()) + assert out2.spliced + @pytest.mark.parametrize('spec,constraint,expected_result', [ ('libelf target=haswell', 'target=broadwell', False), ('libelf target=haswell', 'target=haswell', True), diff --git a/var/spack/repos/builtin.mock/packages/splice-h/package.py b/var/spack/repos/builtin.mock/packages/splice-h/package.py new file mode 100644 index 0000000000..79b91bc963 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/splice-h/package.py @@ -0,0 +1,22 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class SpliceH(AutotoolsPackage): + """Simple package with one optional dependency""" + + homepage = "http://www.example.com" + url = "http://www.example.com/splice-h-1.0.tar.gz" + + version('1.0', '0123456789abcdef0123456789abcdef') + + variant('foo', default=False, description='nope') + variant('bar', default=False, description='nope') + variant('baz', default=False, description='nope') + + depends_on('splice-z') + depends_on('splice-z+foo', when='+foo') diff --git a/var/spack/repos/builtin.mock/packages/splice-t/package.py b/var/spack/repos/builtin.mock/packages/splice-t/package.py new file mode 100644 index 0000000000..ec27fd28b6 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/splice-t/package.py @@ -0,0 +1,18 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class SpliceT(AutotoolsPackage): + """Simple package with one optional dependency""" + + homepage = "http://www.example.com" + url = "http://www.example.com/splice-t-1.0.tar.gz" + + version('1.0', '0123456789abcdef0123456789abcdef') + + depends_on('splice-h') + depends_on('splice-z') diff --git a/var/spack/repos/builtin.mock/packages/splice-z/package.py b/var/spack/repos/builtin.mock/packages/splice-z/package.py new file mode 100644 index 0000000000..e28d359b66 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/splice-z/package.py @@ -0,0 +1,18 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class SpliceZ(AutotoolsPackage): + """Simple package with one optional dependency""" + + homepage = "http://www.example.com" + url = "http://www.example.com/splice-z-1.0.tar.gz" + + version('1.0', '0123456789abcdef0123456789abcdef') + + variant('foo', default=False, description='nope') + variant('bar', default=False, description='nope') |