diff options
Diffstat (limited to 'lib/spack/spack/spec.py')
| -rw-r--r-- | lib/spack/spack/spec.py | 318 |
1 files changed, 217 insertions, 101 deletions
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index edb2c94300..609b20e238 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1,4 +1,4 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -76,10 +76,8 @@ thing. Spack uses ~variant in directory names and in the canonical form of specs to avoid ambiguity. Both are provided because ~ can cause shell expansion when it is the first character in an id typed on the command line. """ -import base64 import sys import collections -import hashlib import itertools import operator import os @@ -108,6 +106,7 @@ import spack.solver import spack.store import spack.util.crypto import spack.util.executable +import spack.util.hash import spack.util.module_cmd as md import spack.util.prefix import spack.util.spack_json as sjson @@ -203,7 +202,7 @@ def colorize_spec(spec): return clr.colorize(re.sub(_separators, insert_color(), str(spec)) + '@.') -@lang.key_ordering +@lang.lazy_lexicographic_ordering class ArchSpec(object): def __init__(self, spec_or_platform_tuple=(None, None, None)): """ Architecture specification a package should be built with. @@ -252,8 +251,10 @@ class ArchSpec(object): return spec_like return ArchSpec(spec_like) - def _cmp_key(self): - return self.platform, self.os, self.target + def _cmp_iter(self): + yield self.platform + yield self.os + yield self.target def _dup(self, other): self.platform = other.platform @@ -534,7 +535,7 @@ class ArchSpec(object): return string in str(self) or string in self.target -@lang.key_ordering +@lang.lazy_lexicographic_ordering class CompilerSpec(object): """The CompilerSpec field represents the compiler or range of compiler versions that a package should be built with. CompilerSpecs have a @@ -623,8 +624,9 @@ class CompilerSpec(object): clone.versions = self.versions.copy() return clone - def _cmp_key(self): - return (self.name, self.versions) + def _cmp_iter(self): + yield self.name + yield self.versions def to_dict(self): d = syaml.syaml_dict([('name', self.name)]) @@ -648,7 +650,7 @@ class CompilerSpec(object): return str(self) -@lang.key_ordering +@lang.lazy_lexicographic_ordering class DependencySpec(object): """DependencySpecs connect two nodes in the DAG, and contain deptypes. @@ -686,10 +688,10 @@ class DependencySpec(object): self.deptypes + dp.canonical_deptype(type) ) - def _cmp_key(self): - return (self.parent.name if self.parent else None, - self.spec.name if self.spec else None, - self.deptypes) + def _cmp_iter(self): + yield self.parent.name if self.parent else None + yield self.spec.name if self.spec else None + yield self.deptypes def __str__(self): return "%s %s--> %s" % (self.parent.name if self.parent else None, @@ -747,8 +749,15 @@ class FlagMap(lang.HashableMap): clone[name] = value return clone - def _cmp_key(self): - return tuple((k, tuple(v)) for k, v in sorted(six.iteritems(self))) + def _cmp_iter(self): + for k, v in sorted(self.items()): + yield k + + def flags(): + for flag in v: + yield flag + + yield flags def __str__(self): sorted_keys = [k for k in sorted(self.keys()) if self[k] != []] @@ -1016,7 +1025,7 @@ class SpecBuildInterface(lang.ObjectWrapper): ) -@lang.key_ordering +@lang.lazy_lexicographic_ordering(set_hash=False) class Spec(object): #: Cache for spec's prefix, computed lazily in the corresponding property @@ -1060,7 +1069,7 @@ class Spec(object): self._hash = None self._build_hash = None - self._cmp_key_cache = None + self._dunder_hash = None self._package = None # Most of these are internal implementation details that can be @@ -1088,6 +1097,12 @@ class Spec(object): # external specs. None signal that it was not set yet. self.extra_attributes = None + # This attribute holds the original build copy of the spec if it is + # deployed differently than it was built. None signals that the spec + # is deployed "as built." + # Build spec should be the actual build spec unless marked dirty. + self._build_spec = None + if isinstance(spec_like, six.string_types): spec_list = SpecParser(self).parse(spec_like) if len(spec_list) > 1: @@ -1302,6 +1317,13 @@ class Spec(object): """ return self._concrete + @property + def spliced(self): + """Returns whether or not this Spec is being deployed as built i.e. + whether or not this Spec has ever been spliced. + """ + return any(s.build_spec is not s for s in self.traverse(root=True)) + def traverse(self, **kwargs): direction = kwargs.get('direction', 'children') depth = kwargs.get('depth', False) @@ -1369,7 +1391,16 @@ class Spec(object): cover = kwargs.get('cover', 'nodes') direction = kwargs.get('direction', 'children') order = kwargs.get('order', 'pre') - deptype = dp.canonical_deptype(deptype) + + # we don't want to run canonical_deptype every time through + # traverse, because it is somewhat expensive. This ensures we + # canonicalize only once. + canonical_deptype = kwargs.get("canonical_deptype", None) + if canonical_deptype is None: + deptype = dp.canonical_deptype(deptype) + kwargs["canonical_deptype"] = deptype + else: + deptype = canonical_deptype # Make sure kwargs have legal values; raise ValueError if not. def validate(name, val, allowed_values): @@ -1473,13 +1504,7 @@ class Spec(object): # this when we move to using package hashing on all specs. node_dict = self.to_node_dict(hash=hash) yaml_text = syaml.dump(node_dict, default_flow_style=True) - sha = hashlib.sha1(yaml_text.encode('utf-8')) - b32_hash = base64.b32encode(sha.digest()).lower() - - if sys.version_info[0] >= 3: - b32_hash = b32_hash.decode('utf-8') - - return b32_hash + return spack.util.hash.b32_hash(yaml_text) def _cached_hash(self, hash, length=None): """Helper function for storing a cached hash on the spec. @@ -1535,7 +1560,7 @@ class Spec(object): def dag_hash_bit_prefix(self, bits): """Get the first <bits> bits of the DAG hash as an integer type.""" - return base32_prefix_bits(self.dag_hash(), bits) + return spack.util.hash.base32_prefix_bits(self.dag_hash(), bits) def to_node_dict(self, hash=ht.dag_hash): """Create a dictionary representing the state of this Spec. @@ -1629,7 +1654,13 @@ class Spec(object): d['patches'] = variant._patches_in_order_of_appearance if hash.package_hash: - d['package_hash'] = self.package.content_hash() + package_hash = self.package.content_hash() + + # Full hashes are in bytes + if (not isinstance(package_hash, six.text_type) + and isinstance(package_hash, six.binary_type)): + package_hash = package_hash.decode('utf-8') + d['package_hash'] = package_hash deps = self.dependencies_dict(deptype=hash.deptype) if deps: @@ -1786,10 +1817,12 @@ class Spec(object): name = next(iter(node)) node = node[name] - spec = Spec(name, full_hash=node.get('full_hash', None)) + spec = Spec() + spec.name = name spec.namespace = node.get('namespace', None) spec._hash = node.get('hash', None) spec._build_hash = node.get('build_hash', None) + spec._full_hash = node.get('full_hash', None) if 'version' in node or 'versions' in node: spec.versions = vn.VersionList.from_dict(node) @@ -2263,6 +2296,8 @@ class Spec(object): # If replacement is external then trim the dependencies if replacement.external: if (spec._dependencies): + for dep in spec.dependencies(): + del dep._dependents[spec.name] changed = True spec._dependencies = DependencyMap() replacement._dependencies = DependencyMap() @@ -2536,17 +2571,27 @@ class Spec(object): else: self._old_concretize(tests) + def _mark_root_concrete(self, value=True): + """Mark just this spec (not dependencies) concrete.""" + if (not value) and self.concrete and self.package.installed: + return + self._normal = value + self._concrete = value + def _mark_concrete(self, value=True): """Mark this spec and its dependencies as concrete. Only for internal use -- client code should use "concretize" unless there is a need to force a spec to be concrete. """ + # if set to false, clear out all hashes (set to None or remove attr) + # may need to change references to respect None for s in self.traverse(): if (not value) and s.concrete and s.package.installed: continue - s._normal = value - s._concrete = value + elif not value: + s.clear_cached_hashes() + s._mark_root_concrete(value) def concretized(self, tests=False): """This is a non-destructive version of concretize(). @@ -3274,7 +3319,7 @@ class Spec(object): """Return list of any virtual deps in this spec.""" return [spec for spec in self.traverse() if spec.virtual] - @property + @property # type: ignore[misc] # decorated prop not supported in mypy @lang.memoized def patches(self): """Return patch objects for any patch sha256 sums on this Spec. @@ -3320,7 +3365,7 @@ class Spec(object): before possibly copying the dependencies of ``other`` onto ``self`` caches (bool or None): preserve cached fields such as - ``_normal``, ``_hash``, and ``_cmp_key_cache``. By + ``_normal``, ``_hash``, and ``_dunder_hash``. By default this is ``False`` if DAG structure would be changed by the copy, ``True`` if it's an exact copy. @@ -3357,6 +3402,7 @@ class Spec(object): self.compiler_flags = other.compiler_flags.copy() self.compiler_flags.spec = self self.variants = other.variants.copy() + self._build_spec = other._build_spec # FIXME: we manage _patches_in_order_of_appearance specially here # to keep it from leaking out of spec.py, but we should figure @@ -3393,13 +3439,13 @@ class Spec(object): if caches: self._hash = other._hash self._build_hash = other._build_hash - self._cmp_key_cache = other._cmp_key_cache + self._dunder_hash = other._dunder_hash self._normal = other._normal self._full_hash = other._full_hash else: self._hash = None self._build_hash = None - self._cmp_key_cache = None + self._dunder_hash = None self._normal = False self._full_hash = None @@ -3518,18 +3564,17 @@ class Spec(object): else: return any(s.satisfies(spec) for s in self.traverse(root=False)) - def sorted_deps(self): - """Return a list of all dependencies sorted by name.""" - deps = self.flat_dependencies() - return tuple(deps[name] for name in sorted(deps)) + def eq_dag(self, other, deptypes=True, vs=None, vo=None): + """True if the full dependency DAGs of specs are equal.""" + if vs is None: + vs = set() + if vo is None: + vo = set() - def _eq_dag(self, other, vs, vo, deptypes): - """Recursive helper for eq_dag and ne_dag. Does the actual DAG - traversal.""" vs.add(id(self)) vo.add(id(other)) - if self.ne_node(other): + if not self.eq_node(other): return False if len(self._dependencies) != len(other._dependencies): @@ -3557,58 +3602,38 @@ class Spec(object): continue # Recursive check for equality - if not s._eq_dag(o, vs, vo, deptypes): + if not s.eq_dag(o, deptypes, vs, vo): return False return True - def eq_dag(self, other, deptypes=True): - """True if the full dependency DAGs of specs are equal.""" - return self._eq_dag(other, set(), set(), deptypes) - - def ne_dag(self, other, deptypes=True): - """True if the full dependency DAGs of specs are not equal.""" - return not self.eq_dag(other, set(), set(), deptypes) - def _cmp_node(self): - """Comparison key for just *this node* and not its deps.""" - # Name or namespace None will lead to invalid comparisons for abstract - # specs. Replace them with the empty string, which is not a valid spec - # name nor namespace so it will not create spurious equalities. - return (self.name or '', - self.namespace or '', - tuple(self.versions), - self.variants, - self.architecture, - self.compiler, - self.compiler_flags) + """Yield comparable elements of just *this node* and not its deps.""" + yield self.name + yield self.namespace + yield self.versions + yield self.variants + yield self.compiler + yield self.compiler_flags + yield self.architecture def eq_node(self, other): """Equality with another spec, not including dependencies.""" - return self._cmp_node() == other._cmp_node() - - def ne_node(self, other): - """Inequality with another spec, not including dependencies.""" - return self._cmp_node() != other._cmp_node() - - def _cmp_key(self): - """This returns a key for the spec *including* DAG structure. - - The key is the concatenation of: - 1. A tuple describing this node in the DAG. - 2. The hash of each of this node's dependencies' cmp_keys. - """ - if self._cmp_key_cache: - return self._cmp_key_cache + return (other is not None) and lang.lazy_eq( + self._cmp_node, other._cmp_node + ) - dep_tuple = tuple( - (d.spec.name, hash(d.spec), tuple(sorted(d.deptypes))) - for name, d in sorted(self._dependencies.items())) + def _cmp_iter(self): + """Lazily yield components of self for comparison.""" + for item in self._cmp_node(): + yield item - key = (self._cmp_node(), dep_tuple) - if self._concrete: - self._cmp_key_cache = key - return key + def deps(): + for _, dep in sorted(self._dependencies.items()): + yield dep.spec.name + yield tuple(sorted(dep.deptypes)) + yield hash(dep.spec) + yield deps def colorized(self): return colorize_spec(self) @@ -3848,7 +3873,9 @@ class Spec(object): 'Format string terminated while reading attribute.' 'Missing terminating }.' ) - return out.getvalue() + + formatted_spec = out.getvalue() + return formatted_spec.strip() def old_format(self, format_string='$_$@$%@+$+$=', **kwargs): """ @@ -4104,12 +4131,12 @@ class Spec(object): kwargs.setdefault('color', None) return self.format(*args, **kwargs) - def dep_string(self): - return ''.join(" ^" + dep.format() for dep in self.sorted_deps()) - def __str__(self): - ret = self.format() + self.dep_string() - return ret.strip() + sorted_nodes = [self] + sorted( + self.traverse(root=False), key=lambda x: x.name + ) + spec_str = " ^".join(d.format() for d in sorted_nodes) + return spec_str.strip() def install_status(self): """Helper for tree to print DB install status.""" @@ -4217,6 +4244,105 @@ class Spec(object): # to give to the attribute the appropriate comparison semantic return self.architecture.target.microarchitecture + @property + def build_spec(self): + return self._build_spec or self + + @build_spec.setter + def build_spec(self, value): + self._build_spec = value + + def splice(self, other, transitive): + """Splices dependency "other" into this ("target") Spec, and return the + result as a concrete Spec. + If transitive, then other and its dependencies will be extrapolated to + a list of Specs and spliced in accordingly. + For example, let there exist a dependency graph as follows: + T + | \ + Z<-H + In this example, Spec T depends on H and Z, and H also depends on Z. + Suppose, however, that we wish to use a differently-built H, known as + H'. This function will splice in the new H' in one of two ways: + 1. transitively, where H' depends on the Z' it was built with, and the + new T* also directly depends on this new Z', or + 2. intransitively, where the new T* and H' both depend on the original + Z. + Since the Spec returned by this splicing function is no longer deployed + the same way it was built, any such changes are tracked by setting the + build_spec to point to the corresponding dependency from the original + Spec. + TODO: Extend this for non-concrete Specs. + """ + assert self.concrete + assert other.concrete + assert other.name in self + + # Multiple unique specs with the same name will collide, so the + # _dependents of these specs should not be trusted. + # Variants may also be ignored here for now... + + if transitive: + self_nodes = dict((s.name, s.copy(deps=False)) + for s in self.traverse(root=True) + if s.name not in other) + other_nodes = dict((s.name, s.copy(deps=False)) + for s in other.traverse(root=True)) + else: + # If we're not doing a transitive splice, then we only want the + # root of other. + self_nodes = dict((s.name, s.copy(deps=False)) + for s in self.traverse(root=True) + if s.name != other.name) + other_nodes = {other.name: other.copy(deps=False)} + + nodes = other_nodes.copy() + nodes.update(self_nodes) + + for name in nodes: + if name in self_nodes: + dependencies = self[name]._dependencies + for dep in dependencies: + nodes[name]._add_dependency(nodes[dep], + dependencies[dep].deptypes) + if any(dep not in self_nodes for dep in dependencies): + nodes[name].build_spec = self[name].build_spec + else: + dependencies = other[name]._dependencies + for dep in dependencies: + nodes[name]._add_dependency(nodes[dep], + dependencies[dep].deptypes) + if any(dep not in other_nodes for dep in dependencies): + nodes[name].build_spec = other[name].build_spec + + # Clear cached hashes + nodes[self.name].clear_cached_hashes() + return nodes[self.name] + + def clear_cached_hashes(self): + """ + Clears all cached hashes in a Spec, while preserving other properties. + """ + for attr in ht.SpecHashDescriptor.hash_types: + if hasattr(self, attr): + setattr(self, attr, None) + + def __hash__(self): + # If the spec is concrete, we leverage the DAG hash and just use + # a 64-bit prefix of it. The DAG hash has the advantage that it's + # computed once per concrete spec, and it's saved -- so if we + # read concrete specs we don't need to recompute the whole hash. + # This is good for large, unchanging specs. + if self.concrete: + if not self._dunder_hash: + self._dunder_hash = self.dag_hash_bit_prefix(64) + return self._dunder_hash + + # This is the normal hash for lazy_lexicographic_ordering. It's + # slow for large specs because it traverses the whole spec graph, + # so we hope it only runs on abstract specs, which are small. + return hash(lang.tuplify(self._cmp_iter)) + class LazySpecCache(collections.defaultdict): """Cache for Specs that uses a spec_like as key, and computes lazily @@ -4627,16 +4753,6 @@ def save_dependency_spec_yamls( fd.write(dep_spec.to_yaml(hash=ht.build_hash)) -def base32_prefix_bits(hash_string, bits): - """Return the first <bits> bits of a base32 string as an integer.""" - if bits > len(hash_string) * 5: - raise ValueError("Too many bits! Requested %d bit prefix of '%s'." - % (bits, hash_string)) - - hash_bytes = base64.b32decode(hash_string, casefold=True) - return spack.util.crypto.prefix_bits(hash_bytes, bits) - - class SpecParseError(spack.error.SpecError): """Wrapper for ParseError for when we're parsing specs.""" def __init__(self, parse_error): |