diff options
| -rw-r--r-- | lib/spack/spack/database.py | 134 | ||||
| -rw-r--r-- | lib/spack/spack/spec.py | 8 |
2 files changed, 73 insertions, 69 deletions
diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 3c7dff96db..02eae97f3f 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -59,7 +59,11 @@ import spack.traverse as tr import spack.util.lock as lk import spack.util.spack_json as sjson import spack.version as vn -from spack.directory_layout import DirectoryLayoutError, InconsistentInstallDirectoryError +from spack.directory_layout import ( + DirectoryLayout, + DirectoryLayoutError, + InconsistentInstallDirectoryError, +) from spack.error import SpackError from spack.util.crypto import bit_length @@ -950,13 +954,17 @@ class Database: raise def _construct_entry_from_directory_layout( - self, directory_layout, old_data, spec, deprecator=None + self, + directory_layout: DirectoryLayout, + old_data: Dict[str, InstallRecord], + spec: "spack.spec.Spec", + deprecator: Optional["spack.spec.Spec"] = None, ): # Try to recover explicit value from old DB, but # default it to True if DB was corrupt. This is # just to be conservative in case a command like # "autoremove" is run by the user after a reindex. - tty.debug("RECONSTRUCTING FROM SPEC.YAML: {0}".format(spec)) + tty.debug(f"Reconstructing from spec file: {spec}") explicit = True inst_time = os.stat(spec.prefix).st_ctime if old_data is not None: @@ -965,17 +973,17 @@ class Database: explicit = old_info.explicit inst_time = old_info.installation_time - extra_args = {"explicit": explicit, "installation_time": inst_time} - self._add(spec, directory_layout, **extra_args) + self._add(spec, directory_layout, explicit=explicit, installation_time=inst_time) if deprecator: self._deprecate(spec, deprecator) - def _construct_from_directory_layout(self, directory_layout, old_data): - # Read first the `spec.yaml` files in the prefixes. They should be - # considered authoritative with respect to DB reindexing, as - # entries in the DB may be corrupted in a way that still makes - # them readable. If we considered DB entries authoritative - # instead, we would perpetuate errors over a reindex. + def _construct_from_directory_layout( + self, directory_layout: DirectoryLayout, old_data: Dict[str, InstallRecord] + ): + # Read first the spec files in the prefixes. They should be considered authoritative with + # respect to DB reindexing, as entries in the DB may be corrupted in a way that still makes + # them readable. If we considered DB entries authoritative instead, we would perpetuate + # errors over a reindex. with directory_layout.disable_upstream_check(): # Initialize data in the reconstructed DB self._data = {} @@ -994,34 +1002,29 @@ class Database: ) processed_specs.add(spec) - for key, entry in old_data.items(): - # We already took care of this spec using - # `spec.yaml` from its prefix. + for entry in old_data.values(): + # We already took care of this spec using spec file from its prefix. if entry.spec in processed_specs: - msg = "SKIPPING RECONSTRUCTION FROM OLD DB: {0}" - msg += " [already reconstructed from spec.yaml]" - tty.debug(msg.format(entry.spec)) + tty.debug( + f"Skipping reconstruction from old db: {entry.spec}" + " [already reconstructed from spec file]" + ) continue - # If we arrived here it very likely means that - # we have external specs that are not dependencies - # of other specs. This may be the case for externally - # installed compilers or externally installed - # applications. - tty.debug("RECONSTRUCTING FROM OLD DB: {0}".format(entry.spec)) + # If we arrived here it very likely means that we have external specs that are not + # dependencies of other specs. This may be the case for externally installed + # compilers or externally installed applications. + tty.debug(f"Reconstructing from old db: {entry.spec}") try: - layout = None if entry.spec.external else directory_layout - kwargs = { - "spec": entry.spec, - "directory_layout": layout, - "explicit": entry.explicit, - "installation_time": entry.installation_time, - } - self._add(**kwargs) + self._add( + spec=entry.spec, + directory_layout=None if entry.spec.external else directory_layout, + explicit=entry.explicit, + installation_time=entry.installation_time, + ) processed_specs.add(entry.spec) except Exception as e: - # Something went wrong, so the spec was not restored - # from old data + # Something went wrong, so the spec was not restored from old data tty.debug(e) self._check_ref_counts() @@ -1117,29 +1120,27 @@ class Database: def _add( self, - spec, - directory_layout=None, - explicit=False, - installation_time=None, - allow_missing=False, + spec: "spack.spec.Spec", + directory_layout: Optional[DirectoryLayout] = None, + explicit: bool = False, + installation_time: Optional[float] = None, + allow_missing: bool = False, ): """Add an install record for this spec to the database. Assumes spec is installed in ``directory_layout.path_for_spec(spec)``. - Also ensures dependencies are present and updated in the DB as - either installed or missing. + Also ensures dependencies are present and updated in the DB as either installed or missing. Args: - spec (spack.spec.Spec): spec to be added + spec: spec to be added directory_layout: layout of the spec installation explicit: Possible values: True, False, any - A spec that was installed following a specific user - request is marked as explicit. If instead it was - pulled-in as a dependency of a user requested spec - it's considered implicit. + A spec that was installed following a specific user request is marked as explicit. + If instead it was pulled-in as a dependency of a user requested spec it's + considered implicit. installation_time: Date and time of installation @@ -1150,29 +1151,25 @@ class Database: raise NonConcreteSpecAddError("Specs added to DB must be concrete.") key = spec.dag_hash() - spec_pkg_hash = spec._package_hash + spec_pkg_hash = spec._package_hash # type: ignore[attr-defined] upstream, record = self.query_by_spec_hash(key) if upstream: return - # Retrieve optional arguments installation_time = installation_time or _now() for edge in spec.edges_to_dependencies(depflag=_TRACKED_DEPENDENCIES): if edge.spec.dag_hash() in self._data: continue - # allow missing build-only deps. This prevents excessive - # warnings when a spec is installed, and its build dep - # is missing a build dep; there's no need to install the - # build dep's build dep first, and there's no need to warn - # about it missing. - dep_allow_missing = allow_missing or edge.depflag == dt.BUILD self._add( edge.spec, directory_layout, explicit=False, installation_time=installation_time, - allow_missing=dep_allow_missing, + # allow missing build-only deps. This prevents excessive warnings when a spec is + # installed, and its build dep is missing a build dep; there's no need to install + # the build dep's build dep first, and there's no need to warn about it missing. + allow_missing=allow_missing or edge.depflag == dt.BUILD, ) # Make sure the directory layout agrees whether the spec is installed @@ -1185,13 +1182,12 @@ class Database: self._installed_prefixes.add(path) except DirectoryLayoutError as e: if not (allow_missing and isinstance(e, InconsistentInstallDirectoryError)): - msg = ( - "{0} is being {1} in the database with prefix {2}, " + action = "updated" if key in self._data else "registered" + tty.warn( + f"{spec.short_spec} is being {action} in the database with prefix {path}, " "but this directory does not contain an installation of " - "the spec, due to: {3}" + f"the spec, due to: {e}" ) - action = "updated" if key in self._data else "registered" - tty.warn(msg.format(spec.short_spec, action, path, str(e))) elif spec.external_path: path = spec.external_path installed = True @@ -1202,23 +1198,27 @@ class Database: if key not in self._data: # Create a new install record with no deps initially. new_spec = spec.copy(deps=False) - extra_args = {"explicit": explicit, "installation_time": installation_time} - # Commands other than 'spack install' may add specs to the DB, - # we can record the source of an installed Spec with 'origin' - if hasattr(spec, "origin"): - extra_args["origin"] = spec.origin - self._data[key] = InstallRecord(new_spec, path, installed, ref_count=0, **extra_args) + self._data[key] = InstallRecord( + new_spec, + path=path, + installed=installed, + ref_count=0, + explicit=explicit, + installation_time=installation_time, + origin=None if not hasattr(spec, "origin") else spec.origin, + ) # Connect dependencies from the DB to the new copy. for dep in spec.edges_to_dependencies(depflag=_TRACKED_DEPENDENCIES): dkey = dep.spec.dag_hash() upstream, record = self.query_by_spec_hash(dkey) + assert record, f"Missing dependency {dep.spec} in DB" new_spec._add_dependency(record.spec, depflag=dep.depflag, virtuals=dep.virtuals) if not upstream: record.ref_count += 1 - # Mark concrete once everything is built, and preserve - # the original hashes of concrete specs. + # Mark concrete once everything is built, and preserve the original hashes of concrete + # specs. new_spec._mark_concrete() new_spec._hash = key new_spec._package_hash = spec_pkg_hash diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 92e70e90da..0e7746603d 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1552,7 +1552,9 @@ class Spec: raise spack.error.SpecError(err_msg.format(name, len(deps))) return deps[0] - def edges_from_dependents(self, name=None, depflag: dt.DepFlag = dt.ALL): + def edges_from_dependents( + self, name=None, depflag: dt.DepFlag = dt.ALL + ) -> List[DependencySpec]: """Return a list of edges connecting this node in the DAG to parents. @@ -1562,7 +1564,9 @@ class Spec: """ return [d for d in self._dependents.select(parent=name, depflag=depflag)] - def edges_to_dependencies(self, name=None, depflag: dt.DepFlag = dt.ALL): + def edges_to_dependencies( + self, name=None, depflag: dt.DepFlag = dt.ALL + ) -> List[DependencySpec]: """Return a list of edges connecting this node in the DAG to children. |