From 6c12630e956129343b075ca538d4059c03634ef2 Mon Sep 17 00:00:00 2001 From: Jordan Galby <67924449+Jordan474@users.noreply.github.com> Date: Wed, 5 Oct 2022 14:01:59 +0200 Subject: Optimize concurrent misc_cache provider index rebuild (#32874) When concurrent misc_cache provider index rebuilds happen, try to rebuild it only once, so we don't exceed misc_cache lock timeout. For example, when using `spack env depfile`, with no previous misc_cache, running `make -f depfile -j8` could run at most 8 concurrent `spack install` locking on misc_cache to rebuild the provider index. If one rebuild takes 30s, before this fix, the "worst" lock could wait up to 30s * 7, easily exceeding misc_cache lock timeout. Now, the "worst" lock would take 30s * 1 + ~1s * 6. --- lib/spack/spack/repo.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py index 814f84049a..5306b8efdf 100644 --- a/lib/spack/spack/repo.py +++ b/lib/spack/spack/repo.py @@ -559,6 +559,9 @@ class FastPackageChecker(Mapping): def last_mtime(self): return max(sinfo.st_mtime for sinfo in self._packages_to_stats.values()) + def modified_since(self, since): + return [name for name, sinfo in self._packages_to_stats.items() if sinfo.st_mtime > since] + def __getitem__(self, item): return self._packages_to_stats[item] @@ -739,8 +742,7 @@ class RepoIndex(object): # Compute which packages needs to be updated in the cache misc_cache = spack.caches.misc_cache index_mtime = misc_cache.mtime(cache_filename) - - needs_update = [x for x, sinfo in self.checker.items() if sinfo.st_mtime > index_mtime] + needs_update = self.checker.modified_since(index_mtime) index_existed = misc_cache.init_entry(cache_filename) if index_existed and not needs_update: @@ -753,6 +755,12 @@ class RepoIndex(object): with misc_cache.write_transaction(cache_filename) as (old, new): indexer.read(old) if old else indexer.create() + # Compute which packages needs to be updated **again** in case someone updated them + # while we waited for the lock + new_index_mtime = misc_cache.mtime(cache_filename) + if new_index_mtime != index_mtime: + needs_update = self.checker.modified_since(new_index_mtime) + for pkg_name in needs_update: namespaced_name = "%s.%s" % (self.namespace, pkg_name) indexer.update(namespaced_name) -- cgit v1.2.3-70-g09d2