summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJordan Galby <67924449+Jordan474@users.noreply.github.com>2022-10-05 14:01:59 +0200
committerGitHub <noreply@github.com>2022-10-05 06:01:59 -0600
commit6c12630e956129343b075ca538d4059c03634ef2 (patch)
treedb3f1e6b9c96bdfc74edfa32f22537ec7beadfe1
parent53cea629b774f7d255e37b2c75ad4f25f06764db (diff)
downloadspack-6c12630e956129343b075ca538d4059c03634ef2.tar.gz
spack-6c12630e956129343b075ca538d4059c03634ef2.tar.bz2
spack-6c12630e956129343b075ca538d4059c03634ef2.tar.xz
spack-6c12630e956129343b075ca538d4059c03634ef2.zip
Optimize concurrent misc_cache provider index rebuild (#32874)
When concurrent misc_cache provider index rebuilds happen, try to rebuild it only once, so we don't exceed misc_cache lock timeout. For example, when using `spack env depfile`, with no previous misc_cache, running `make -f depfile -j8` could run at most 8 concurrent `spack install` locking on misc_cache to rebuild the provider index. If one rebuild takes 30s, before this fix, the "worst" lock could wait up to 30s * 7, easily exceeding misc_cache lock timeout. Now, the "worst" lock would take 30s * 1 + ~1s * 6.
-rw-r--r--lib/spack/spack/repo.py12
1 files changed, 10 insertions, 2 deletions
diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py
index 814f84049a..5306b8efdf 100644
--- a/lib/spack/spack/repo.py
+++ b/lib/spack/spack/repo.py
@@ -559,6 +559,9 @@ class FastPackageChecker(Mapping):
def last_mtime(self):
return max(sinfo.st_mtime for sinfo in self._packages_to_stats.values())
+ def modified_since(self, since):
+ return [name for name, sinfo in self._packages_to_stats.items() if sinfo.st_mtime > since]
+
def __getitem__(self, item):
return self._packages_to_stats[item]
@@ -739,8 +742,7 @@ class RepoIndex(object):
# Compute which packages needs to be updated in the cache
misc_cache = spack.caches.misc_cache
index_mtime = misc_cache.mtime(cache_filename)
-
- needs_update = [x for x, sinfo in self.checker.items() if sinfo.st_mtime > index_mtime]
+ needs_update = self.checker.modified_since(index_mtime)
index_existed = misc_cache.init_entry(cache_filename)
if index_existed and not needs_update:
@@ -753,6 +755,12 @@ class RepoIndex(object):
with misc_cache.write_transaction(cache_filename) as (old, new):
indexer.read(old) if old else indexer.create()
+ # Compute which packages needs to be updated **again** in case someone updated them
+ # while we waited for the lock
+ new_index_mtime = misc_cache.mtime(cache_filename)
+ if new_index_mtime != index_mtime:
+ needs_update = self.checker.modified_since(new_index_mtime)
+
for pkg_name in needs_update:
namespaced_name = "%s.%s" % (self.namespace, pkg_name)
indexer.update(namespaced_name)