summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorHarmen Stoppels <harmenstoppels@gmail.com>2022-12-13 17:07:11 +0100
committerGitHub <noreply@github.com>2022-12-13 17:07:11 +0100
commit333da47dc7d7ad098c0e91bd02452129777d7f46 (patch)
tree09c58cf7edc546d59cf75611bfdf071cc2a1f5d9
parent8b68b4ae725007a72b1877356b05e957be2397fb (diff)
downloadspack-333da47dc7d7ad098c0e91bd02452129777d7f46.tar.gz
spack-333da47dc7d7ad098c0e91bd02452129777d7f46.tar.bz2
spack-333da47dc7d7ad098c0e91bd02452129777d7f46.tar.xz
spack-333da47dc7d7ad098c0e91bd02452129777d7f46.zip
Don't fetch to order mirrors (#34359)
When installing binary tarballs, Spack has to download from its binary mirrors. Sometimes Spack has cache available for these mirrors. That cache helps to order mirrors to increase the likelihood of getting a direct hit. However, currently, when Spack can't find a spec in any local cache of mirrors, it's very dumb: - A while ago it used to query each mirror to see if it had a spec, and use that information to order the mirror again, only to go about and do exactly a part of what it just did: fetch the spec from that mirror confused - Recently, it was changed to download a full index.json, which can be multiple dozens of MBs of data and may take a minute to process thanks to the blazing fast performance you get with Python. In a typical use case of concretizing with reuse, the full index.json is already available, and it likely that the local cache gives a perfect mirror ordering on install. (There's typically no need to update any caches). However, in the use case of Gitlab CI, the build jobs don't have cache, and it would be smart to just do direct fetches instead of all the redundant work of (1) and/or (2). Also, direct fetches from mirrors will soon be fast enough to prefer these direct fetches over the excruciating slowness of index.json files.
-rw-r--r--lib/spack/spack/binary_distribution.py9
-rw-r--r--lib/spack/spack/installer.py17
2 files changed, 15 insertions, 11 deletions
diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py
index cae3985326..4a4f999641 100644
--- a/lib/spack/spack/binary_distribution.py
+++ b/lib/spack/spack/binary_distribution.py
@@ -266,10 +266,7 @@ class BinaryCacheIndex(object):
None, just assumes all configured mirrors.
"""
if find_hash not in self._mirrors_for_spec:
- # Not found in the cached index, pull the latest from the server.
- self.update(with_cooldown=True)
- if find_hash not in self._mirrors_for_spec:
- return None
+ return []
results = self._mirrors_for_spec[find_hash]
if not mirrors_to_check:
return results
@@ -2084,8 +2081,8 @@ def get_mirrors_for_spec(spec=None, mirrors_to_check=None, index_only=False):
spec (spack.spec.Spec): The spec to look for in binary mirrors
mirrors_to_check (dict): Optionally override the configured mirrors
with the mirrors in this dictionary.
- index_only (bool): Do not attempt direct fetching of ``spec.json``
- files from remote mirrors, only consider the indices.
+ index_only (bool): When ``index_only`` is set to ``True``, only the local
+ cache is checked, no requests are made.
Return:
A list of objects, each containing a ``mirror_url`` and ``spec`` key
diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py
index 60891b75c4..08d4db6ab7 100644
--- a/lib/spack/spack/installer.py
+++ b/lib/spack/spack/installer.py
@@ -48,6 +48,7 @@ import spack.binary_distribution as binary_distribution
import spack.compilers
import spack.error
import spack.hooks
+import spack.mirror
import spack.package_base
import spack.package_prefs as prefs
import spack.repo
@@ -419,18 +420,24 @@ def _try_install_from_binary_cache(pkg, explicit, unsigned=False, timer=timer.NU
otherwise, ``False``
timer (Timer):
"""
+ # Early exit if no mirrors are configured.
+ if not spack.mirror.MirrorCollection():
+ return False
+
pkg_id = package_id(pkg)
tty.debug("Searching for binary cache of {0}".format(pkg_id))
timer.start("search")
- matches = binary_distribution.get_mirrors_for_spec(pkg.spec)
+ matches = binary_distribution.get_mirrors_for_spec(pkg.spec, index_only=True)
timer.stop("search")
- if not matches:
- return False
-
return _process_binary_cache_tarball(
- pkg, pkg.spec, explicit, unsigned, mirrors_for_spec=matches, timer=timer
+ pkg,
+ pkg.spec,
+ explicit,
+ unsigned,
+ mirrors_for_spec=matches,
+ timer=timer,
)