summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorPatrick Gartung <gartung@fnal.gov>2020-01-30 16:06:50 -0600
committerGitHub <noreply@github.com>2020-01-30 16:06:50 -0600
commited501eaab275f50dc3d689ee9e8d6c7fa707fd94 (patch)
tree5e82c97f5d5cd2a8a033d5e7ad16f724d45287b1 /lib
parent12a99f4a2df3f4bbb3dc85414cd3b7aaa2849a6c (diff)
downloadspack-ed501eaab275f50dc3d689ee9e8d6c7fa707fd94.tar.gz
spack-ed501eaab275f50dc3d689ee9e8d6c7fa707fd94.tar.bz2
spack-ed501eaab275f50dc3d689ee9e8d6c7fa707fd94.tar.xz
spack-ed501eaab275f50dc3d689ee9e8d6c7fa707fd94.zip
Bypass build_cache/index.html read when trying to download spec.yaml for concretized spec. (#14698)
* Add binary_distribution::get_spec which takes concretized spec Add binary_distribution::try_download_specs for downloading of spec.yaml files to cache get_spec is used by package::try_install_from_binary_cache to download only the spec.yaml for the concretized spec if it exists.
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/spack/binary_distribution.py86
-rw-r--r--lib/spack/spack/package.py3
2 files changed, 68 insertions, 21 deletions
diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py
index b898f27f49..3e5dc89313 100644
--- a/lib/spack/spack/binary_distribution.py
+++ b/lib/spack/spack/binary_distribution.py
@@ -660,16 +660,79 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
shutil.rmtree(tmpdir)
-#: Internal cache for get_specs
+# Internal cache for downloaded specs
_cached_specs = None
+def try_download_specs(urls=None, force=False):
+ '''
+ Try to download the urls and cache them
+ '''
+ global _cached_specs
+ _cached_specs = []
+ if urls is None:
+ return {}
+ for link in urls:
+ with Stage(link, name="build_cache", keep=True) as stage:
+ if force and os.path.exists(stage.save_filename):
+ os.remove(stage.save_filename)
+ if not os.path.exists(stage.save_filename):
+ try:
+ stage.fetch()
+ except fs.FetchError:
+ continue
+ with open(stage.save_filename, 'r') as f:
+ # read the spec from the build cache file. All specs
+ # in build caches are concrete (as they are built) so
+ # we need to mark this spec concrete on read-in.
+ spec = Spec.from_yaml(f)
+ spec._mark_concrete()
+ _cached_specs.append(spec)
+
+ return _cached_specs
+
+
+def get_spec(spec=None, force=False):
+ """
+ Check if spec.yaml exists on mirrors and return it if it does
+ """
+ global _cached_specs
+ urls = set()
+ if spec is None:
+ return {}
+ specfile_name = tarball_name(spec, '.spec.yaml')
+ if _cached_specs:
+ tty.debug("Using previously-retrieved specs")
+ return _cached_specs
+
+ if not spack.mirror.MirrorCollection():
+ tty.debug("No Spack mirrors are currently configured")
+ return {}
+
+ for mirror in spack.mirror.MirrorCollection().values():
+ fetch_url_build_cache = url_util.join(
+ mirror.fetch_url, _build_cache_relative_path)
+
+ mirror_dir = url_util.local_file_path(fetch_url_build_cache)
+ if mirror_dir:
+ tty.msg("Finding buildcaches in %s" % mirror_dir)
+ link = url_util.join(fetch_url_build_cache, specfile_name)
+ urls.add(link)
+
+ else:
+ tty.msg("Finding buildcaches at %s" %
+ url_util.format(fetch_url_build_cache))
+ link = url_util.join(fetch_url_build_cache, specfile_name)
+ urls.add(link)
+
+ return try_download_specs(urls=urls, force=force)
+
+
def get_specs(force=False, use_arch=False, names=None):
"""
Get spec.yaml's for build caches available on mirror
"""
global _cached_specs
-
arch = architecture.Arch(architecture.platform(),
'default_os', 'default_target')
arch_pattern = ('([^-]*-[^-]*-[^-]*)')
@@ -716,25 +779,8 @@ def get_specs(force=False, use_arch=False, names=None):
m = name_re.search(link)
if m:
urls.add(link)
- _cached_specs = []
- for link in urls:
- with Stage(link, name="build_cache", keep=True) as stage:
- if force and os.path.exists(stage.save_filename):
- os.remove(stage.save_filename)
- if not os.path.exists(stage.save_filename):
- try:
- stage.fetch()
- except fs.FetchError:
- continue
- with open(stage.save_filename, 'r') as f:
- # read the spec from the build cache file. All specs
- # in build caches are concrete (as they are built) so
- # we need to mark this spec concrete on read-in.
- spec = Spec.from_yaml(f)
- spec._mark_concrete()
- _cached_specs.append(spec)
- return _cached_specs
+ return try_download_specs(urls=urls, force=force)
def get_keys(install=False, trust=False, force=False):
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 6ad1ebe1f0..d146be9af9 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -1510,7 +1510,8 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)):
def try_install_from_binary_cache(self, explicit):
tty.msg('Searching for binary cache of %s' % self.name)
- specs = binary_distribution.get_specs(use_arch=True, names=[self.name])
+ specs = binary_distribution.get_spec(spec=self.spec,
+ force=False)
binary_spec = spack.spec.Spec.from_dict(self.spec.to_dict())
binary_spec._mark_concrete()
if binary_spec not in specs: