From ed501eaab275f50dc3d689ee9e8d6c7fa707fd94 Mon Sep 17 00:00:00 2001
From: Patrick Gartung <gartung@fnal.gov>
Date: Thu, 30 Jan 2020 16:06:50 -0600
Subject: Bypass build_cache/index.html read when trying to download spec.yaml
 for concretized spec. (#14698)

* Add binary_distribution::get_spec which takes concretized spec
Add binary_distribution::try_download_specs for downloading of spec.yaml files to cache
get_spec is used by package::try_install_from_binary_cache to download only the spec.yaml
for the concretized spec if it exists.
---
 lib/spack/spack/binary_distribution.py | 86 ++++++++++++++++++++++++++--------
 lib/spack/spack/package.py             |  3 +-
 2 files changed, 68 insertions(+), 21 deletions(-)

diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py
index b898f27f49..3e5dc89313 100644
--- a/lib/spack/spack/binary_distribution.py
+++ b/lib/spack/spack/binary_distribution.py
@@ -660,16 +660,79 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
         shutil.rmtree(tmpdir)
 
 
-#: Internal cache for get_specs
+# Internal cache for downloaded specs
 _cached_specs = None
 
 
+def try_download_specs(urls=None, force=False):
+    '''
+    Try to download the urls and cache them
+    '''
+    global _cached_specs
+    _cached_specs = []
+    if urls is None:
+        return {}
+    for link in urls:
+        with Stage(link, name="build_cache", keep=True) as stage:
+            if force and os.path.exists(stage.save_filename):
+                os.remove(stage.save_filename)
+            if not os.path.exists(stage.save_filename):
+                try:
+                    stage.fetch()
+                except fs.FetchError:
+                    continue
+            with open(stage.save_filename, 'r') as f:
+                # read the spec from the build cache file. All specs
+                # in build caches are concrete (as they are built) so
+                # we need to mark this spec concrete on read-in.
+                spec = Spec.from_yaml(f)
+                spec._mark_concrete()
+                _cached_specs.append(spec)
+
+    return _cached_specs
+
+
+def get_spec(spec=None, force=False):
+    """
+    Check if spec.yaml exists on mirrors and return it if it does
+    """
+    global _cached_specs
+    urls = set()
+    if spec is None:
+        return {}
+    specfile_name = tarball_name(spec, '.spec.yaml')
+    if _cached_specs:
+        tty.debug("Using previously-retrieved specs")
+        return _cached_specs
+
+    if not spack.mirror.MirrorCollection():
+        tty.debug("No Spack mirrors are currently configured")
+        return {}
+
+    for mirror in spack.mirror.MirrorCollection().values():
+        fetch_url_build_cache = url_util.join(
+            mirror.fetch_url, _build_cache_relative_path)
+
+        mirror_dir = url_util.local_file_path(fetch_url_build_cache)
+        if mirror_dir:
+            tty.msg("Finding buildcaches in %s" % mirror_dir)
+            link = url_util.join(fetch_url_build_cache, specfile_name)
+            urls.add(link)
+
+        else:
+            tty.msg("Finding buildcaches at %s" %
+                    url_util.format(fetch_url_build_cache))
+            link = url_util.join(fetch_url_build_cache, specfile_name)
+            urls.add(link)
+
+    return try_download_specs(urls=urls, force=force)
+
+
 def get_specs(force=False, use_arch=False, names=None):
     """
     Get spec.yaml's for build caches available on mirror
     """
     global _cached_specs
-
     arch = architecture.Arch(architecture.platform(),
                              'default_os', 'default_target')
     arch_pattern = ('([^-]*-[^-]*-[^-]*)')
@@ -716,25 +779,8 @@ def get_specs(force=False, use_arch=False, names=None):
                 m = name_re.search(link)
                 if m:
                     urls.add(link)
-    _cached_specs = []
-    for link in urls:
-        with Stage(link, name="build_cache", keep=True) as stage:
-            if force and os.path.exists(stage.save_filename):
-                os.remove(stage.save_filename)
-            if not os.path.exists(stage.save_filename):
-                try:
-                    stage.fetch()
-                except fs.FetchError:
-                    continue
-            with open(stage.save_filename, 'r') as f:
-                # read the spec from the build cache file. All specs
-                # in build caches are concrete (as they are built) so
-                # we need to mark this spec concrete on read-in.
-                spec = Spec.from_yaml(f)
-                spec._mark_concrete()
-                _cached_specs.append(spec)
 
-    return _cached_specs
+    return try_download_specs(urls=urls, force=force)
 
 
 def get_keys(install=False, trust=False, force=False):
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 6ad1ebe1f0..d146be9af9 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -1510,7 +1510,8 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)):
 
     def try_install_from_binary_cache(self, explicit):
         tty.msg('Searching for binary cache of %s' % self.name)
-        specs = binary_distribution.get_specs(use_arch=True, names=[self.name])
+        specs = binary_distribution.get_spec(spec=self.spec,
+                                             force=False)
         binary_spec = spack.spec.Spec.from_dict(self.spec.to_dict())
         binary_spec._mark_concrete()
         if binary_spec not in specs:
-- 
cgit v1.2.3-70-g09d2