{title}

From 075c3e0d9247e9c1716ac44e7052366ac59ff110 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Wed, 30 Sep 2020 16:37:35 -0600 Subject: Merge pull request #18359 from scottwittenburg/add-binary-distribution-cache-manager Add binary distribution cache manager --- lib/spack/spack/binary_distribution.py | 510 +++++++++++++++++++++++------- lib/spack/spack/cmd/install.py | 6 + lib/spack/spack/database.py | 1 + lib/spack/spack/installer.py | 35 +- lib/spack/spack/schema/buildcache_spec.py | 1 - lib/spack/spack/schema/config.py | 1 + lib/spack/spack/schema/spec.py | 1 + lib/spack/spack/spec.py | 3 + lib/spack/spack/test/bindist.py | 69 +++- lib/spack/spack/test/cmd/install.py | 58 ++++ lib/spack/spack/test/installer.py | 9 +- 11 files changed, 563 insertions(+), 131 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 3e70ddf792..268f4bbe0d 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -17,8 +17,9 @@ import ruamel.yaml as yaml import json -from six.moves.urllib.error import URLError +from six.moves.urllib.error import URLError, HTTPError +import llnl.util.lang import llnl.util.tty as tty from llnl.util.filesystem import mkdirp @@ -26,6 +27,7 @@ import spack.cmd import spack.config as config import spack.database as spack_db import spack.fetch_strategy as fs +import spack.util.file_cache as file_cache import spack.relocate as relocate import spack.util.gpg import spack.util.spack_json as sjson @@ -36,23 +38,296 @@ import spack.util.web as web_util from spack.spec import Spec from spack.stage import Stage + +#: default root, relative to the Spack install path +default_manager_root = os.path.join(spack.paths.opt_path, 'spack') + _build_cache_relative_path = 'build_cache' _build_cache_keys_relative_path = '_pgp' -BUILD_CACHE_INDEX_TEMPLATE = ''' - - - {title} - - -

-{path_list} - - - -''' -BUILD_CACHE_INDEX_ENTRY_TEMPLATE = '

{path}

' +class BinaryDistributionCacheManager(object): + """ + The BinaryDistributionCacheManager stores and manages both: + + 1. cached buildcache index files (the index.json describing the built + specs available on a mirror) + + 2. a cache of all the conrcrete built specs available on all the + configured mirrors. + """ + + def __init__(self, cache_root=None): + self._cache_root = cache_root or default_manager_root + self._index_cache_root = os.path.join(self._cache_root, 'indices') + self._index_contents_key = 'contents.json' + self._index_file_cache = None + self._local_index_cache = None + self._built_spec_cache = {} + + def _init_local_index_cache(self): + if not self._index_file_cache: + self._index_file_cache = file_cache.FileCache( + self._index_cache_root) + + cache_key = self._index_contents_key + self._index_file_cache.init_entry(cache_key) + + cache_path = self._index_file_cache.cache_path(cache_key) + + self._local_index_cache = {} + if os.path.isfile(cache_path): + with self._index_file_cache.read_transaction( + cache_key) as cache_file: + self._local_index_cache = json.load(cache_file) + + @property + def spec_cache(self): + """The cache of specs and mirrors they live on""" + return self._built_spec_cache + + def clear_spec_cache(self): + self._built_spec_cache = {} + + def _write_local_index_cache(self): + self._init_local_index_cache() + cache_key = self._index_contents_key + with self._index_file_cache.write_transaction(cache_key) as (old, new): + json.dump(self._local_index_cache, new) + + def regenerate_spec_cache(self): + self._init_local_index_cache() + + for mirror_url in self._local_index_cache: + cache_entry = self._local_index_cache[mirror_url] + cached_index_path = cache_entry['index_path'] + self._associate_built_specs_with_mirror(cached_index_path, + mirror_url) + + def _associate_built_specs_with_mirror(self, cache_key, mirror_url): + self._init_local_index_cache() + + tmpdir = tempfile.mkdtemp() + + try: + db_root_dir = os.path.join(tmpdir, 'db_root') + db = spack_db.Database(None, db_dir=db_root_dir, + enable_transaction_locking=False) + + self._index_file_cache.init_entry(cache_key) + cache_path = self._index_file_cache.cache_path(cache_key) + with self._index_file_cache.read_transaction(cache_key): + db._read_from_file(cache_path) + + spec_list = db.query_local(installed=False) + + for indexed_spec in spec_list: + indexed_spec_dag_hash = indexed_spec.dag_hash() + if indexed_spec_dag_hash not in self._built_spec_cache: + self._built_spec_cache[indexed_spec_dag_hash] = [] + for entry in self._built_spec_cache[indexed_spec_dag_hash]: + if entry['mirror_url'] == mirror_url: + break + else: + self._built_spec_cache[indexed_spec_dag_hash].append({ + "mirror_url": mirror_url, + "spec": indexed_spec, + }) + finally: + shutil.rmtree(tmpdir) + + def get_all_built_specs(self): + spec_list = [] + for dag_hash in self._built_spec_cache: + # in the absence of further information, all concrete specs + # with the same DAG hash are equivalent, so we can just + # return the first one in the list. + if len(self._built_spec_cache[dag_hash]) > 0: + spec_list.append(self._built_spec_cache[dag_hash][0]['spec']) + + return spec_list + + def find_built_spec(self, spec): + """Find the built spec corresponding to ``spec``. + + If the spec can be found among the configured binary mirrors, an + object is returned that contains the concrete spec and the mirror url + where it can be found. Otherwise, ``None`` is returned. + + Args: + spec (Spec): Concrete spec to find + + Returns: + An list of objects containing the found specs and mirror url where + each can be found, e.g.: + + .. code-block:: python + + [ + { + "spec": , + "mirror_url": + } + ] + """ + find_hash = spec.dag_hash() + if find_hash not in self._built_spec_cache: + return None + + return self._built_spec_cache[find_hash] + + def update_spec(self, spec, found_list): + """ + Take list of {'mirror_url': m, 'spec': s} objects and update the local + built_spec_cache + """ + spec_dag_hash = spec.dag_hash() + + if spec_dag_hash not in self._built_spec_cache: + self._built_spec_cache[spec_dag_hash] = found_list + else: + current_list = self._built_spec_cache[spec_dag_hash] + for new_entry in found_list: + for cur_entry in current_list: + if new_entry['mirror_url'] == cur_entry['mirror_url']: + cur_entry['spec'] = new_entry['spec'] + break + else: + current_list.append = { + 'mirror_url': new_entry['mirror_url'], + 'spec': new_entry['spec'], + } + + def update_local_index_cache(self): + """ Make sure local cache of buildcache index files is up to date.""" + self._init_local_index_cache() + + mirrors = spack.mirror.MirrorCollection() + configured_mirror_urls = [m.fetch_url for m in mirrors.values()] + items_to_remove = [] + + # First compare the mirror urls currently present in the cache to the + # configured mirrors. If we have a cached index for a mirror which is + # no longer configured, we should remove it from the cache. For any + # cached indices corresponding to currently configured mirrors, we need + # to check if the cache is still good, or needs to be updated. + # Finally, if there are configured mirrors for which we don't have a + # cache entry, we need to fetch and cache the indices from those + # mirrors. + + for cached_mirror_url in self._local_index_cache: + cache_entry = self._local_index_cache[cached_mirror_url] + cached_index_hash = cache_entry['index_hash'] + cached_index_path = cache_entry['index_path'] + if cached_mirror_url in configured_mirror_urls: + # May need to fetch the index and update the local caches + self.fetch_and_cache_index( + cached_mirror_url, expect_hash=cached_index_hash) + else: + # No longer have this mirror, cached index should be removed + items_to_remove.append({ + 'url': cached_mirror_url, + 'cache_key': os.path.join(self._index_cache_root, + cached_index_path) + }) + + # Clean up items to be removed, identified above + for item in items_to_remove: + url = item['url'] + cache_key = item['cache_key'] + self._index_file_cache.remove(cache_key) + del self._local_index_cache[url] + + # Iterate the configured mirrors now. Any mirror urls we do not + # already have in our cache must be fetched, stored, and represented + # locally. + for mirror_url in configured_mirror_urls: + if mirror_url not in self._local_index_cache: + # Need to fetch the index and update the local caches + self.fetch_and_cache_index(mirror_url) + + self._write_local_index_cache() + + def fetch_and_cache_index(self, mirror_url, expect_hash=None): + """ Fetch a buildcache index file from a remote mirror and cache it. + + If we already have a cached index from this mirror, then we first + check if the hash has changed, and we avoid fetching it if not. + + Args: + mirror_url (str): Base url of mirror + expect_hash (str): If provided, this hash will be compared against + the index hash we retrieve from the mirror, to determine if we + need to fetch the index or not. + """ + index_fetch_url = url_util.join( + mirror_url, _build_cache_relative_path, 'index.json') + hash_fetch_url = url_util.join( + mirror_url, _build_cache_relative_path, 'index.json.hash') + + fetched_hash = '' + + # Fetch the hash first so we can check if we actually need to fetch + # the index itself. + try: + _, _, fs = web_util.read_from_url( + hash_fetch_url, 'text/plain') + fetched_hash = codecs.getreader('utf-8')(fs).read() + except (URLError, web_util.SpackWebError) as url_err: + tty.debug('Unable to read index hash {0}'.format( + hash_fetch_url), url_err, 1) + + # If we were expecting some hash and found that we got it, we're done + if expect_hash and fetched_hash == expect_hash: + tty.debug('Cached index for {0} already up to date'.format( + mirror_url)) + return + + tty.debug('Fetching index from {0}'.format(index_fetch_url)) + + # Fetch index itself + try: + _, _, fs = web_util.read_from_url( + index_fetch_url, 'application/json') + index_object_str = codecs.getreader('utf-8')(fs).read() + except (URLError, web_util.SpackWebError) as url_err: + tty.debug('Unable to read index {0}'.format(index_fetch_url), + url_err, 1) + return + + locally_computed_hash = compute_hash(index_object_str) + + if fetched_hash is not None and locally_computed_hash != fetched_hash: + msg_tmpl = ('Computed hash ({0}) did not match remote ({1}), ' + 'indicating error in index transmission') + tty.error(msg_tmpl.format(locally_computed_hash, expect_hash)) + return + + url_hash = compute_hash(mirror_url) + + cache_key = '{0}_{1}.json'.format( + url_hash[:10], locally_computed_hash[:10]) + self._index_file_cache.init_entry(cache_key) + with self._index_file_cache.write_transaction(cache_key) as (old, new): + new.write(index_object_str) + + self._local_index_cache[mirror_url] = { + 'index_hash': locally_computed_hash, + 'index_path': cache_key, + } + + +def _cache_manager(): + """Get the singleton store instance.""" + cache_root = spack.config.get( + 'config:binary_distribution_cache_root', default_manager_root) + cache_root = spack.util.path.canonicalize_path(cache_root) + + return BinaryDistributionCacheManager(cache_root) + + +#: Singleton cache_manager instance +cache_manager = llnl.util.lang.Singleton(_cache_manager) class NoOverwriteException(spack.error.SpackError): @@ -118,6 +393,10 @@ class NewLayoutException(spack.error.SpackError): super(NewLayoutException, self).__init__(msg) +def compute_hash(data): + return hashlib.sha256(data.encode('utf-8')).hexdigest() + + def build_cache_relative_path(): return _build_cache_relative_path @@ -315,11 +594,29 @@ def generate_package_index(cache_prefix): with open(index_json_path, 'w') as f: db._write_to_file(f) + # Read the index back in and compute it's hash + with open(index_json_path) as f: + index_string = f.read() + index_hash = compute_hash(index_string) + + # Write the hash out to a local file + index_hash_path = os.path.join(db_root_dir, 'index.json.hash') + with open(index_hash_path, 'w') as f: + f.write(index_hash) + + # Push the index itself web_util.push_to_url( index_json_path, url_util.join(cache_prefix, 'index.json'), keep_original=False, extra_args={'ContentType': 'application/json'}) + + # Push the hash + web_util.push_to_url( + index_hash_path, + url_util.join(cache_prefix, 'index.json.hash'), + keep_original=False, + extra_args={'ContentType': 'text/plain'}) finally: shutil.rmtree(tmpdir) @@ -470,13 +767,6 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False, spec.prefix, spack.store.layout.root) buildinfo['relative_rpaths'] = rel spec_dict['buildinfo'] = buildinfo - spec_dict['full_hash'] = spec.full_hash() - - tty.debug('The full_hash ({0}) of {1} will be written into {2}'.format( - spec_dict['full_hash'], - spec.name, - url_util.format(remote_specfile_path))) - tty.debug(spec.tree()) with open(specfile_path, 'w') as outfile: outfile.write(syaml.dump(spec_dict)) @@ -527,10 +817,20 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False, return None -def download_tarball(spec): +def download_tarball(spec, url=None): """ Download binary tarball for given package into stage area Return True if successful + + Args: + spec (Spec): Concrete spec + url (str): If provided, this is the preferred url, other configured + mirrors will only be used if the tarball can't be retrieved from + this preferred url. + + Returns: + Path to the downloaded tarball, or ``None`` if the tarball could not + be downloaded from any configured mirrors. """ if not spack.mirror.MirrorCollection(): tty.die("Please add a spack mirror to allow " + @@ -538,12 +838,20 @@ def download_tarball(spec): tarball = tarball_path_name(spec, '.spack') + urls_to_try = [] + + if url: + urls_to_try.append(url_util.join( + url, _build_cache_relative_path, tarball)) + for mirror in spack.mirror.MirrorCollection().values(): - url = url_util.join( - mirror.fetch_url, _build_cache_relative_path, tarball) + if not url or url != mirror.fetch_url: + urls_to_try.append(url_util.join( + mirror.fetch_url, _build_cache_relative_path, tarball)) + for try_url in urls_to_try: # stage the tarball into standard place - stage = Stage(url, name="build_cache", keep=True) + stage = Stage(try_url, name="build_cache", keep=True) stage.create() try: stage.fetch() @@ -551,6 +859,8 @@ def download_tarball(spec): except fs.FetchError: continue + # If we arrive here, something went wrong, maybe this should be an + # exception? return None @@ -834,117 +1144,95 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False, os.remove(filename) -# Internal cache for downloaded specs -_cached_specs = set() - - -def try_download_specs(urls=None, force=False): - ''' - Try to download the urls and cache them - ''' - global _cached_specs - if urls is None: - return {} - for link in urls: - with Stage(link, name="build_cache", keep=True) as stage: - if force and os.path.exists(stage.save_filename): - os.remove(stage.save_filename) - if not os.path.exists(stage.save_filename): - try: - stage.fetch() - except fs.FetchError: - continue - with open(stage.save_filename, 'r') as f: - # read the spec from the build cache file. All specs - # in build caches are concrete (as they are built) so - # we need to mark this spec concrete on read-in. - spec = Spec.from_yaml(f) - spec._mark_concrete() - _cached_specs.add(spec) - - return _cached_specs - - -def get_spec(spec=None, force=False): +def try_direct_fetch(spec, force=False, full_hash_match=False): """ - Check if spec.yaml exists on mirrors and return it if it does + Try to find the spec directly on the configured mirrors """ - global _cached_specs - urls = set() - if spec is None: - return {} specfile_name = tarball_name(spec, '.spec.yaml') - - if not spack.mirror.MirrorCollection(): - tty.debug("No Spack mirrors are currently configured") - return {} - - if _cached_specs and spec in _cached_specs: - return _cached_specs + lenient = not full_hash_match + found_specs = [] for mirror in spack.mirror.MirrorCollection().values(): - fetch_url_build_cache = url_util.join( - mirror.fetch_url, _build_cache_relative_path) + buildcache_fetch_url = url_util.join( + mirror.fetch_url, _build_cache_relative_path, specfile_name) - mirror_dir = url_util.local_file_path(fetch_url_build_cache) - if mirror_dir: - tty.debug('Finding buildcaches in {0}'.format(mirror_dir)) - link = url_util.join(fetch_url_build_cache, specfile_name) - urls.add(link) + try: + _, _, fs = web_util.read_from_url( + buildcache_fetch_url, 'text/plain') + fetched_spec_yaml = codecs.getreader('utf-8')(fs).read() + except (URLError, web_util.SpackWebError, HTTPError) as url_err: + tty.debug('Did not find {0} on {1}'.format( + specfile_name, buildcache_fetch_url), url_err) + continue - else: - tty.debug('Finding buildcaches at {0}' - .format(url_util.format(fetch_url_build_cache))) - link = url_util.join(fetch_url_build_cache, specfile_name) - urls.add(link) + # read the spec from the build cache file. All specs in build caches + # are concrete (as they are built) so we need to mark this spec + # concrete on read-in. + fetched_spec = Spec.from_yaml(fetched_spec_yaml) + fetched_spec._mark_concrete() - return try_download_specs(urls=urls, force=force) + # Do not recompute the full hash for the fetched spec, instead just + # read the property. + if lenient or fetched_spec._full_hash == spec.full_hash(): + found_specs.append({ + 'mirror_url': mirror.fetch_url, + 'spec': fetched_spec, + }) + return found_specs -def get_specs(): + +def get_spec(spec=None, force=False, full_hash_match=False): """ - Get spec.yaml's for build caches available on mirror + Check if concrete spec exists on mirrors and return an object + indicating the mirrors on which it can be found """ - global _cached_specs + if spec is None: + return [] if not spack.mirror.MirrorCollection(): tty.debug("No Spack mirrors are currently configured") return {} - for mirror in spack.mirror.MirrorCollection().values(): - fetch_url_build_cache = url_util.join( - mirror.fetch_url, _build_cache_relative_path) + results = [] + lenient = not full_hash_match - tty.debug('Finding buildcaches at {0}' - .format(url_util.format(fetch_url_build_cache))) + def filter_candidates(possibles): + filtered_candidates = [] + for candidate in possibles: + spec_full_hash = spec.full_hash() + candidate_full_hash = candidate['spec']._full_hash + if lenient or spec_full_hash == candidate_full_hash: + filtered_candidates.append(candidate) + return filtered_candidates - index_url = url_util.join(fetch_url_build_cache, 'index.json') + candidates = cache_manager.find_built_spec(spec) + if candidates: + results = filter_candidates(candidates) - try: - _, _, file_stream = web_util.read_from_url( - index_url, 'application/json') - index_object = codecs.getreader('utf-8')(file_stream).read() - except (URLError, web_util.SpackWebError) as url_err: - tty.debug('Failed to read index {0}'.format(index_url), url_err, 1) - # Continue on to the next mirror - continue + # Maybe we just didn't have the latest information from the mirror, so + # try to fetch directly. + if not results: + results = try_direct_fetch(spec, + force=force, + full_hash_match=full_hash_match) + if results: + cache_manager.update_spec(spec, results) - tmpdir = tempfile.mkdtemp() - index_file_path = os.path.join(tmpdir, 'index.json') - with open(index_file_path, 'w') as fd: - fd.write(index_object) + return results - db_root_dir = os.path.join(tmpdir, 'db_root') - db = spack_db.Database(None, db_dir=db_root_dir, - enable_transaction_locking=False) - db._read_from_file(index_file_path) - spec_list = db.query_local(installed=False) +def get_specs(): + """ + Get concrete specs for build caches available on mirrors + """ + cache_manager.update_local_index_cache() + cache_manager.regenerate_spec_cache() + return cache_manager.get_all_built_specs() - for indexed_spec in spec_list: - _cached_specs.add(indexed_spec) - return _cached_specs +def clear_spec_cache(): + cache_manager.clear_spec_cache() def get_keys(install=False, trust=False, force=False, mirrors=None): diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 69158275f6..e9d9416d3e 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -45,6 +45,7 @@ def update_kwargs_from_args(args, kwargs): 'explicit': True, # Always true for install command 'stop_at': args.until, 'unsigned': args.unsigned, + 'full_hash_match': args.full_hash_match, }) kwargs.update({ @@ -107,6 +108,11 @@ the dependencies""" '--no-check-signature', action='store_true', dest='unsigned', default=False, help="do not check signatures of binary packages") + subparser.add_argument( + '--require-full-hash-match', action='store_true', + dest='full_hash_match', default=False, help="""when installing from +binary mirrors, do not install binary package unless the full hash of the +remote spec matches that of the local spec""") subparser.add_argument( '--show-log-on-error', action='store_true', help="print full build log to stderr if build fails") diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 01fb4fc7a5..090544dc86 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -1112,6 +1112,7 @@ class Database(object): # the original hash of concrete specs. new_spec._mark_concrete() new_spec._hash = key + new_spec._full_hash = spec._full_hash else: # If it is already there, mark it as installed. diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index 401e2428b3..ffb2b679c6 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -213,7 +213,8 @@ def _hms(seconds): return ' '.join(parts) -def _install_from_cache(pkg, cache_only, explicit, unsigned=False): +def _install_from_cache(pkg, cache_only, explicit, unsigned=False, + full_hash_match=False): """ Extract the package from binary cache @@ -229,8 +230,8 @@ def _install_from_cache(pkg, cache_only, explicit, unsigned=False): (bool) ``True`` if the package was extract from binary cache, ``False`` otherwise """ - installed_from_cache = _try_install_from_binary_cache(pkg, explicit, - unsigned) + installed_from_cache = _try_install_from_binary_cache( + pkg, explicit, unsigned=unsigned, full_hash_match=full_hash_match) pkg_id = package_id(pkg) if not installed_from_cache: pre = 'No binary for {0} found'.format(pkg_id) @@ -302,7 +303,8 @@ def _process_external_package(pkg, explicit): spack.store.db.add(spec, None, explicit=explicit) -def _process_binary_cache_tarball(pkg, binary_spec, explicit, unsigned): +def _process_binary_cache_tarball(pkg, binary_spec, explicit, unsigned, + preferred_mirror_url=None): """ Process the binary cache tarball. @@ -317,7 +319,8 @@ def _process_binary_cache_tarball(pkg, binary_spec, explicit, unsigned): (bool) ``True`` if the package was extracted from binary cache, else ``False`` """ - tarball = binary_distribution.download_tarball(binary_spec) + tarball = binary_distribution.download_tarball(binary_spec, + url=preferred_mirror_url) # see #10063 : install from source if tarball doesn't exist if tarball is None: tty.msg('{0} exists in binary cache but with different hash' @@ -333,7 +336,8 @@ def _process_binary_cache_tarball(pkg, binary_spec, explicit, unsigned): return True -def _try_install_from_binary_cache(pkg, explicit, unsigned=False): +def _try_install_from_binary_cache(pkg, explicit, unsigned=False, + full_hash_match=False): """ Try to extract the package from binary cache. @@ -345,13 +349,18 @@ def _try_install_from_binary_cache(pkg, explicit, unsigned=False): """ pkg_id = package_id(pkg) tty.debug('Searching for binary cache of {0}'.format(pkg_id)) - specs = binary_distribution.get_spec(pkg.spec, force=False) - binary_spec = spack.spec.Spec.from_dict(pkg.spec.to_dict()) - binary_spec._mark_concrete() - if binary_spec not in specs: + matches = binary_distribution.get_spec( + pkg.spec, force=False, full_hash_match=full_hash_match) + + if not matches: return False - return _process_binary_cache_tarball(pkg, binary_spec, explicit, unsigned) + # In the absence of guidance from user or some other reason to prefer one + # mirror over another, any match will suffice, so just pick the first one. + preferred_mirror = matches[0]['mirror_url'] + binary_spec = matches[0]['spec'] + return _process_binary_cache_tarball(pkg, binary_spec, explicit, unsigned, + preferred_mirror_url=preferred_mirror) def _update_explicit_entry_in_db(pkg, rec, explicit): @@ -1044,6 +1053,7 @@ class PackageInstaller(object): unsigned = kwargs.get('unsigned', False) use_cache = kwargs.get('use_cache', True) verbose = kwargs.get('verbose', False) + full_hash_match = kwargs.get('full_hash_match', False) pkg = task.pkg pkg_id = package_id(pkg) @@ -1055,7 +1065,8 @@ class PackageInstaller(object): # Use the binary cache if requested if use_cache and \ - _install_from_cache(pkg, cache_only, explicit, unsigned): + _install_from_cache(pkg, cache_only, explicit, unsigned, + full_hash_match): self._update_installed(task) if task.compiler: spack.compilers.add_compilers_to_config( diff --git a/lib/spack/spack/schema/buildcache_spec.py b/lib/spack/spack/schema/buildcache_spec.py index 69eae2fafc..5ba07a27f1 100644 --- a/lib/spack/spack/schema/buildcache_spec.py +++ b/lib/spack/spack/schema/buildcache_spec.py @@ -26,7 +26,6 @@ schema = { 'relative_rpaths': {'type': 'boolean'}, }, }, - 'full_hash': {'type': 'string'}, 'spec': { 'type': 'array', 'items': spack.schema.spec.properties, diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index 7315300aa6..0c12ea19b8 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -87,6 +87,7 @@ properties = { ], }, 'allow_sgid': {'type': 'boolean'}, + 'binary_distribution_cache_root': {'type': 'string'}, }, }, } diff --git a/lib/spack/spack/schema/spec.py b/lib/spack/spack/schema/spec.py index 8ec17e7e76..0fe999e045 100644 --- a/lib/spack/spack/schema/spec.py +++ b/lib/spack/spack/schema/spec.py @@ -81,6 +81,7 @@ properties = { ], 'properties': { 'hash': {'type': 'string'}, + 'full_hash': {'type': 'string'}, 'version': { 'oneOf': [ {'type': 'string'}, diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 0b4e7dfacf..eb5e8f9369 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1552,6 +1552,8 @@ class Spec(object): if not self._concrete: d['concrete'] = False + else: + d['full_hash'] = self._full_hash if 'patches' in self.variants: variant = self.variants['patches'] @@ -1734,6 +1736,7 @@ class Spec(object): # specs read in are concrete unless marked abstract spec._concrete = node.get('concrete', True) + spec._full_hash = node.get('full_hash', None) if 'patches' in node: patches = node['patches'] diff --git a/lib/spack/spack/test/bindist.py b/lib/spack/spack/test/bindist.py index 9b2f1b1408..142b4a72a5 100644 --- a/lib/spack/spack/test/bindist.py +++ b/lib/spack/spack/test/bindist.py @@ -302,7 +302,7 @@ def test_default_rpaths_install_nondefault_layout(tmpdir, args = parser.parse_args(install_args) buildcache.buildcache(parser, args) - bindist._cached_specs = set() + bindist.clear_spec_cache() spack.stage.purge() margs = mparser.parse_args( ['rm', '--scope', 'site', 'test-mirror-def']) @@ -358,7 +358,7 @@ def test_relative_rpaths_create_default_layout(tmpdir, uargs = uparser.parse_args(['-y', '--dependents', gspec.name]) uninstall.uninstall(uparser, uargs) - bindist._cached_specs = set() + bindist.clear_spec_cache() spack.stage.purge() @@ -395,7 +395,7 @@ def test_relative_rpaths_install_default_layout(tmpdir, buildcache.setup_parser(parser) # set default buildcache args - install_args = ['install', '-a', '-u', + install_args = ['install', '-a', '-u', '-f', cspec.name] # install buildcache created with relativized rpaths @@ -460,7 +460,7 @@ def test_relative_rpaths_install_nondefault(tmpdir, buildcache.setup_parser(parser) # Set default buildcache args - install_args = ['install', '-a', '-u', '%s' % cspec.name] + install_args = ['install', '-a', '-u', '-f', '%s' % cspec.name] # test install in non-default install path scheme and relative path args = parser.parse_args(install_args) @@ -508,3 +508,64 @@ def test_push_and_fetch_keys(mock_gnupghome): new_keys = spack.util.gpg.public_keys() assert len(new_keys) == 1 assert new_keys[0] == fpr + + +@pytest.mark.requires_executables(*args) +@pytest.mark.disable_clean_stage_check +@pytest.mark.maybeslow +@pytest.mark.nomockstage +@pytest.mark.usefixtures('default_config', 'cache_directory', + 'install_dir_non_default_layout') +def test_built_spec_cache(tmpdir, + install_mockery): + """ Test what's the situation now """ + global mirror_path_rel + + mparser = argparse.ArgumentParser() + mirror.setup_parser(mparser) + margs = mparser.parse_args( + ['add', '--scope', 'site', 'test-mirror-rel', 'file://%s' % mirror_path_rel]) + mirror.mirror(mparser, margs) + + # setup argument parser + parser = argparse.ArgumentParser() + buildcache.setup_parser(parser) + + list_args = ['list', '-a', '-l'] + args = parser.parse_args(list_args) + buildcache.buildcache(parser, args) + + gspec = Spec('garply') + gspec.concretize() + + cspec = Spec('corge') + cspec.concretize() + + full_hash_map = { + 'garply': gspec.full_hash(), + 'corge': cspec.full_hash(), + } + + gspec_results = bindist.get_spec(gspec) + + gspec_mirrors = {} + for result in gspec_results: + s = result['spec'] + assert(s._full_hash == full_hash_map[s.name]) + assert(result['mirror_url'] not in gspec_mirrors) + gspec_mirrors[result['mirror_url']] = True + + cspec_results = bindist.get_spec(cspec, full_hash_match=True) + + cspec_mirrors = {} + for result in cspec_results: + s = result['spec'] + assert(s._full_hash == full_hash_map[s.name]) + assert(result['mirror_url'] not in cspec_mirrors) + cspec_mirrors[result['mirror_url']] = True + + bindist.clear_spec_cache() + + margs = mparser.parse_args( + ['rm', '--scope', 'site', 'test-mirror-rel']) + mirror.mirror(mparser, margs) diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py index 9eb2338649..4af3e8a339 100644 --- a/lib/spack/spack/test/cmd/install.py +++ b/lib/spack/spack/test/cmd/install.py @@ -809,3 +809,61 @@ def test_install_fails_no_args_suggests_env_activation(tmpdir): assert 'requires a package argument or active environment' in output assert 'spack env activate .' in output assert 'using the `spack.yaml` in this directory' in output + + +def test_cache_install_full_hash_match( + install_mockery_mutable_config, mock_packages, mock_fetch, + mock_archive, mutable_config, monkeypatch, tmpdir): + """Make sure installing from cache respects full hash argument""" + + # Create a temp mirror directory for buildcache usage + mirror_dir = tmpdir.join('mirror_dir') + mirror_url = 'file://{0}'.format(mirror_dir.strpath) + + s = Spec('libdwarf').concretized() + + # Install a package + install(s.name) + + # Put installed package in the buildcache + buildcache('create', '-u', '-a', '-f', '-d', mirror_dir.strpath, s.name) + + # Now uninstall the package + uninstall('-y', s.name) + + # Configure the mirror with the binary package in it + mirror('add', 'test-mirror', mirror_url) + + # Make sure we get the binary version by default + install_output = install('--no-check-signature', s.name, output=str) + expect_msg = 'Extracting {0} from binary cache'.format(s.name) + + assert expect_msg in install_output + + uninstall('-y', s.name) + + # Now monkey patch Spec to change the full hash on the package + def fake_full_hash(spec): + print('fake_full_hash') + return 'tal4c7h4z0gqmixb1eqa92mjoybxn5l6' + monkeypatch.setattr(spack.spec.Spec, 'full_hash', fake_full_hash) + + # Check that even if the full hash changes, we install from binary when + # we don't explicitly require the full hash to match + install_output = install('--no-check-signature', s.name, output=str) + expect_msg = 'Extracting {0} from binary cache'.format(s.name) + + assert expect_msg in install_output + + uninstall('-y', s.name) + + # Finally, make sure that if we insist on the full hash match, spack + # installs from source. + install_output = install('--require-full-hash-match', s.name, output=str) + expect_msg = 'No binary for {0} found: installing from source'.format( + s.name) + + assert expect_msg in install_output + + uninstall('-y', s.name) + mirror('rm', 'test-mirror') diff --git a/lib/spack/spack/test/installer.py b/lib/spack/spack/test/installer.py index 0b3f409641..700e5d490e 100644 --- a/lib/spack/spack/test/installer.py +++ b/lib/spack/spack/test/installer.py @@ -177,7 +177,7 @@ def test_process_binary_cache_tarball_none(install_mockery, monkeypatch, def test_process_binary_cache_tarball_tar(install_mockery, monkeypatch, capfd): """Tests of _process_binary_cache_tarball with a tar file.""" - def _spec(spec): + def _spec(spec, url=None): return spec # Skip binary distribution functionality since assume tested elsewhere @@ -196,9 +196,12 @@ def test_process_binary_cache_tarball_tar(install_mockery, monkeypatch, capfd): def test_try_install_from_binary_cache(install_mockery, mock_packages, monkeypatch, capsys): """Tests SystemExit path for_try_install_from_binary_cache.""" - def _spec(spec, force): + def _spec(spec, force, full_hash_match=False): spec = spack.spec.Spec('mpi').concretized() - return {spec: None} + return [{ + 'mirror_url': 'notused', + 'spec': spec, + }] spec = spack.spec.Spec('mpich') spec.concretize() -- cgit v1.2.3-60-g2f50