diff options
| -rw-r--r-- | lib/spack/spack/__init__.py | 3 | ||||
| -rw-r--r-- | lib/spack/spack/binary_distribution.py | 53 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/buildcache.py | 7 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/install.py | 6 | ||||
| -rw-r--r-- | lib/spack/spack/package.py | 23 |
5 files changed, 62 insertions, 30 deletions
diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index f7fa9ca836..6ffd5f838c 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -134,6 +134,9 @@ misc_cache_path = canonicalize_path( misc_cache = FileCache(misc_cache_path) +binary_cache_retrieved_specs = set() + + #: Directories where to search for templates template_dirs = spack.config.get_config('config')['template_dirs'] template_dirs = [canonicalize_path(x) for x in template_dirs] diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index a8bcaf5b1f..da835983c9 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -417,15 +417,18 @@ def get_specs(force=False): """ Get spec.yaml's for build caches available on mirror """ + if spack.binary_cache_retrieved_specs: + tty.debug("Using previously-retrieved specs") + previously_retrieved = spack.binary_cache_retrieved_specs + return previously_retrieved + mirrors = spack.config.get_config('mirrors') if len(mirrors) == 0: - tty.die("Please add a spack mirror to allow " + - "download of build caches.") + tty.warn("No Spack mirrors are currently configured") + return {} + path = str(spack.architecture.sys_type()) - specs = set() urls = set() - from collections import defaultdict - durls = defaultdict(list) for key in mirrors: url = mirrors[key] if url.startswith('file'): @@ -442,25 +445,27 @@ def get_specs(force=False): for link in links: if re.search("spec.yaml", link) and re.search(path, link): urls.add(link) - for link in urls: - with Stage(link, name="build_cache", keep=True) as stage: - if force and os.path.exists(stage.save_filename): - os.remove(stage.save_filename) - if not os.path.exists(stage.save_filename): - try: - stage.fetch() - except fs.FetchError: - continue - with open(stage.save_filename, 'r') as f: - # read the spec from the build cache file. All specs - # in build caches are concrete (as they aer built) so - # we need to mark this spec concrete on read-in. - spec = spack.spec.Spec.from_yaml(f) - spec._mark_concrete() - - specs.add(spec) - durls[spec].append(link) - return specs, durls + + specs = set() + for link in urls: + with Stage(link, name="build_cache", keep=True) as stage: + if force and os.path.exists(stage.save_filename): + os.remove(stage.save_filename) + if not os.path.exists(stage.save_filename): + try: + stage.fetch() + except fs.FetchError: + continue + with open(stage.save_filename, 'r') as f: + # read the spec from the build cache file. All specs + # in build caches are concrete (as they are built) so + # we need to mark this spec concrete on read-in. + spec = spack.spec.Spec.from_yaml(f) + spec._mark_concrete() + specs.add(spec) + + spack.binary_cache_retrieved_specs = specs + return specs def get_keys(install=False, yes_to_all=False, force=False): diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py index 3d3396d8b7..94070c4833 100644 --- a/lib/spack/spack/cmd/buildcache.py +++ b/lib/spack/spack/cmd/buildcache.py @@ -146,7 +146,7 @@ def match_downloaded_specs(pkgs, allow_multiple_matches=False, force=False): # List of specs that match expressions given via command line specs_from_cli = [] has_errors = False - specs, links = bindist.get_specs(force) + specs = bindist.get_specs(force) for pkg in pkgs: matches = [] tty.msg("buildcache spec(s) matching %s \n" % pkg) @@ -296,10 +296,7 @@ def install_tarball(spec, args): def listspecs(args): - force = False - if args.force: - force = True - specs, links = bindist.get_specs(force) + specs = bindist.get_specs(args.force) if args.packages: pkgs = set(args.packages) for pkg in pkgs: diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index f9b2e69d00..37000d7176 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -74,6 +74,9 @@ the dependencies""" '--restage', action='store_true', help="if a partial install is detected, delete prior state") subparser.add_argument( + '--use-cache', action='store_true', dest='use_cache', + help="check for pre-built Spack packages in mirrors") + subparser.add_argument( '--show-log-on-error', action='store_true', help="print full build log to stderr if build fails") subparser.add_argument( @@ -395,7 +398,8 @@ def install(parser, args, **kwargs): 'make_jobs': args.jobs, 'verbose': args.verbose, 'fake': args.fake, - 'dirty': args.dirty + 'dirty': args.dirty, + 'use_cache': args.use_cache }) if args.run_tests: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index caf47afc95..2081046a0d 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -60,6 +60,7 @@ import spack.repository import spack.url import spack.util.web import spack.multimethod +import spack.binary_distribution as binary_distribution from llnl.util.filesystem import mkdirp, join_path, touch, ancestor from llnl.util.filesystem import working_dir, install_tree, install @@ -1266,6 +1267,18 @@ class PackageBase(with_metaclass(PackageMeta, object)): message = '{s.name}@{s.version} : marking the package explicit' tty.msg(message.format(s=self)) + def try_install_from_binary_cache(self, explicit): + tty.msg('Searching for binary cache of %s' % self.name) + specs = binary_distribution.get_specs() + if self.spec not in specs: + return False + tty.msg('Installing %s from binary cache' % self.name) + tarball = binary_distribution.download_tarball(self.spec) + binary_distribution.extract_tarball( + self.spec, tarball, yes_to_all=False, force=False) + spack.store.db.add(self.spec, spack.store.layout, explicit=explicit) + return True + def do_install(self, keep_prefix=False, keep_stage=False, @@ -1349,6 +1362,16 @@ class PackageBase(with_metaclass(PackageMeta, object)): tty.msg(colorize('@*{Installing} @*g{%s}' % self.name)) + if kwargs.get('use_cache', False): + if self.try_install_from_binary_cache(explicit): + tty.msg('Successfully installed %s from binary cache' + % self.name) + print_pkg(self.prefix) + return + + tty.msg('No binary for %s found: installing from source' + % self.name) + # Set run_tests flag before starting build. self.run_tests = spack.package_testing.check(self.name) |