From 50bd4d2e4ed3fed83afa30e7796eaa99989bc3ce Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 2 Dec 2015 16:18:25 +0100 Subject: mirror : deals correctly with variants that optionally enable resources (if they are archive URLs) --- lib/spack/spack/mirror.py | 81 +++++++++++++++++++++++++++++++++------------- lib/spack/spack/package.py | 19 +++++++---- 2 files changed, 72 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 306c8085aa..1c38d79164 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -26,7 +26,7 @@ This file contains code for creating spack mirror directories. A mirror is an organized hierarchy containing specially named archive files. This enabled spack to know where to find files in a mirror if -the main server for a particualr package is down. Or, if the computer +the main server for a particular package is down. Or, if the computer where spack is run is not connected to the internet, it allows spack to download packages directly from a mirror (e.g., on an intranet). """ @@ -42,7 +42,7 @@ import spack.fetch_strategy as fs from spack.spec import Spec from spack.stage import Stage from spack.version import * -from spack.util.compression import extension +from spack.util.compression import extension, allowed_archive def mirror_archive_filename(spec): @@ -87,11 +87,26 @@ def get_matching_versions(specs, **kwargs): if v.satisfies(spec.versions): s = Spec(pkg.name) s.versions = VersionList([v]) + s.variants = spec.variants.copy() matching.append(s) return matching +def suggest_archive_basename(resource): + """ + Return a tentative basename for an archive. Raise an exception if the name is among the allowed archive types. + + :param fetcher: + :return: + """ + basename = os.path.basename(resource.fetcher.url) + if not allowed_archive(basename): + raise RuntimeError("%s is not an allowed archive tye" % basename) + return basename + + + def create(path, specs, **kwargs): """Create a directory to be used as a spack mirror, and fill it with package archives. @@ -108,7 +123,7 @@ def create(path, specs, **kwargs): Return Value: Returns a tuple of lists: (present, mirrored, error) - * present: Package specs that were already prsent. + * present: Package specs that were already present. * mirrored: Package specs that were successfully mirrored. * error: Package specs that failed to mirror due to some error. @@ -140,6 +155,7 @@ def create(path, specs, **kwargs): error = [] # Iterate through packages and download all the safe tarballs for each of them + everything_already_exists = True for spec in version_specs: pkg = spec.package @@ -152,26 +168,47 @@ def create(path, specs, **kwargs): if os.path.exists(archive_path): tty.msg("Already added %s" % spec.format("$_$@")) + else: + everything_already_exists = False + # Set up a stage and a fetcher for the download + unique_fetch_name = spec.format("$_$@") + fetcher = fs.for_package_version(pkg, pkg.version) + stage = Stage(fetcher, name=unique_fetch_name) + fetcher.set_stage(stage) + + # Do the fetch and checksum if necessary + fetcher.fetch() + if not kwargs.get('no_checksum', False): + fetcher.check() + tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version)) + + # Fetchers have to know how to archive their files. Use + # that to move/copy/create an archive in the mirror. + fetcher.archive(archive_path) + tty.msg("Added %s." % spec.format("$_$@")) + + # Fetch resources if they are associated with the spec + resources = pkg._get_resources() + for resource in resources: + resource_archive_path = join_path(subdir, suggest_archive_basename(resource)) + if os.path.exists(resource_archive_path): + tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version)) + continue + everything_already_exists = False + resource_stage_folder = pkg._resource_stage(resource) + resource_stage = Stage(resource.fetcher, name=resource_stage_folder) + resource.fetcher.set_stage(resource_stage) + resource.fetcher.fetch() + if not kwargs.get('no_checksum', False): + resource.fetcher.check() + tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version)) + resource.fetcher.archive(resource_archive_path) + tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version)) + + if everything_already_exists: present.append(spec) - continue - - # Set up a stage and a fetcher for the download - unique_fetch_name = spec.format("$_$@") - fetcher = fs.for_package_version(pkg, pkg.version) - stage = Stage(fetcher, name=unique_fetch_name) - fetcher.set_stage(stage) - - # Do the fetch and checksum if necessary - fetcher.fetch() - if not kwargs.get('no_checksum', False): - fetcher.check() - tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version)) - - # Fetchers have to know how to archive their files. Use - # that to move/copy/create an archive in the mirror. - fetcher.archive(archive_path) - tty.msg("Added %s." % spec.format("$_$@")) - mirrored.append(spec) + else: + mirrored.append(spec) except Exception, e: if spack.debug: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 3f48231c75..dcb514af00 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -644,12 +644,14 @@ class Package(object): # Fetch resources resources = self._get_resources() for resource in resources: - pieces = ['resource', resource.name, self.spec.dag_hash()] - resource_stage_folder = '-'.join(pieces) - stage = Stage(resource.fetcher, name=resource_stage_folder) - resource.fetcher.set_stage(stage) - resource.fetcher.fetch() - resource.fetcher.check() + resource_stage_folder = self._resource_stage(resource) + # FIXME : works only for URLFetchStrategy + resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url)) + resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror) + resource.fetcher.set_stage(resource_stage) + # Delegate to stage object to trigger mirror logic + resource_stage.fetch() + resource_stage.check() ########## self._fetch_time = time.time() - start_time @@ -766,6 +768,11 @@ class Package(object): resources.extend(resource_list) return resources + def _resource_stage(self, resource): + pieces = ['resource', resource.name, self.spec.dag_hash()] + resource_stage_folder = '-'.join(pieces) + return resource_stage_folder + def _build_logger(self, log_path): """Create a context manager to log build output.""" -- cgit v1.2.3-60-g2f50