diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/spack/fetch_strategy.py | 52 | ||||
| -rw-r--r-- | lib/spack/spack/mirror.py | 20 | ||||
| -rw-r--r-- | lib/spack/spack/package.py | 11 | ||||
| -rw-r--r-- | lib/spack/spack/stage.py | 34 |
4 files changed, 73 insertions, 44 deletions
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 4374587250..d1a817d82f 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -116,6 +116,14 @@ class FetchStrategy(object): def archive(self, destination): pass # Used to create tarball for mirror. + @property + def cachable(self): + """Return whether the fetcher is capable of caching the + resource it retrieves. This generally is determined by + whether the resource is identifiably associated with a + specific package version.""" + pass + def __str__(self): # Should be human readable URL. return "FetchStrategy.__str___" @@ -272,6 +280,10 @@ class URLFetchStrategy(FetchStrategy): """Path to the source archive within this stage directory.""" return self.stage.archive_file + @property + def cachable(self): + return bool(self.digest) + @_needs_stage def expand(self): if not self.expand_archive: @@ -555,6 +567,10 @@ class GitFetchStrategy(VCSFetchStrategy): return self._git + @property + def cachable(self): + return bool(self.commit or self.tag) + @_needs_stage def fetch(self): self.stage.chdir() @@ -671,6 +687,10 @@ class SvnFetchStrategy(VCSFetchStrategy): self._svn = which('svn', required=True) return self._svn + @property + def cachable(self): + return bool(self.revision) + @_needs_stage def fetch(self): self.stage.chdir() @@ -754,6 +774,10 @@ class HgFetchStrategy(VCSFetchStrategy): self._hg = which('hg', required=True) return self._hg + @property + def cachable(self): + return bool(self.revision) + @_needs_stage def fetch(self): self.stage.chdir() @@ -860,22 +884,30 @@ def for_package_version(pkg, version): raise InvalidArgsError(pkg, version) +def from_list_url(pkg): + """If a package provides a URL which lists URLs for resources by + version, this can can create a fetcher for a URL discovered for + the specified package's version.""" + if pkg.list_url: + try: + versions = pkg.fetch_remote_versions() + try: + url_from_list = versions[pkg.version] + return URLFetchStrategy(url=url_from_list, digest=None) + except KeyError: + tty.msg("Can not find version %s in url_list" % + self.version) + except: + tty.msg("Could not determine url from list_url.") + + class FsCache(object): def __init__(self, root): self.root = os.path.abspath(root) def store(self, fetcher, relativeDst): - unique = False - uidGroups = [['tag', 'commit'], ['digest'], ['revision']] - for grp in uidGroups: - try: - unique |= any(getattr(fetcher, x) for x in grp) - except AttributeError: - pass - if unique: - break - if not unique: + if not fetcher.cachable: return dst = join_path(self.root, relativeDst) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 0f72e4e25c..97aeef5434 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -44,7 +44,7 @@ from spack.version import * from spack.util.compression import allowed_archive -def mirror_archive_filename(spec, fetcher): +def mirror_archive_filename(spec, fetcher, resourceId=None): """Get the name of the spec's archive in the mirror.""" if not spec.version.concrete: raise ValueError("mirror.path requires spec with concrete version.") @@ -67,15 +67,18 @@ def mirror_archive_filename(spec, fetcher): # Otherwise we'll make a .tar.gz ourselves ext = 'tar.gz' - filename = "%s-%s" % (spec.package.name, spec.version) - if ext: - filename += ".%s" % ext + if resourceId: + filename = "%s-%s" % (resourceId, spec.version) + ".%s" % ext + else: + filename = "%s-%s" % (spec.package.name, spec.version) + ".%s" % ext + return filename -def mirror_archive_path(spec, fetcher): +def mirror_archive_path(spec, fetcher, resourceId=None): """Get the relative path to the spec's archive within a mirror.""" - return join_path(spec.name, mirror_archive_filename(spec, fetcher)) + return join_path( + spec.name, mirror_archive_filename(spec, fetcher, resourceId)) def get_matching_versions(specs, **kwargs): @@ -204,8 +207,9 @@ def add_single_spec(spec, mirror_root, categories, **kwargs): name = spec.format("$_$@") else: resource = stage.resource - archive_path = join_path( - subdir, suggest_archive_basename(resource)) + archive_path = os.path.abspath(join_path( + mirror_root, + mirror_archive_path(spec, fetcher, resource.name))) name = "{resource} ({pkg}).".format( resource=resource.name, pkg=spec.format("$_$@")) subdir = os.path.dirname(archive_path) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 0d3c3c0ad5..df40d34d33 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -687,7 +687,8 @@ class PackageBase(object): def _make_resource_stage(self, root_stage, fetcher, resource): resource_stage_folder = self._resource_stage(resource) - resource_mirror = join_path(self.name, os.path.basename(fetcher.url)) + resource_mirror = spack.mirror.mirror_archive_path( + self.spec, fetcher, resource.name) stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource, @@ -702,8 +703,12 @@ class PackageBase(object): # Construct a path where the stage should build.. s = self.spec stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash()) - # Build the composite stage - stage = Stage(fetcher, mirror_path=mp, name=stage_name, path=self.path) + + # Check list_url alternative archive URLs + dynamic_fetcher = fs.from_list_url(self) + alternate_fetchers = [dynamic_fetcher] if dynamic_fetcher else None + stage = Stage(fetcher, mirror_path=mp, name=stage_name, path=self.path, + alternate_fetchers=alternate_fetchers) return stage def _make_stage(self): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index ff10a38ca8..8377f9651f 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -161,7 +161,8 @@ class Stage(object): def __init__( self, url_or_fetch_strategy, - name=None, mirror_path=None, keep=False, path=None, lock=True): + name=None, mirror_path=None, keep=False, path=None, lock=True, + alternate_fetchers=None): """Create a stage object. Parameters: url_or_fetch_strategy @@ -197,6 +198,7 @@ class Stage(object): self.fetcher.set_stage(self) # self.fetcher can change with mirrors. self.default_fetcher = self.fetcher + self.alternate_fetchers = alternate_fetchers # used for mirrored archives of repositories. self.skip_checksum_for_mirror = True @@ -408,28 +410,14 @@ class Stage(object): fetchers.insert( 0, fs.URLFetchStrategy( url, digest, expand=expand, extension=extension)) - fetchers.insert( - 0, spack.fetch_cache.fetcher( - self.mirror_path, digest, expand=expand, - extension=extension)) - - # Look for the archive in list_url - package_name = os.path.dirname(self.mirror_path) - pkg = spack.repo.get(package_name) - if pkg.list_url is not None and pkg.url is not None: - try: - archive_version = spack.url.parse_version( - self.default_fetcher.url) - versions = pkg.fetch_remote_versions() - try: - url_from_list = versions[Version(archive_version)] - fetchers.append(fs.URLFetchStrategy( - url_from_list, digest)) - except KeyError: - tty.msg("Can not find version %s in url_list" % - archive_version) - except: - tty.msg("Could not determine url from list_url.") + if self.default_fetcher.cachable: + fetchers.insert( + 0, spack.fetch_cache.fetcher( + self.mirror_path, digest, expand=expand, + extension=extension)) + + if self.alternate_fetchers: + fetchers.extend(self.alternate_fetchers) for fetcher in fetchers: try: |