diff options
| -rw-r--r-- | lib/spack/spack/__init__.py | 4 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/test.py | 20 | ||||
| -rw-r--r-- | lib/spack/spack/fetch_strategy.py | 16 | ||||
| -rw-r--r-- | lib/spack/spack/mirror.py | 8 | ||||
| -rw-r--r-- | lib/spack/spack/package.py | 12 | ||||
| -rw-r--r-- | lib/spack/spack/stage.py | 12 |
6 files changed, 69 insertions, 3 deletions
diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index c7a155597f..927e4c1422 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -46,6 +46,10 @@ var_path = join_path(spack_root, "var", "spack") stage_path = join_path(var_path, "stage") repos_path = join_path(var_path, "repos") share_path = join_path(spack_root, "share", "spack") +cache_path = join_path(var_path, "cache") + +import spack.fetch_strategy +cache = spack.fetch_strategy.FsCache(cache_path) prefix = spack_root opt_path = join_path(prefix, "opt") diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index cb9dd26c71..36810321ef 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -31,6 +31,7 @@ from llnl.util.lang import list_modules import spack import spack.test +from spack.fetch_strategy import FetchError description ="Run unit tests" @@ -50,6 +51,24 @@ def setup_parser(subparser): help="verbose output") +class MockCache(object): + def store(self, copyCmd, relativeDst): + pass + + def fetcher(self, targetPath, digest): + return MockCacheFetcher() + + +class MockCacheFetcher(object): + def set_stage(self, stage): + pass + + def fetch(self): + raise FetchError("Mock cache always fails for tests") + + def __str__(self): + return "[mock fetcher]" + def test(parser, args): if args.list: print "Available tests:" @@ -66,4 +85,5 @@ def test(parser, args): if not os.path.exists(outputDir): mkdirp(outputDir) + spack.cache = MockCache() spack.test.run(args.names, outputDir, args.verbose) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 7c8cebe0c9..4bbf7bb34c 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -305,7 +305,7 @@ class URLFetchStrategy(FetchStrategy): if not extension(destination) == extension(self.archive_file): raise ValueError("Cannot archive without matching extensions.") - shutil.move(self.archive_file, destination) + shutil.copy(self.archive_file, destination) @_needs_stage def check(self): @@ -805,6 +805,20 @@ def for_package_version(pkg, version): raise InvalidArgsError(pkg, version) +class FsCache(object): + def __init__(self, root): + self.root = os.path.abspath(root) + + def store(self, copyCmd, relativeDst): + dst = join_path(self.root, relativeDst) + mkdirp(os.path.dirname(dst)) + copyCmd(dst) + + def fetcher(self, targetPath, digest): + url = "file://" + join_path(self.root, targetPath) + return URLFetchStrategy(url, digest) + + class FetchError(spack.error.SpackError): def __init__(self, msg, long_msg=None): super(FetchError, self).__init__(msg, long_msg) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 0bbcfba6b4..81a2f6afb7 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -61,7 +61,13 @@ def mirror_archive_filename(spec, fetcher): # Otherwise we'll make a .tar.gz ourselves ext = 'tar.gz' - filename = "%s-%s" % (spec.package.name, spec.version) + tokens = [spec.package.name, spec.version] + digests = spec.package.digests + if digests: + # If a package has multiple digests, any one is sufficient to identify it + digestType, digest = sorted(digests.iteritems())[0] + tokens.extend([digestType, digest]) + filename = '-'.join(str(t) for t in tokens) if ext: filename += ".%s" % ext return filename diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 2e7d8a7709..70ec243726 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -413,6 +413,15 @@ class Package(object): raise ValueError("Can only get of package with concrete version.") return self.spec.versions[0] + @property + def digests(self): + """All digests for the concretized package version.""" + versionInfo = self.versions[self.version] + digests = {} + if 'md5' in versionInfo: + digests['md5'] = versionInfo['md5'] + return digests + @memoized def version_urls(self): """Return a list of URLs for different versions of this @@ -734,6 +743,9 @@ class Package(object): if spack.do_checksum and self.version in self.versions: self.stage.check() + self.stage.cache_local() + + def do_stage(self, mirror_only=False): """Unpacks the fetched tarball, then changes into the expanded tarball directory.""" diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index a76ec168ad..f28934d10a 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -304,6 +304,7 @@ class Stage(object): # Add URL strategies for all the mirrors with the digest for url in urls: fetchers.insert(0, fs.URLFetchStrategy(url, digest)) + fetchers.insert(0, spack.cache.fetcher(self.mirror_path, digest)) for fetcher in fetchers: try: @@ -320,6 +321,7 @@ class Stage(object): self.fetcher = self.default_fetcher raise fs.FetchError(errMessage, None) + def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" @@ -333,6 +335,11 @@ class Stage(object): else: self.fetcher.check() + + def cache_local(self): + spack.cache.store(self.fetcher.archive, self.mirror_path) + + def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded archive. Fail if the stage is not set up or if the archive is not yet @@ -436,7 +443,7 @@ class ResourceStage(Stage): shutil.move(source_path, destination_path) -@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy']) +@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy', 'cache_local']) class StageComposite: """ Composite for Stage type objects. The first item in this composite is considered to be the root package, and @@ -511,6 +518,9 @@ class DIYStage(object): # No need to destroy DIY stage. pass + def cache_local(self): + tty.msg("Sources for DIY stages are not cached") + def _get_mirrors(): """Get mirrors from spack configuration.""" |