From 16fa40b893054d8bd7f13625f204bf02d74a27b5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 18 Mar 2016 15:50:24 -0700 Subject: (1) add a var/cache directory under spack. (2) downloads from URLFetchStrategy check the cache and skip the download if the source is available there. --- lib/spack/spack/__init__.py | 2 ++ lib/spack/spack/fetch_strategy.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 0ba42bbbfc..22436e46c4 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -46,6 +46,8 @@ var_path = join_path(spack_root, "var", "spack") stage_path = join_path(var_path, "stage") repos_path = join_path(var_path, "repos") share_path = join_path(spack_root, "share", "spack") +cache_path = join_path(spack_root, "var", "cache") +mkdirp(cache_path) prefix = spack_root opt_path = join_path(prefix, "opt") diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 0d0a7db8a9..3a774a35cd 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -156,6 +156,11 @@ class URLFetchStrategy(FetchStrategy): if self.archive_file: tty.msg("Already downloaded %s" % self.archive_file) return + cached = self.check_cache() + if cached: + tty.msg("Cached %s." % cached) + shutil.copy(cached, "./") + return tty.msg("Trying to fetch from %s" % self.url) @@ -211,6 +216,9 @@ class URLFetchStrategy(FetchStrategy): if not self.archive_file: raise FailedDownloadError(self.url) + else: + shutil.copy(self.archive_file, spack.cache_path) + @property def archive_file(self): @@ -283,6 +291,17 @@ class URLFetchStrategy(FetchStrategy): "%s checksum failed for %s" % (checker.hash_name, self.archive_file), "Expected %s but got %s" % (self.digest, checker.sum)) + + def check_cache(self): + if not self.digest: + return + checker = crypto.Checker(self.digest) + paths = (join_path(spack.cache_path, f) for f in os.listdir(spack.cache_path)) + for p in paths: + if checker.check(p): + return p + + @_needs_stage def reset(self): """Removes the source path if it exists, then re-expands the archive.""" -- cgit v1.2.3-60-g2f50 From ac7323118e9c3ddfc7b27992e545e59de2f32c7f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 18 Mar 2016 16:34:45 -0700 Subject: rename for clarity --- lib/spack/spack/fetch_strategy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 3a774a35cd..73083a0f5a 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -156,7 +156,7 @@ class URLFetchStrategy(FetchStrategy): if self.archive_file: tty.msg("Already downloaded %s" % self.archive_file) return - cached = self.check_cache() + cached = self.search_cache() if cached: tty.msg("Cached %s." % cached) shutil.copy(cached, "./") @@ -292,7 +292,7 @@ class URLFetchStrategy(FetchStrategy): "Expected %s but got %s" % (self.digest, checker.sum)) - def check_cache(self): + def search_cache(self): if not self.digest: return checker = crypto.Checker(self.digest) -- cgit v1.2.3-60-g2f50 From fd067dd8b888ac4c2cc6cacec44d2ba978b04e8a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 18 Mar 2016 17:00:13 -0700 Subject: since only archives with checksums can be retrieved from the cache, make sure that an archive without a checksum isnt placed there (this wouldn't cause an error but does waste space and might be confusing) --- lib/spack/spack/fetch_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 73083a0f5a..27fb38b7a6 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -216,7 +216,7 @@ class URLFetchStrategy(FetchStrategy): if not self.archive_file: raise FailedDownloadError(self.url) - else: + elif self.digest: shutil.copy(self.archive_file, spack.cache_path) -- cgit v1.2.3-60-g2f50 From d632266a40e8d472892991144391b1862231fec0 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 18 Mar 2016 17:15:45 -0700 Subject: move cache to var/spack/cache --- lib/spack/spack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 22436e46c4..6fb472b15d 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -46,7 +46,7 @@ var_path = join_path(spack_root, "var", "spack") stage_path = join_path(var_path, "stage") repos_path = join_path(var_path, "repos") share_path = join_path(spack_root, "share", "spack") -cache_path = join_path(spack_root, "var", "cache") +cache_path = join_path(var_path, "cache") mkdirp(cache_path) prefix = spack_root -- cgit v1.2.3-60-g2f50 From ee5e507ff60e56f9d83de9b7e2c19e929dcd3481 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 21 Mar 2016 20:48:12 -0700 Subject: pursuing a strategy using fetch.archive and treating var/spack/cache as a mirror. this should support both URLFetchStrategy as well as VCSFetchStrategy (the previous strategy supported only the former). this won't work until URLFetchStrategy.archive is updated --- lib/spack/spack/fetch_strategy.py | 17 ----------------- lib/spack/spack/package.py | 2 ++ lib/spack/spack/stage.py | 13 ++++++++++++- 3 files changed, 14 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 27fb38b7a6..d1b3ce9291 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -156,11 +156,6 @@ class URLFetchStrategy(FetchStrategy): if self.archive_file: tty.msg("Already downloaded %s" % self.archive_file) return - cached = self.search_cache() - if cached: - tty.msg("Cached %s." % cached) - shutil.copy(cached, "./") - return tty.msg("Trying to fetch from %s" % self.url) @@ -216,8 +211,6 @@ class URLFetchStrategy(FetchStrategy): if not self.archive_file: raise FailedDownloadError(self.url) - elif self.digest: - shutil.copy(self.archive_file, spack.cache_path) @property @@ -292,16 +285,6 @@ class URLFetchStrategy(FetchStrategy): "Expected %s but got %s" % (self.digest, checker.sum)) - def search_cache(self): - if not self.digest: - return - checker = crypto.Checker(self.digest) - paths = (join_path(spack.cache_path, f) for f in os.listdir(spack.cache_path)) - for p in paths: - if checker.check(p): - return p - - @_needs_stage def reset(self): """Removes the source path if it exists, then re-expands the archive.""" diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b488e4c49d..d007f37aeb 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -715,6 +715,8 @@ class Package(object): if spack.do_checksum and self.version in self.versions: self.stage.check() + self.stage.cache_local() + def do_stage(self, mirror_only=False): """Unpacks the fetched tarball, then changes into the expanded tarball diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index f88f82fc2d..8933ad6da2 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -273,6 +273,7 @@ class Stage(object): # the root, so we add a '/' if it is not present. mirror_roots = [root if root.endswith('/') else root + '/' for root in mirrors.values()] + mirror_roots.append("file://" + os.path.abspath(spack.cache_path) + os.sep) urls = [urljoin(root, self.mirror_path) for root in mirror_roots] # If this archive is normally fetched from a tarball URL, @@ -305,6 +306,7 @@ class Stage(object): self.fetcher = self.default_fetcher raise fs.FetchError(errMessage, None) + def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" @@ -318,6 +320,15 @@ class Stage(object): else: self.fetcher.check() + + def cache_local(self): + archiveDst = join_path(os.path.abspath(spack.cache_path), self.mirror_path) + mkdirp(os.path.dirname(archiveDst)) + # TODO: this moves the archive for URLFetchStrategy vs. a copy - edit + # to do a move? + self.fetcher.archive(archiveDst) + + def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded archive. Fail if the stage is not set up or if the archive is not yet @@ -421,7 +432,7 @@ class ResourceStage(Stage): shutil.move(source_path, destination_path) -@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy']) +@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy', 'cache_local']) class StageComposite: """ Composite for Stage type objects. The first item in this composite is considered to be the root package, and -- cgit v1.2.3-60-g2f50 From b255f02762375cee064b062837581e5466fdb908 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 21 Mar 2016 20:50:26 -0700 Subject: undoing whitespace-only diff --- lib/spack/spack/fetch_strategy.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index d1b3ce9291..0d0a7db8a9 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -212,7 +212,6 @@ class URLFetchStrategy(FetchStrategy): if not self.archive_file: raise FailedDownloadError(self.url) - @property def archive_file(self): """Path to the source archive within this stage directory.""" @@ -284,7 +283,6 @@ class URLFetchStrategy(FetchStrategy): "%s checksum failed for %s" % (checker.hash_name, self.archive_file), "Expected %s but got %s" % (self.digest, checker.sum)) - @_needs_stage def reset(self): """Removes the source path if it exists, then re-expands the archive.""" -- cgit v1.2.3-60-g2f50 From 41a97c8f80b8e93a7b180bd4d4a5a4286ce6f311 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 21 Mar 2016 20:55:23 -0700 Subject: temporarily wrap archiving with conditional to avoid moving (this still causes a failure on the initial download) --- lib/spack/spack/stage.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 8933ad6da2..61faec6de9 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -323,10 +323,11 @@ class Stage(object): def cache_local(self): archiveDst = join_path(os.path.abspath(spack.cache_path), self.mirror_path) - mkdirp(os.path.dirname(archiveDst)) - # TODO: this moves the archive for URLFetchStrategy vs. a copy - edit - # to do a move? - self.fetcher.archive(archiveDst) + if not os.path.exists(archiveDst): #tmp conditional + mkdirp(os.path.dirname(archiveDst)) + # TODO: this moves the archive for URLFetchStrategy vs. a copy - + # edit to do a move? + self.fetcher.archive(archiveDst) def expand_archive(self): -- cgit v1.2.3-60-g2f50 From 75460d8586bf62dcedaf4eb5acdb2de75f1d662a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 22 Mar 2016 10:43:43 -0700 Subject: URLFetchStrategy.archive does a copy vs. a move now --- lib/spack/spack/fetch_strategy.py | 2 +- lib/spack/spack/stage.py | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 0d0a7db8a9..fc5d7e231c 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -268,7 +268,7 @@ class URLFetchStrategy(FetchStrategy): if not extension(destination) == extension(self.archive_file): raise ValueError("Cannot archive without matching extensions.") - shutil.move(self.archive_file, destination) + shutil.copy(self.archive_file, destination) @_needs_stage def check(self): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 61faec6de9..54359bddce 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -323,11 +323,8 @@ class Stage(object): def cache_local(self): archiveDst = join_path(os.path.abspath(spack.cache_path), self.mirror_path) - if not os.path.exists(archiveDst): #tmp conditional - mkdirp(os.path.dirname(archiveDst)) - # TODO: this moves the archive for URLFetchStrategy vs. a copy - - # edit to do a move? - self.fetcher.archive(archiveDst) + mkdirp(os.path.dirname(archiveDst)) + self.fetcher.archive(archiveDst) def expand_archive(self): -- cgit v1.2.3-60-g2f50 From cb9fba98d8d0c389f3295d5d30cdb650e6ea79b7 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 22 Mar 2016 19:37:47 -0700 Subject: (1) relocate cache for tests (2) initial approach for restoring unit tests (just for git tests although the same concept applies to the other unit tests which are failing - namely those for svn and hg) --- lib/spack/spack/cmd/test.py | 4 ++++ lib/spack/spack/test/git_fetch.py | 4 ++++ 2 files changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index ddc6cb4fce..9a85c7d270 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import shutil from pprint import pprint from llnl.util.filesystem import join_path, mkdirp @@ -66,4 +67,7 @@ def test(parser, args): if not os.path.exists(outputDir): mkdirp(outputDir) + spack.cache_path = join_path(spack.var_path, "test-cache") + mkdirp(spack.cache_path) spack.test.run(args.names, outputDir, args.verbose) + shutil.rmtree(spack.cache_path) diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 3578044116..76814f089a 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -30,6 +30,8 @@ from spack.test.mock_packages_test import * from spack.test.mock_repo import MockGitRepo from spack.version import ver +import shutil +import os class GitFetchTest(MockPackagesTest): """Tests fetching from a dummy git repository.""" @@ -49,6 +51,8 @@ class GitFetchTest(MockPackagesTest): """Destroy the stage space used by this test.""" super(GitFetchTest, self).tearDown() self.repo.destroy() + for d in os.listdir(spack.cache_path): + shutil.rmtree(os.path.join(spack.cache_path, d)) def assert_rev(self, rev): """Check that the current git revision is equal to the supplied rev.""" -- cgit v1.2.3-60-g2f50 From ed0f6f75a7215c959431377ffe7e4faf09dc88d6 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 23 Mar 2016 19:49:28 -0700 Subject: clear test cache before and after each MockPackagesTest (I think Ive got a better way to avoid test fragility but Ill add this for now) --- lib/spack/spack/cmd/test.py | 2 +- lib/spack/spack/test/git_fetch.py | 5 ----- lib/spack/spack/test/mock_packages_test.py | 7 +++++++ 3 files changed, 8 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index 9a85c7d270..233cd186f0 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -70,4 +70,4 @@ def test(parser, args): spack.cache_path = join_path(spack.var_path, "test-cache") mkdirp(spack.cache_path) spack.test.run(args.names, outputDir, args.verbose) - shutil.rmtree(spack.cache_path) + shutil.rmtree(spack.cache_path, ignore_errors=True) diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 76814f089a..d3e1206c13 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -30,9 +30,6 @@ from spack.test.mock_packages_test import * from spack.test.mock_repo import MockGitRepo from spack.version import ver -import shutil -import os - class GitFetchTest(MockPackagesTest): """Tests fetching from a dummy git repository.""" @@ -51,8 +48,6 @@ class GitFetchTest(MockPackagesTest): """Destroy the stage space used by this test.""" super(GitFetchTest, self).tearDown() self.repo.destroy() - for d in os.listdir(spack.cache_path): - shutil.rmtree(os.path.join(spack.cache_path, d)) def assert_rev(self, rev): """Check that the current git revision is equal to the supplied rev.""" diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 6d24a84150..85c15d4a6a 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -125,9 +125,16 @@ class MockPackagesTest(unittest.TestCase): pkg.dependencies.update(deps) + def rm_cache(self): + shutil.rmtree(spack.cache_path, ignore_errors=True) + + def setUp(self): + self.rm_cache() + mkdirp(spack.cache_path) self.initmock() def tearDown(self): + self.rm_cache() self.cleanmock() -- cgit v1.2.3-60-g2f50 From dbfa6c925ec60c89004f6ccd88e985f71f650a69 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 23 Mar 2016 20:18:58 -0700 Subject: replace references to cache directory with references to new cache object. tests may assign a mock cache but by default it is None (this will avoid any implicit caching behavior confusing unit tests) --- lib/spack/llnl/util/filesystem.py | 9 +++++++++ lib/spack/spack/__init__.py | 5 ++++- lib/spack/spack/cmd/test.py | 4 +--- lib/spack/spack/stage.py | 4 +--- lib/spack/spack/test/mock_packages_test.py | 12 +++++++++--- 5 files changed, 24 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index c4665c284c..7586d514d1 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -392,3 +392,12 @@ def remove_linked_tree(path): os.unlink(path) else: shutil.rmtree(path, True) + +class FsCache(object): + def __init__(self, root): + self.root = os.path.abspath(root) + + def store(self, copyCmd, relativeDst): + dst = join_path(self.root, relativeDst) + mkdirp(os.path.dirname(dst)) + copyCmd(dst) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 6fb472b15d..0041d50624 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -47,7 +47,10 @@ stage_path = join_path(var_path, "stage") repos_path = join_path(var_path, "repos") share_path = join_path(spack_root, "share", "spack") cache_path = join_path(var_path, "cache") -mkdirp(cache_path) + +# TODO: i get a complaint if i dont qualify this, fix that +import llnl.util.filesystem +cache = llnl.util.filesystem.FsCache(cache_path) prefix = spack_root opt_path = join_path(prefix, "opt") diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index 233cd186f0..1a02521814 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -67,7 +67,5 @@ def test(parser, args): if not os.path.exists(outputDir): mkdirp(outputDir) - spack.cache_path = join_path(spack.var_path, "test-cache") - mkdirp(spack.cache_path) + spack.cache = None spack.test.run(args.names, outputDir, args.verbose) - shutil.rmtree(spack.cache_path, ignore_errors=True) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 54359bddce..780f391603 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -322,9 +322,7 @@ class Stage(object): def cache_local(self): - archiveDst = join_path(os.path.abspath(spack.cache_path), self.mirror_path) - mkdirp(os.path.dirname(archiveDst)) - self.fetcher.archive(archiveDst) + spack.cache.store(self.fetcher.archive, self.mirror_path) def expand_archive(self): diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 85c15d4a6a..06d7e7d4e1 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -34,6 +34,8 @@ from ordereddict_backport import OrderedDict from spack.repository import RepoPath from spack.spec import Spec +import llnl.util.tty as tty + mock_compiler_config = """\ compilers: all: @@ -130,11 +132,15 @@ class MockPackagesTest(unittest.TestCase): def setUp(self): - self.rm_cache() - mkdirp(spack.cache_path) + spack.cache = MockCache() self.initmock() def tearDown(self): - self.rm_cache() + spack.cache = None self.cleanmock() + +class MockCache(object): + def store(self, copyCmd, relativeDst): + tty.warn("Copying " + str(relativeDst)) + pass -- cgit v1.2.3-60-g2f50 From 13bf7d4ff11e6897d4b688c16564f924e40f657e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 24 Mar 2016 12:02:39 -0700 Subject: (1) move definition of MockCache to test command (no definitions or extra work is required in MockPackagesTest) (2) removing outdated logic (which originated in this branch) and minor cleanup --- lib/spack/spack/cmd/test.py | 7 ++++++- lib/spack/spack/test/git_fetch.py | 1 + lib/spack/spack/test/mock_packages_test.py | 13 ------------- 3 files changed, 7 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index 1a02521814..3c405c5c6b 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -51,6 +51,11 @@ def setup_parser(subparser): help="verbose output") +class MockCache(object): + def store(self, copyCmd, relativeDst): + pass + + def test(parser, args): if args.list: print "Available tests:" @@ -67,5 +72,5 @@ def test(parser, args): if not os.path.exists(outputDir): mkdirp(outputDir) - spack.cache = None + spack.cache = MockCache() spack.test.run(args.names, outputDir, args.verbose) diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index d3e1206c13..3578044116 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -30,6 +30,7 @@ from spack.test.mock_packages_test import * from spack.test.mock_repo import MockGitRepo from spack.version import ver + class GitFetchTest(MockPackagesTest): """Tests fetching from a dummy git repository.""" diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 06d7e7d4e1..6d24a84150 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -34,8 +34,6 @@ from ordereddict_backport import OrderedDict from spack.repository import RepoPath from spack.spec import Spec -import llnl.util.tty as tty - mock_compiler_config = """\ compilers: all: @@ -127,20 +125,9 @@ class MockPackagesTest(unittest.TestCase): pkg.dependencies.update(deps) - def rm_cache(self): - shutil.rmtree(spack.cache_path, ignore_errors=True) - - def setUp(self): - spack.cache = MockCache() self.initmock() def tearDown(self): - spack.cache = None self.cleanmock() - -class MockCache(object): - def store(self, copyCmd, relativeDst): - tty.warn("Copying " + str(relativeDst)) - pass -- cgit v1.2.3-60-g2f50 From fe71ba992d26b66a4a9492d9b0fbf17b1410b1e1 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 24 Mar 2016 12:16:50 -0700 Subject: remove unused import --- lib/spack/spack/cmd/test.py | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index 3c405c5c6b..82bf3928c3 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -23,7 +23,6 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import shutil from pprint import pprint from llnl.util.filesystem import join_path, mkdirp -- cgit v1.2.3-60-g2f50 From 142d1f5cbc098a4e8a0046148400bb2f40e839bc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 24 Mar 2016 19:28:21 -0700 Subject: stage creates cache fetcher with cache object (so it can be mocked for tests) --- lib/spack/llnl/util/filesystem.py | 9 --------- lib/spack/spack/__init__.py | 4 ++-- lib/spack/spack/cmd/test.py | 14 ++++++++++++++ lib/spack/spack/fetch_strategy.py | 14 ++++++++++++++ lib/spack/spack/stage.py | 2 +- 5 files changed, 31 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 7586d514d1..c4665c284c 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -392,12 +392,3 @@ def remove_linked_tree(path): os.unlink(path) else: shutil.rmtree(path, True) - -class FsCache(object): - def __init__(self, root): - self.root = os.path.abspath(root) - - def store(self, copyCmd, relativeDst): - dst = join_path(self.root, relativeDst) - mkdirp(os.path.dirname(dst)) - copyCmd(dst) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 0041d50624..6e7cb217c8 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -49,8 +49,8 @@ share_path = join_path(spack_root, "share", "spack") cache_path = join_path(var_path, "cache") # TODO: i get a complaint if i dont qualify this, fix that -import llnl.util.filesystem -cache = llnl.util.filesystem.FsCache(cache_path) +import spack.fetch_strategy +cache = fetch_strategy.FsCache(cache_path) prefix = spack_root opt_path = join_path(prefix, "opt") diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index 82bf3928c3..8c9dea3c66 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -31,6 +31,7 @@ from llnl.util.lang import list_modules import spack import spack.test +from spack.fetch_strategy import FetchError description ="Run unit tests" @@ -54,6 +55,19 @@ class MockCache(object): def store(self, copyCmd, relativeDst): pass + def fetcher(self, targetPath, digest): + return MockCacheFetcher() + + +class MockCacheFetcher(object): + def set_stage(self, stage): + pass + + def fetch(self): + raise FetchError("Mock cache always fails for tests") + + def __str__(self): + return "[mock fetcher]" def test(parser, args): if args.list: diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index fc5d7e231c..b696a12e7a 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -689,6 +689,20 @@ def for_package_version(pkg, version): raise InvalidArgsError(pkg, version) +class FsCache(object): + def __init__(self, root): + self.root = os.path.abspath(root) + + def store(self, copyCmd, relativeDst): + dst = join_path(self.root, relativeDst) + mkdirp(os.path.dirname(dst)) + copyCmd(dst) + + def fetcher(self, targetPath, digest): + url = "file://" + join_path(self.root, targetPath) + return URLFetchStrategy(url, digest) + + class FetchError(spack.error.SpackError): def __init__(self, msg, long_msg=None): super(FetchError, self).__init__(msg, long_msg) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 780f391603..9f2619f43e 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -273,7 +273,6 @@ class Stage(object): # the root, so we add a '/' if it is not present. mirror_roots = [root if root.endswith('/') else root + '/' for root in mirrors.values()] - mirror_roots.append("file://" + os.path.abspath(spack.cache_path) + os.sep) urls = [urljoin(root, self.mirror_path) for root in mirror_roots] # If this archive is normally fetched from a tarball URL, @@ -290,6 +289,7 @@ class Stage(object): # Add URL strategies for all the mirrors with the digest for url in urls: fetchers.insert(0, fs.URLFetchStrategy(url, digest)) + fetchers.insert(0, spack.cache.fetcher(self.mirror_path, digest)) for fetcher in fetchers: try: -- cgit v1.2.3-60-g2f50 From 6423eab917f6914cfce253315c9063762e6ae749 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 24 Mar 2016 19:45:10 -0700 Subject: implemented cache_local method for DIY stage (as a noop) --- lib/spack/spack/stage.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 9f2619f43e..e8239d27be 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -503,6 +503,8 @@ class DIYStage(object): # No need to destroy DIY stage. pass + def cache_local(self): + tty.msg("Sources for DIY stages are not cached") def _get_mirrors(): """Get mirrors from spack configuration.""" -- cgit v1.2.3-60-g2f50 From bd5abb292254b40140d1b61849cb86851718b335 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 24 Mar 2016 19:48:15 -0700 Subject: spacing issue --- lib/spack/spack/stage.py | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index e8239d27be..d5ee231ef7 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -506,6 +506,7 @@ class DIYStage(object): def cache_local(self): tty.msg("Sources for DIY stages are not cached") + def _get_mirrors(): """Get mirrors from spack configuration.""" config = spack.config.get_config('mirrors') -- cgit v1.2.3-60-g2f50 From 06c98320a88924234a86b038ca7d3a60a8361f8c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 25 Mar 2016 18:38:26 -0700 Subject: handle case where file contents change but resource name does not (e.g. if resource maintainer uses same name for each new version of a package) --- lib/spack/spack/fetch_strategy.py | 10 ++++++++++ lib/spack/spack/mirror.py | 5 ++++- 2 files changed, 14 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index b696a12e7a..9938f011d0 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -107,6 +107,8 @@ class FetchStrategy(object): def archive(self, destination): pass # Used to create tarball for mirror. + def file_hash(self): pass # Identifies the resource to be retrieved + def __str__(self): # Should be human readable URL. return "FetchStrategy.__str___" @@ -217,6 +219,10 @@ class URLFetchStrategy(FetchStrategy): """Path to the source archive within this stage directory.""" return self.stage.archive_file + @property + def file_hash(self): + return self.digest + @_needs_stage def expand(self): if not self.expand_archive: @@ -349,6 +355,10 @@ class VCSFetchStrategy(FetchStrategy): self.stage.chdir() tar('-czf', destination, os.path.basename(self.stage.source_path)) + @property + def file_hash(self): + return None + def __str__(self): return "VCS: %s" % self.url diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 6981f69ac0..cb2588fb29 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -61,7 +61,10 @@ def mirror_archive_filename(spec, fetcher): # Otherwise we'll make a .tar.gz ourselves ext = 'tar.gz' - filename = "%s-%s" % (spec.package.name, spec.version) + tokens = [spec.package.name, spec.version] + if fetcher.file_hash: + tokens.append(fetcher.file_hash) + filename = '-'.join(str(t) for t in tokens) if ext: filename += ".%s" % ext return filename -- cgit v1.2.3-60-g2f50 From bee224c567ee735547bb183cd6a1d6e04309c81a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 29 Mar 2016 18:25:22 -0700 Subject: mirror archive filename now includes the digest type as well as the digest --- lib/spack/spack/fetch_strategy.py | 10 ---------- lib/spack/spack/mirror.py | 7 +++++-- lib/spack/spack/package.py | 7 +++++++ 3 files changed, 12 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 9938f011d0..b696a12e7a 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -107,8 +107,6 @@ class FetchStrategy(object): def archive(self, destination): pass # Used to create tarball for mirror. - def file_hash(self): pass # Identifies the resource to be retrieved - def __str__(self): # Should be human readable URL. return "FetchStrategy.__str___" @@ -219,10 +217,6 @@ class URLFetchStrategy(FetchStrategy): """Path to the source archive within this stage directory.""" return self.stage.archive_file - @property - def file_hash(self): - return self.digest - @_needs_stage def expand(self): if not self.expand_archive: @@ -355,10 +349,6 @@ class VCSFetchStrategy(FetchStrategy): self.stage.chdir() tar('-czf', destination, os.path.basename(self.stage.source_path)) - @property - def file_hash(self): - return None - def __str__(self): return "VCS: %s" % self.url diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index cb2588fb29..c929a092b4 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -62,8 +62,11 @@ def mirror_archive_filename(spec, fetcher): ext = 'tar.gz' tokens = [spec.package.name, spec.version] - if fetcher.file_hash: - tokens.append(fetcher.file_hash) + package = spack.repo.get(spec) + digests = package.digests + if digests: + if 'md5' in digests: + tokens.extend(['md5', digests['md5']]) filename = '-'.join(str(t) for t in tokens) if ext: filename += ".%s" % ext diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index d007f37aeb..2feea51fa5 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -397,6 +397,13 @@ class Package(object): raise ValueError("Can only get of package with concrete version.") return self.spec.versions[0] + @property + def digests(self): + versionInfo = self.versions[self.version] + digests = {} + if 'md5' in versionInfo: + digests['md5'] = versionInfo['md5'] + return digests @memoized def version_urls(self): -- cgit v1.2.3-60-g2f50 From ce4de6227e339bab98f8a73013bce1898f275b6f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 29 Mar 2016 18:45:58 -0700 Subject: (1) access package via spec property (2) use any digest to form archive filename --- lib/spack/spack/mirror.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index c929a092b4..78db22f73b 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -62,11 +62,11 @@ def mirror_archive_filename(spec, fetcher): ext = 'tar.gz' tokens = [spec.package.name, spec.version] - package = spack.repo.get(spec) - digests = package.digests + digests = spec.package.digests if digests: - if 'md5' in digests: - tokens.extend(['md5', digests['md5']]) + # If a package has multiple digests, any one is sufficient to identify it + digestType, digest = digests.iteritems().next() + tokens.extend([digestType, digest]) filename = '-'.join(str(t) for t in tokens) if ext: filename += ".%s" % ext -- cgit v1.2.3-60-g2f50 From 03d907e1e5bfc01f7cdb457dbcd58bb903683e34 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 29 Mar 2016 18:48:25 -0700 Subject: in the case of multiple digests, avoid creating different mirror filenames from run to run (as long as the available digests do not change) --- lib/spack/spack/mirror.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 78db22f73b..64f589ad8b 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -65,7 +65,7 @@ def mirror_archive_filename(spec, fetcher): digests = spec.package.digests if digests: # If a package has multiple digests, any one is sufficient to identify it - digestType, digest = digests.iteritems().next() + digestType, digest = sorted(digests.iteritems())[0] tokens.extend([digestType, digest]) filename = '-'.join(str(t) for t in tokens) if ext: -- cgit v1.2.3-60-g2f50 From c40559433bdfadb6059070060114880c9042d799 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 29 Mar 2016 18:54:24 -0700 Subject: added docstring --- lib/spack/spack/package.py | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 2feea51fa5..d1479ec9de 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -399,6 +399,7 @@ class Package(object): @property def digests(self): + """All digests for the concretized package version.""" versionInfo = self.versions[self.version] digests = {} if 'md5' in versionInfo: -- cgit v1.2.3-60-g2f50 From a0c42a3fd1a8c5319ff8e02313b7f12e8c59349d Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 29 Mar 2016 18:58:18 -0700 Subject: removed stale TODO --- lib/spack/spack/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 6e7cb217c8..a2d61fa2f7 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -48,9 +48,8 @@ repos_path = join_path(var_path, "repos") share_path = join_path(spack_root, "share", "spack") cache_path = join_path(var_path, "cache") -# TODO: i get a complaint if i dont qualify this, fix that import spack.fetch_strategy -cache = fetch_strategy.FsCache(cache_path) +cache = spack.fetch_strategy.FsCache(cache_path) prefix = spack_root opt_path = join_path(prefix, "opt") -- cgit v1.2.3-60-g2f50 From c6d8208150f7827da5a32c4c9187714ccb6651a1 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 1 Apr 2016 22:51:14 -0400 Subject: Added recursive module loading --- lib/spack/spack/cmd/load.py | 2 +- lib/spack/spack/cmd/module.py | 87 ++++++++++++++++++++++++++++++++----------- 2 files changed, 67 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/load.py b/lib/spack/spack/cmd/load.py index 30d86c3b01..aa299a0bb3 100644 --- a/lib/spack/spack/cmd/load.py +++ b/lib/spack/spack/cmd/load.py @@ -31,7 +31,7 @@ def setup_parser(subparser): """Parser is only constructed so that this prints a nice help message with -h. """ subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, help='Spec of package to load with modules.') + 'spec', nargs=argparse.REMAINDER, help="Spec of package to load with modules. (If -, read specs from STDIN)") def load(parser, args): diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index 315d9fc926..b79737ab22 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -48,40 +48,85 @@ def setup_parser(subparser): find_parser = sp.add_parser('find', help='Find module files for packages.') find_parser.add_argument( 'module_type', help="Type of module to find file for. [" + '|'.join(module_types) + "]") + find_parser.add_argument( + '-r', '--dependencies', action='store_true', dest='recurse_dependencies', + help='Recursively traverse dependencies for modules to load.') + + find_parser.add_argument( + '-s', '--shell', action='store_true', dest='shell', + help='Generate shell script (instead of input for module command)') + find_parser.add_argument('spec', nargs='+', help='spec to find a module file for.') -def module_find(mtype, spec_array): + + +def module_find(mtype, flags, spec_array): """Look at all installed packages and see if the spec provided matches any. If it does, check whether there is a module file of type there, and print out the name that the user should type to use that package's module. """ - if mtype not in module_types: - tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types))) - specs = spack.cmd.parse_specs(spec_array) - if len(specs) > 1: - tty.die("You can only pass one spec.") - spec = specs[0] + # -------------------------------------- + def _find_modules(spec, modules_list): + """Finds all modules and sub-modules for a spec""" + if str(spec.version) == 'system': + # No Spack module for system-installed packages + return + + if flags.recurse_dependencies: + for dep in spec.dependencies.values(): + _find_modules(dep, modules_list) - specs = spack.installed_db.query(spec) - if len(specs) == 0: - tty.die("No installed packages match spec %s" % spec) + mod = module_types[mtype](spec) + if not os.path.isfile(mod.file_name): + tty.die("No %s module is installed for %s" % (mtype, spec)) + modules_list.append((spec, mod)) - if len(specs) > 1: - tty.error("Multiple matches for spec %s. Choose one:" % spec) - for s in specs: - sys.stderr.write(s.tree(color=True)) - sys.exit(1) - mt = module_types[mtype] - mod = mt(specs[0]) - if not os.path.isfile(mod.file_name): - tty.die("No %s module is installed for %s" % (mtype, spec)) + # -------------------------------------- - print(mod.use_name) + if mtype not in module_types: + tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types))) + raw_specs = spack.cmd.parse_specs(spec_array) + modules = set() # Modules we will load + seen = set() + for raw_spec in raw_specs: + + # ----------- Make sure the spec only resolves to ONE thing + specs = spack.installed_db.query(raw_spec) + if len(specs) == 0: + tty.die("No installed packages match spec %s" % raw_spec) + + if len(specs) > 1: + tty.error("Multiple matches for spec %s. Choose one:" % spec) + for s in specs: + sys.stderr.write(s.tree(color=True)) + sys.exit(1) + spec = specs[0] + + # ----------- Chase down modules for it and all its dependencies + modules_dups = list() + _find_modules(spec, modules_dups) + + # Remove duplicates while keeping order + modules_unique = list() + for spec,mod in modules_dups: + if mod.use_name not in seen: + modules_unique.append((spec,mod)) + seen.add(mod.use_name) + + # Output... + if flags.shell: + module_cmd = {'tcl' : 'module load', 'dotkit' : 'dotkit use'}[mtype] + for spec,mod in modules_unique: + if flags.shell: + print '# %s' % spec.format() + print '%s %s' % (module_cmd, mod.use_name) + else: + print mod.use_name def module_refresh(): """Regenerate all module files for installed packages known to @@ -104,4 +149,4 @@ def module(parser, args): module_refresh() elif args.module_command == 'find': - module_find(args.module_type, args.spec) + module_find(args.module_type, args, args.spec) -- cgit v1.2.3-60-g2f50 From 281835887a91e13bee17b5954a9ff8c7db8e299c Mon Sep 17 00:00:00 2001 From: citibeth Date: Fri, 1 Apr 2016 22:52:36 -0400 Subject: Added documentation for recursive modules --- lib/spack/docs/basic_usage.rst | 69 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) (limited to 'lib') diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index accf09cc2a..78fc73d738 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -943,6 +943,75 @@ used ``gcc``. You could therefore just type: To identify just the one built with the Intel compiler. +Recursive Modules +`````````````````` + +In some cases, it is desirable to load not just a module, but also all +the modules it depends on. This is not required for most modules +because Spack builds binaries with RPATH support. However, not all +packages use RPATH to find their dependencies: this can be true in +particular for Python extensions, which are currently *not* built with +RPATH. + +Modules may be loaded recursively with the command: + +.. code-block:: sh + + $ module load `spack module tcl --dependencies ... + +More than one spec may be placed on the command line here. + +Module Comamnds for Shell Scripts +`````````````````````````````````` + +Although Spack is flexbile, the ``module`` command is much faster. +This could become an issue when emitting a series of ``spack load`` +commands inside a shell script. By adding the ``--shell`` flag, +``spack module find`` may also be used to generate code that can be +cut-and-pasted into a shell script. For example: + +.. code-block:: sh + + $ spack module find tcl --dependencies --shell py-numpy git + # bzip2@1.0.6%gcc@4.9.3=linux-x86_64 + module load bzip2-1.0.6-gcc-4.9.3-ktnrhkrmbbtlvnagfatrarzjojmkvzsx + # ncurses@6.0%gcc@4.9.3=linux-x86_64 + module load ncurses-6.0-gcc-4.9.3-kaazyneh3bjkfnalunchyqtygoe2mncv + # zlib@1.2.8%gcc@4.9.3=linux-x86_64 + module load zlib-1.2.8-gcc-4.9.3-v3ufwaahjnviyvgjcelo36nywx2ufj7z + # sqlite@3.8.5%gcc@4.9.3=linux-x86_64 + module load sqlite-3.8.5-gcc-4.9.3-a3eediswgd5f3rmto7g3szoew5nhehbr + # readline@6.3%gcc@4.9.3=linux-x86_64 + module load readline-6.3-gcc-4.9.3-se6r3lsycrwxyhreg4lqirp6xixxejh3 + # python@3.5.1%gcc@4.9.3=linux-x86_64 + module load python-3.5.1-gcc-4.9.3-5q5rsrtjld4u6jiicuvtnx52m7tfhegi + # py-setuptools@20.5%gcc@4.9.3=linux-x86_64 + module load py-setuptools-20.5-gcc-4.9.3-4qr2suj6p6glepnedmwhl4f62x64wxw2 + # py-nose@1.3.7%gcc@4.9.3=linux-x86_64 + module load py-nose-1.3.7-gcc-4.9.3-pwhtjw2dvdvfzjwuuztkzr7b4l6zepli + # openblas@0.2.17%gcc@4.9.3+shared=linux-x86_64 + module load openblas-0.2.17-gcc-4.9.3-pw6rmlom7apfsnjtzfttyayzc7nx5e7y + # py-numpy@1.11.0%gcc@4.9.3+blas+lapack=linux-x86_64 + module load py-numpy-1.11.0-gcc-4.9.3-mulodttw5pcyjufva4htsktwty4qd52r + # curl@7.47.1%gcc@4.9.3=linux-x86_64 + module load curl-7.47.1-gcc-4.9.3-ohz3fwsepm3b462p5lnaquv7op7naqbi + # autoconf@2.69%gcc@4.9.3=linux-x86_64 + module load autoconf-2.69-gcc-4.9.3-bkibjqhgqm5e3o423ogfv2y3o6h2uoq4 + # cmake@3.5.0%gcc@4.9.3~doc+ncurses+openssl~qt=linux-x86_64 + module load cmake-3.5.0-gcc-4.9.3-x7xnsklmgwla3ubfgzppamtbqk5rwn7t + # expat@2.1.0%gcc@4.9.3=linux-x86_64 + module load expat-2.1.0-gcc-4.9.3-6pkz2ucnk2e62imwakejjvbv6egncppd + # git@2.8.0-rc2%gcc@4.9.3+curl+expat=linux-x86_64 + module load git-2.8.0-rc2-gcc-4.9.3-3bib4hqtnv5xjjoq5ugt3inblt4xrgkd + +The script may be further edited by removing unnecessary modules. +This script may be directly executed in bash via + +.. code-block :: sh + + source <( spack module find tcl --dependencies --shell py-numpy git ) + + Regenerating Module files ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- cgit v1.2.3-60-g2f50 From de1ec4be8b0c7bc8c36e88a460e64b3be38cff35 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 6 Jun 2016 12:26:13 -0700 Subject: change source archive caching to omit digest from name and instead calculate and compare the checksum. This achieves the original goal of discarding stale cache files without preserving multiple files for the same version. --- lib/spack/spack/fetch_strategy.py | 23 +++++++++++++++++++++-- lib/spack/spack/mirror.py | 8 +------- lib/spack/spack/package.py | 9 --------- 3 files changed, 22 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 4bbf7bb34c..fde2a8805e 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -343,7 +343,7 @@ class URLFetchStrategy(FetchStrategy): def __repr__(self): url = self.url if self.url else "no url" - return "URLFetchStrategy<%s>" % url + return "%s<%s>" % (self.__class__.__name__, url) def __str__(self): if self.url: @@ -352,6 +352,25 @@ class URLFetchStrategy(FetchStrategy): return "[no url]" +class URLMirrorFetchStrategy(URLFetchStrategy): + """The resource associated with a URL at a mirror may be out of date. + """ + def __init__(self, *args, **kwargs): + super(URLMirrorFetchStrategy, self).__init__(*args, **kwargs) + + @_needs_stage + def fetch(self): + super(URLMirrorFetchStrategy, self).fetch() + if self.digest: + try: + self.check() + except ChecksumError: + # Future fetchers will assume they don't need to download if the + # file remains + os.remove(self.archive_file) + raise + + class VCSFetchStrategy(FetchStrategy): def __init__(self, name, *rev_types, **kwargs): super(VCSFetchStrategy, self).__init__() @@ -816,7 +835,7 @@ class FsCache(object): def fetcher(self, targetPath, digest): url = "file://" + join_path(self.root, targetPath) - return URLFetchStrategy(url, digest) + return URLMirrorFetchStrategy(url, digest) class FetchError(spack.error.SpackError): diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 81a2f6afb7..0bbcfba6b4 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -61,13 +61,7 @@ def mirror_archive_filename(spec, fetcher): # Otherwise we'll make a .tar.gz ourselves ext = 'tar.gz' - tokens = [spec.package.name, spec.version] - digests = spec.package.digests - if digests: - # If a package has multiple digests, any one is sufficient to identify it - digestType, digest = sorted(digests.iteritems())[0] - tokens.extend([digestType, digest]) - filename = '-'.join(str(t) for t in tokens) + filename = "%s-%s" % (spec.package.name, spec.version) if ext: filename += ".%s" % ext return filename diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 70ec243726..cbf50e56f6 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -413,15 +413,6 @@ class Package(object): raise ValueError("Can only get of package with concrete version.") return self.spec.versions[0] - @property - def digests(self): - """All digests for the concretized package version.""" - versionInfo = self.versions[self.version] - digests = {} - if 'md5' in versionInfo: - digests['md5'] = versionInfo['md5'] - return digests - @memoized def version_urls(self): """Return a list of URLs for different versions of this -- cgit v1.2.3-60-g2f50 From a2754894ea67e0e751121411ff92d60ca68ab089 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Jun 2016 16:26:54 -0700 Subject: (1) FsCache store now takes a fetcher vs. just a copy command (2) use [1] to conditionally cache resource: only save it if there is a feature which identifies it uniquely (for example do not cache a repository if it pulls the latest state vs. a particular tag/commit) --- lib/spack/spack/fetch_strategy.py | 16 ++++++++++++++-- lib/spack/spack/stage.py | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index fde2a8805e..3221716b91 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -828,10 +828,22 @@ class FsCache(object): def __init__(self, root): self.root = os.path.abspath(root) - def store(self, copyCmd, relativeDst): + def store(self, fetcher, relativeDst): + unique = False + uidGroups = [['tag', 'commit'], ['digest'], ['revision']] + for grp in uidGroups: + try: + unique |= any(getattr(fetcher, x) for x in grp) + except AttributeError: + pass + if unique: + break + if not unique: + return + dst = join_path(self.root, relativeDst) mkdirp(os.path.dirname(dst)) - copyCmd(dst) + fetcher.archive(dst) def fetcher(self, targetPath, digest): url = "file://" + join_path(self.root, targetPath) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index f28934d10a..b08cce43b8 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -337,7 +337,7 @@ class Stage(object): def cache_local(self): - spack.cache.store(self.fetcher.archive, self.mirror_path) + spack.cache.store(self.fetcher, self.mirror_path) def expand_archive(self): -- cgit v1.2.3-60-g2f50 From 3b71d78f3cddeb1b622b9a49146adadbd1b1d9dc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 8 Jun 2016 09:57:56 -0700 Subject: rename URLMirrorFetchStrategy to CacheURLFetchStrategy since it isnt used to manage all mirror URLs - just the cache (the specific behavior that a URL may refer to a stale resource doesn't necessarily apply to mirrors) --- lib/spack/spack/fetch_strategy.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 3221716b91..2607d0a7f4 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -352,15 +352,14 @@ class URLFetchStrategy(FetchStrategy): return "[no url]" -class URLMirrorFetchStrategy(URLFetchStrategy): - """The resource associated with a URL at a mirror may be out of date. - """ +class CacheURLFetchStrategy(URLFetchStrategy): + """The resource associated with a cache URL may be out of date.""" def __init__(self, *args, **kwargs): - super(URLMirrorFetchStrategy, self).__init__(*args, **kwargs) + super(CacheURLFetchStrategy, self).__init__(*args, **kwargs) @_needs_stage def fetch(self): - super(URLMirrorFetchStrategy, self).fetch() + super(CacheURLFetchStrategy, self).fetch() if self.digest: try: self.check() @@ -847,7 +846,7 @@ class FsCache(object): def fetcher(self, targetPath, digest): url = "file://" + join_path(self.root, targetPath) - return URLMirrorFetchStrategy(url, digest) + return CacheURLFetchStrategy(url, digest) class FetchError(spack.error.SpackError): -- cgit v1.2.3-60-g2f50 From 8f846d507d7523d5ce6f0a90b33d9409bab2a72c Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Wed, 22 Jun 2016 15:36:04 -0400 Subject: Added --prefix to module command. --- lib/spack/spack/cmd/module.py | 48 ++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index b79737ab22..443e95a3c2 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -1,49 +1,45 @@ ############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # # For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. +# conditions of the GNU Lesser General Public License for more details. # -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys +from __future__ import print_function import os import shutil -import argparse +import sys import llnl.util.tty as tty -from llnl.util.lang import partition_list -from llnl.util.filesystem import mkdirp - import spack.cmd +from llnl.util.filesystem import mkdirp from spack.modules import module_types from spack.util.string import * -from spack.spec import Spec - -description ="Manipulate modules and dotkits." +description = "Manipulate modules and dotkits." def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='module_command') - refresh_parser = sp.add_parser('refresh', help='Regenerate all module files.') + sp.add_parser('refresh', help='Regenerate all module files.') find_parser = sp.add_parser('find', help='Find module files for packages.') find_parser.add_argument( @@ -56,6 +52,10 @@ def setup_parser(subparser): '-s', '--shell', action='store_true', dest='shell', help='Generate shell script (instead of input for module command)') + find_parser.add_argument( + '-p', '--prefix', dest='prefix', + help='Prepend to module names when issuing module load commands') + find_parser.add_argument('spec', nargs='+', help='spec to find a module file for.') @@ -66,6 +66,9 @@ def module_find(mtype, flags, spec_array): matches any. If it does, check whether there is a module file of type there, and print out the name that the user should type to use that package's module. + prefix: + Prepend this to module names when issuing "module load" commands. + Some systems seem to need it. """ # -------------------------------------- @@ -88,7 +91,8 @@ def module_find(mtype, flags, spec_array): # -------------------------------------- if mtype not in module_types: - tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types))) + tty.die("Invalid module type: '%s'. Options are %s" % + (mtype, comma_or(module_types))) raw_specs = spack.cmd.parse_specs(spec_array) modules = set() # Modules we will load @@ -123,10 +127,10 @@ def module_find(mtype, flags, spec_array): module_cmd = {'tcl' : 'module load', 'dotkit' : 'dotkit use'}[mtype] for spec,mod in modules_unique: if flags.shell: - print '# %s' % spec.format() - print '%s %s' % (module_cmd, mod.use_name) + print('# %s' % spec.format()) + print('%s %s%s' % (module_cmd, flags.prefix, mod.use_name)) else: - print mod.use_name + print(mod.use_name) def module_refresh(): """Regenerate all module files for installed packages known to @@ -139,11 +143,9 @@ def module_refresh(): shutil.rmtree(cls.path, ignore_errors=False) mkdirp(cls.path) for spec in specs: - tty.debug(" Writing file for %s" % spec) cls(spec).write() - def module(parser, args): if args.module_command == 'refresh': module_refresh() -- cgit v1.2.3-60-g2f50 From ea88d9cfe9b13a5335b367d74a9008f60a5d9c53 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 24 Jun 2016 16:43:31 -0500 Subject: Make spack create automatically add dependencies for build system --- lib/spack/spack/cmd/create.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 8cbb367f86..71bd196598 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -95,9 +95,9 @@ class ${class_name}(Package): url = "${url}" ${versions} -${extends} - # FIXME: Add dependencies if this package requires them. - # depends_on("foo") + + # FIXME: Add additional dependencies if required. + ${dependencies} def install(self, spec, prefix): ${install} @@ -136,8 +136,17 @@ def setup_parser(subparser): class ConfigureGuesser(object): def __call__(self, stage): """Try to guess the type of build system used by the project. - Set the appropriate default installation instructions and any - necessary extensions for Python and R.""" + Set any necessary build dependencies or extensions. + Set the appropriate default installation instructions.""" + + # Build dependencies and extensions + dependenciesDict = { + 'autotools': "# depends_on('foo')", + 'cmake': "depends_on('cmake')", + 'scons': "depends_on('scons')", + 'python': "extends('python')", + 'R': "extends('R')" + } # Default installation instructions installDict = { @@ -214,16 +223,12 @@ class ConfigureGuesser(object): self.build_system = build_system + # Set any necessary build dependencies or extensions. + self.dependencies = dependenciesDict[build_system] + # Set the appropriate default installation instructions self.install = installDict[build_system] - # Set any necessary extensions for Python and R - extensions = '' - if build_system in ['python', 'R']: - extensions = "\n extends('{0}')\n".format(build_system) - - self.extends = extensions - def guess_name_and_version(url, args): # Try to deduce name and version of the new package from the URL @@ -361,7 +366,7 @@ def create(parser, args): class_name=mod_to_class(name), url=url, versions=make_version_calls(ver_hash_tuples), - extends=guesser.extends, + dependencies=guesser.dependencies, install=guesser.install)) # If everything checks out, go ahead and edit. -- cgit v1.2.3-60-g2f50 From 798dabc8f2b740cb70563fa141beb0ce268ce09a Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 27 Jun 2016 09:01:53 -0500 Subject: Forgot the unknown key/value pair --- lib/spack/spack/cmd/create.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 71bd196598..f5f234d7a7 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -145,7 +145,8 @@ class ConfigureGuesser(object): 'cmake': "depends_on('cmake')", 'scons': "depends_on('scons')", 'python': "extends('python')", - 'R': "extends('R')" + 'R': "extends('R')", + 'unknown': "# depends_on('foo')" } # Default installation instructions -- cgit v1.2.3-60-g2f50 From c99663211379389f416c5ba50a759555e2b1f9ec Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 29 Jun 2016 01:41:55 -0700 Subject: Add spack purge --cache to purge local archive cache. --- .gitignore | 1 + lib/spack/spack/cmd/purge.py | 26 ++++++++++++++++++++++++-- lib/spack/spack/fetch_strategy.py | 7 +++++-- 3 files changed, 30 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/.gitignore b/.gitignore index 643e5d9b03..040df3eafd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /var/spack/stage +/var/spack/cache *.pyc /opt/ *~ diff --git a/lib/spack/spack/cmd/purge.py b/lib/spack/spack/cmd/purge.py index 7b33ef7f69..f4e27a3969 100644 --- a/lib/spack/spack/cmd/purge.py +++ b/lib/spack/spack/cmd/purge.py @@ -22,9 +22,31 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import spack import spack.stage as stage -description = "Remove all temporary build files and downloaded archives" +description = "Remove temporary build files and/or downloaded archives" + + +def setup_parser(subparser): + subparser.add_argument( + '-s', '--stage', action='store_true', default=True, + help="Remove all temporary build stages (default).") + subparser.add_argument( + '-c', '--cache', action='store_true', help="Remove cached downloads.") + subparser.add_argument( + '-a', '--all', action='store_true', + help="Remove all of the above.") + def purge(parser, args): - stage.purge() + # Special case: no flags. + if not any((args.stage, args.cache, args.all)): + stage.purge() + return + + # handle other flags with fall through. + if args.stage or args.all: + stage.purge() + if args.cache or args.all: + spack.cache.destroy() diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 6f28ec34b2..69c04f7920 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -361,7 +361,7 @@ class CacheURLFetchStrategy(URLFetchStrategy): """The resource associated with a cache URL may be out of date.""" def __init__(self, *args, **kwargs): super(CacheURLFetchStrategy, self).__init__(*args, **kwargs) - + @_needs_stage def fetch(self): super(CacheURLFetchStrategy, self).fetch() @@ -853,11 +853,14 @@ class FsCache(object): dst = join_path(self.root, relativeDst) mkdirp(os.path.dirname(dst)) fetcher.archive(dst) - + def fetcher(self, targetPath, digest): url = "file://" + join_path(self.root, targetPath) return CacheURLFetchStrategy(url, digest) + def destroy(self): + shutil.rmtree(self.root, ignore_errors=True) + class FetchError(spack.error.SpackError): -- cgit v1.2.3-60-g2f50 From 06de8c56af31880fe8f304fb3b5a148c7c9f3646 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 29 Jun 2016 12:35:38 -0500 Subject: Fix missing code blocks in licensed software documentation --- lib/spack/docs/packaging_guide.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 54b886310a..433760f8c8 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -776,7 +776,7 @@ Spack will create a global license file located at file using the editor set in ``$EDITOR``, or vi if unset. It will look like this: -.. code-block:: +.. code-block:: sh # A license is required to use pgi. # @@ -807,7 +807,7 @@ You can add your license directly to this file, or tell FlexNet to use a license stored on a separate license server. Here is an example that points to a license server called licman1: -.. code-block:: +.. code-block:: sh SERVER licman1.mcs.anl.gov 00163eb7fba5 27200 USE_SERVER -- cgit v1.2.3-60-g2f50 From cfd380d514606754c4ee3faceb4f00287fce0876 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 29 Jun 2016 18:23:40 -0700 Subject: added documentation for automatic caching of resources fetched during installs --- lib/spack/docs/developer_guide.rst | 9 +++++---- lib/spack/docs/mirrors.rst | 18 ++++++++++++++++++ lib/spack/docs/packaging_guide.rst | 20 ++++++++++++++++---- 3 files changed, 39 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index 0b618aa683..d28fe4b2a5 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -80,10 +80,11 @@ with a high level view of Spack's directory structure:: var/ spack/ <- build & stage directories - repos/ <- contains package repositories - builtin/ <- pkg repository that comes with Spack - repo.yaml <- descriptor for the builtin repository - packages/ <- directories under here contain packages + repos/ <- contains package repositories + builtin/ <- pkg repository that comes with Spack + repo.yaml <- descriptor for the builtin repository + packages/ <- directories under here contain packages + cache/ <- saves resources downloaded during installs opt/ spack/ <- packages are installed here diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index dad04d053b..583575a565 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -214,3 +214,21 @@ Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``:: If you want to change the order in which mirrors are searched for packages, you can edit this file and reorder the sections. Spack will search the topmost mirror first and the bottom-most mirror last. + +.. _caching: + +Local Default Cache +---------------------------- + +Spack caches resources that are downloaded as part of installs. The cache is +a valid spack mirror: it uses the same directory structure and naming scheme +as other Spack mirrors (so it can be copied anywhere and referenced with a URL +like other mirrors). The mirror is maintained locally (within the Spack +installation directory) at :file:`var/spack/cache/`. It is always enabled (and +is always searched first when attempting to retrieve files for an installation) +but can be cleared with :ref:`purge `; the cache directory can also +be deleted manually without issue. + +Caching includes retrieved tarball archives and source control repositories, but +only resources with an associated digest or commit ID (e.g. a revision number +for SVN) will be cached. diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 54b886310a..2c3d139d29 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -703,6 +703,13 @@ Fetching a revision Subversion branches are handled as part of the directory structure, so you can check out a branch or tag by changing the ``url``. +Automatic caching of files fetched during installation +------------------------------------------------------ + +Spack maintains a cache (described :ref:`here `) which saves files +retrieved during package installations to avoid re-downloading in the case that +a package is installed with a different specification (but the same version) or +reinstalled on account of a change in the hashing scheme. .. _license: @@ -2618,11 +2625,16 @@ build process will start from scratch. ``spack purge`` ~~~~~~~~~~~~~~~~~ -Cleans up all of Spack's temporary files. Use this to recover disk -space if temporary files from interrupted or failed installs -accumulate in the staging area. This is equivalent to running ``spack -clean`` for every package you have fetched or staged. +Cleans up all of Spack's temporary and cached files. This can be used to +recover disk space if temporary files from interrupted or failed installs +accumulate in the staging area. + +When called with ``--stage`` or ``--all`` (or without arguments, in which case +the default is ``--all``) this removes all staged files; this is equivalent to +running ``spack clean`` for every package you have fetched or staged. +When called with ``--cache`` or ``--all`` this will clear all resources +:ref:`cached ` during installs. Keeping the stage directory on success ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- cgit v1.2.3-60-g2f50