From 102ac7bcf1bc7fd134b10a9c54e40302d4f1345b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 9 Aug 2016 00:24:54 -0700 Subject: Move provider cache to home directory and refactor Transactions Major stuff: - Created a FileCache for managing user cache files in Spack. Currently just handles virtuals. - Moved virtual cache from the repository to the home directory so that users do not need write access to Spack repositories to use them. - Refactored `Transaction` class in `database.py` -- moved it to `LockTransaction` in `lock.py` and made it reusable by other classes. Other additions: - Added tests for file cache and transactions. - Added a few more tests for database - Fixed bug in DB where writes could happen even if exceptions were raised during a transaction. - `spack uninstall` now attempts to repair the database when it discovers that a prefix doesn't exist but a DB record does. --- lib/spack/llnl/util/lock.py | 69 ++++++++++++- lib/spack/spack/__init__.py | 9 +- lib/spack/spack/cmd/purge.py | 10 +- lib/spack/spack/cmd/test.py | 10 +- lib/spack/spack/cmd/uninstall.py | 3 +- lib/spack/spack/config.py | 2 +- lib/spack/spack/database.py | 61 +++-------- lib/spack/spack/file_cache.py | 181 +++++++++++++++++++++++++++++++++ lib/spack/spack/modules.py | 3 +- lib/spack/spack/package.py | 10 +- lib/spack/spack/repository.py | 58 +++-------- lib/spack/spack/stage.py | 5 +- lib/spack/spack/test/__init__.py | 1 + lib/spack/spack/test/database.py | 34 +++++++ lib/spack/spack/test/file_cache.py | 84 +++++++++++++++ lib/spack/spack/test/lock.py | 163 ++++++++++++++++++++++++++++- lib/spack/spack/test/mock_database.py | 6 +- lib/spack/spack/test/provider_index.py | 4 - 18 files changed, 600 insertions(+), 113 deletions(-) create mode 100644 lib/spack/spack/file_cache.py create mode 100644 lib/spack/spack/test/file_cache.py (limited to 'lib') diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py index 479a1b0167..e1f5b4878a 100644 --- a/lib/spack/llnl/util/lock.py +++ b/lib/spack/llnl/util/lock.py @@ -28,6 +28,9 @@ import errno import time import socket +__all__ = ['Lock', 'LockTransaction', 'WriteTransaction', 'ReadTransaction', + 'LockError'] + # Default timeout in seconds, after which locks will raise exceptions. _default_timeout = 60 @@ -63,7 +66,9 @@ class Lock(object): fcntl.lockf(self._fd, op | fcntl.LOCK_NB) if op == fcntl.LOCK_EX: - os.write(self._fd, "pid=%s,host=%s" % (os.getpid(), socket.getfqdn())) + os.write( + self._fd, + "pid=%s,host=%s" % (os.getpid(), socket.getfqdn())) return except IOError as error: @@ -170,6 +175,66 @@ class Lock(object): return False +class LockTransaction(object): + """Simple nested transaction context manager that uses a file lock. + + This class can trigger actions when the lock is acquired for the + first time and released for the last. + + If the acquire_fn returns a value, it is used as the return value for + __enter__, allowing it to be passed as the `as` argument of a `with` + statement. + + If acquire_fn returns a context manager, *its* `__enter__` function will be + called in `__enter__` after acquire_fn, and its `__exit__` funciton will be + called before `release_fn` in `__exit__`, allowing you to nest a context + manager to be used along with the lock. + + Timeout for lock is customizable. + + """ + + def __init__(self, lock, acquire_fn=None, release_fn=None, + timeout=_default_timeout): + self._lock = lock + self._timeout = timeout + self._acquire_fn = acquire_fn + self._release_fn = release_fn + self._as = None + + def __enter__(self): + if self._enter() and self._acquire_fn: + self._as = self._acquire_fn() + if hasattr(self._as, '__enter__'): + return self._as.__enter__() + else: + return self._as + + def __exit__(self, type, value, traceback): + if self._exit(): + if self._as and hasattr(self._as, '__exit__'): + self._as.__exit__(type, value, traceback) + if self._release_fn: + self._release_fn(type, value, traceback) + if value: + raise value + + +class ReadTransaction(LockTransaction): + def _enter(self): + return self._lock.acquire_read(self._timeout) + + def _exit(self): + return self._lock.release_read() + + +class WriteTransaction(LockTransaction): + def _enter(self): + return self._lock.acquire_write(self._timeout) + + def _exit(self): + return self._lock.release_write() + + class LockError(Exception): """Raised when an attempt to acquire a lock times out.""" - pass diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index d67585aac4..a6e21987c8 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -50,8 +50,15 @@ repos_path = join_path(var_path, "repos") share_path = join_path(spack_root, "share", "spack") cache_path = join_path(var_path, "cache") +# User configuration location +user_config_path = os.path.expanduser('~/.spack') + import spack.fetch_strategy -cache = spack.fetch_strategy.FsCache(cache_path) +fetch_cache = spack.fetch_strategy.FsCache(cache_path) + +from spack.file_cache import FileCache +user_cache_path = join_path(user_config_path, 'cache') +user_cache = FileCache(user_cache_path) prefix = spack_root opt_path = join_path(prefix, "opt") diff --git a/lib/spack/spack/cmd/purge.py b/lib/spack/spack/cmd/purge.py index f4e27a3969..26745810a8 100644 --- a/lib/spack/spack/cmd/purge.py +++ b/lib/spack/spack/cmd/purge.py @@ -33,7 +33,11 @@ def setup_parser(subparser): '-s', '--stage', action='store_true', default=True, help="Remove all temporary build stages (default).") subparser.add_argument( - '-c', '--cache', action='store_true', help="Remove cached downloads.") + '-d', '--downloads', action='store_true', + help="Remove cached downloads.") + subparser.add_argument( + '-u', '--user-cache', action='store_true', + help="Remove caches in user home directory. Includes virtual indices.") subparser.add_argument( '-a', '--all', action='store_true', help="Remove all of the above.") @@ -49,4 +53,6 @@ def purge(parser, args): if args.stage or args.all: stage.purge() if args.cache or args.all: - spack.cache.destroy() + spack.fetch_cache.destroy() + if args.user_cache or args.all: + spack.user_cache.destroy() diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index 36810321ef..2667b42820 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -41,10 +41,10 @@ def setup_parser(subparser): subparser.add_argument( '-l', '--list', action='store_true', dest='list', help="Show available tests") subparser.add_argument( - '--createXmlOutput', action='store_true', dest='createXmlOutput', + '--createXmlOutput', action='store_true', dest='createXmlOutput', help="Create JUnit XML from test results") subparser.add_argument( - '--xmlOutputDir', dest='xmlOutputDir', + '--xmlOutputDir', dest='xmlOutputDir', help="Nose creates XML files in this directory") subparser.add_argument( '-v', '--verbose', action='store_true', dest='verbose', @@ -62,7 +62,7 @@ class MockCache(object): class MockCacheFetcher(object): def set_stage(self, stage): pass - + def fetch(self): raise FetchError("Mock cache always fails for tests") @@ -82,8 +82,8 @@ def test(parser, args): outputDir = join_path(os.getcwd(), "test-output") else: outputDir = os.path.abspath(args.xmlOutputDir) - + if not os.path.exists(outputDir): mkdirp(outputDir) - spack.cache = MockCache() + spack.fetch_cache = MockCache() spack.test.run(args.names, outputDir, args.verbose) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index a17b7c685c..dbe6cd6584 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -184,7 +184,8 @@ def uninstall(parser, args): uninstall_list = list(set(uninstall_list)) if has_error: - tty.die('You can use spack uninstall --dependents to uninstall these dependencies as well') # NOQA: ignore=E501 + tty.die('You can use spack uninstall --dependents ' + 'to uninstall these dependencies as well') if not args.yes_to_all: tty.msg("The following packages will be uninstalled : ") diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 31f0eb3a56..a4e274893c 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -525,7 +525,7 @@ ConfigScope('defaults', os.path.join(spack.etc_path, 'spack', 'defaults')) ConfigScope('site', os.path.join(spack.etc_path, 'spack')) """User configuration can override both spack defaults and site config.""" -ConfigScope('user', os.path.expanduser('~/.spack')) +ConfigScope('user', spack.user_config_path) def highest_precedence_scope(): diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 317b0d5784..5ce42b2e67 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -165,11 +165,11 @@ class Database(object): def write_transaction(self, timeout=_db_lock_timeout): """Get a write lock context manager for use in a `with` block.""" - return WriteTransaction(self, self._read, self._write, timeout) + return WriteTransaction(self.lock, self._read, self._write, timeout) def read_transaction(self, timeout=_db_lock_timeout): """Get a read lock context manager for use in a `with` block.""" - return ReadTransaction(self, self._read, None, timeout) + return ReadTransaction(self.lock, self._read, timeout=timeout) def _write_to_yaml(self, stream): """Write out the databsae to a YAML file. @@ -352,12 +352,22 @@ class Database(object): "Invalid ref_count: %s: %d (expected %d), in DB %s" % (key, found, expected, self._index_path)) - def _write(self): + def _write(self, type, value, traceback): """Write the in-memory database index to its file path. - Does no locking. + This is a helper function called by the WriteTransaction context + manager. If there is an exception while the write lock is active, + nothing will be written to the database file, but the in-memory database + *may* be left in an inconsistent state. It will be consistent after the + start of the next transaction, when it read from disk again. + + This routine does no locking. """ + # Do not write if exceptions were raised + if type is not None: + return + temp_file = self._index_path + ( '.%s.%s.temp' % (socket.getfqdn(), os.getpid())) @@ -589,49 +599,6 @@ class Database(object): return key in self._data and not self._data[key].installed -class _Transaction(object): - """Simple nested transaction context manager that uses a file lock. - - This class can trigger actions when the lock is acquired for the - first time and released for the last. - - Timeout for lock is customizable. - """ - - def __init__(self, db, - acquire_fn=None, - release_fn=None, - timeout=_db_lock_timeout): - self._db = db - self._timeout = timeout - self._acquire_fn = acquire_fn - self._release_fn = release_fn - - def __enter__(self): - if self._enter() and self._acquire_fn: - self._acquire_fn() - - def __exit__(self, type, value, traceback): - if self._exit() and self._release_fn: - self._release_fn() - - -class ReadTransaction(_Transaction): - def _enter(self): - return self._db.lock.acquire_read(self._timeout) - - def _exit(self): - return self._db.lock.release_read() - - -class WriteTransaction(_Transaction): - def _enter(self): - return self._db.lock.acquire_write(self._timeout) - - def _exit(self): - return self._db.lock.release_write() - - class CorruptDatabaseError(SpackError): def __init__(self, path, msg=''): super(CorruptDatabaseError, self).__init__( diff --git a/lib/spack/spack/file_cache.py b/lib/spack/spack/file_cache.py new file mode 100644 index 0000000000..2124df9c9c --- /dev/null +++ b/lib/spack/spack/file_cache.py @@ -0,0 +1,181 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import shutil + +from llnl.util.filesystem import * +from llnl.util.lock import * + +import spack +from spack.error import SpackError + + +class FileCache(object): + """This class manages cached data in the filesystem. + + - Cache files are fetched and stored by unique keys. Keys can be relative + paths, so that thre can be some hierarchy in the cache. + + - The FileCache handles locking cache files for reading and writing, so + client code need not manage locks for cache entries. + + """ + def __init__(self, root): + """Create a file cache object. + + This will create the cache directory if it does not exist yet. + + """ + self.root = root.rstrip(os.path.sep) + if not os.path.exists(self.root): + mkdirp(self.root) + + self._locks = {} + + def purge(self): + """Remove all files under the cache root.""" + for f in os.listdir(self.root): + path = join_path(self.root, f) + shutil.rmtree(f) + + def cache_path(self, key): + """Path to the file in the cache for a particular key.""" + return join_path(self.root, key) + + def _lock_path(self, key): + """Path to the file in the cache for a particular key.""" + keyfile = os.path.basename(key) + keydir = os.path.dirname(key) + + return join_path(self.root, keydir, '.' + keyfile + '.lock') + + def _get_lock(self, key): + """Create a lock for a key, if necessary, and return a lock object.""" + if key not in self._locks: + lock_file = self._lock_path(key) + if not os.path.exists(lock_file): + touch(lock_file) + self._locks[key] = Lock(lock_file) + return self._locks[key] + + def init_entry(self, key): + """Ensure we can access a cache file. Create a lock for it if needed. + + Return whether the cache file exists yet or not. + """ + cache_path = self.cache_path(key) + + exists = os.path.exists(cache_path) + if exists: + if not os.path.isfile(cache_path): + raise CacheError("Cache file is not a file: %s" % cache_path) + + if not os.access(cache_path, os.R_OK|os.W_OK): + raise CacheError("Cannot access cache file: %s" % cache_path) + else: + # if the file is hierarchical, make parent directories + parent = os.path.dirname(cache_path) + if parent.rstrip(os.path.sep) != self.root: + mkdirp(parent) + + if not os.access(parent, os.R_OK|os.W_OK): + raise CacheError("Cannot access cache directory: %s" % parent) + + # ensure lock is created for this key + self._get_lock(key) + return exists + + def read_transaction(self, key): + """Get a read transaction on a file cache item. + + Returns a ReadTransaction context manager and opens the cache file for + reading. You can use it like this: + + with spack.user_cache.read_transaction(key) as cache_file: + cache_file.read() + + """ + return ReadTransaction( + self._get_lock(key), lambda: open(self.cache_path(key))) + + def write_transaction(self, key): + """Get a write transaction on a file cache item. + + Returns a WriteTransaction context manager that opens a temporary file + for writing. Once the context manager finishes, if nothing went wrong, + moves the file into place on top of the old file atomically. + + """ + class WriteContextManager(object): + def __enter__(cm): + cm.orig_filename = self.cache_path(key) + cm.orig_file = None + if os.path.exists(cm.orig_filename): + cm.orig_file = open(cm.orig_filename, 'r') + + cm.tmp_filename = self.cache_path(key) + '.tmp' + cm.tmp_file = open(cm.tmp_filename, 'w') + + return cm.orig_file, cm.tmp_file + + def __exit__(cm, type, value, traceback): + if cm.orig_file: + cm.orig_file.close() + cm.tmp_file.close() + + if value: + # remove tmp on exception & raise it + shutil.rmtree(cm.tmp_filename, True) + raise value + else: + os.rename(cm.tmp_filename, cm.orig_filename) + + return WriteTransaction(self._get_lock(key), WriteContextManager) + + + def mtime(self, key): + """Return modification time of cache file, or 0 if it does not exist. + + Time is in units returned by os.stat in the mtime field, which is + platform-dependent. + + """ + if not self.init_entry(key): + return 0 + else: + sinfo = os.stat(self.cache_path(key)) + return sinfo.st_mtime + + + def remove(self, key): + lock = self._get_lock(key) + try: + lock.acquire_write() + os.unlink(self.cache_path(key)) + finally: + lock.release_write() + os.unlink(self._lock_path(key)) + +class CacheError(SpackError): pass diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 8701a31c49..8ac6a77d13 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -520,7 +520,8 @@ class Dotkit(EnvModule): def prerequisite(self, spec): tty.warn('prerequisites: not supported by dotkit module files') - tty.warn('\tYou may want to check ~/.spack/modules.yaml') + tty.warn('\tYou may want to check %s/modules.yaml' + % spack.user_config_path) return '' diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 43aefbf65e..475155937c 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1198,7 +1198,15 @@ class Package(object): def do_uninstall(self, force=False): if not self.installed: - raise InstallError(str(self.spec) + " is not installed.") + # prefix may not exist, but DB may be inconsistent. Try to fix by + # removing, but omit hooks. + specs = spack.installed_db.query(self.spec, installed=True) + if specs: + spack.installed_db.remove(specs[0]) + tty.msg("Removed stale DB entry for %s" % self.spec.short_spec) + return + else: + raise InstallError(str(self.spec) + " is not installed.") if not force: dependents = self.installed_dependents diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 58747ba25d..a0904a2cde 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -41,6 +41,7 @@ import llnl.util.tty as tty from llnl.util.lock import Lock from llnl.util.filesystem import * +import spack import spack.error import spack.config import spack.spec @@ -414,17 +415,6 @@ class Repo(object): check(os.path.isdir(self.packages_path), "No directory '%s' found in '%s'" % (repo_config_name, root)) - self.index_file = join_path(self.root, repo_index_name) - check(not os.path.exists(self.index_file) or - (os.path.isfile(self.index_file) and os.access(self.index_file, os.R_OK|os.W_OK)), - "Cannot access repository index file in %s" % root) - - # lock file for reading/writing the index - self._lock_path = join_path(self.root, 'lock') - if not os.path.exists(self._lock_path): - touch(self._lock_path) - self._lock = Lock(self._lock_path) - # Read configuration and validate namespace config = self._read_config() check('namespace' in config, '%s must define a namespace.' @@ -461,6 +451,8 @@ class Repo(object): # make sure the namespace for packages in this repo exists. self._create_namespace() + # Unique filename for cache of virtual dependency providers + self._cache_file = 'providers/%s-index.yaml' % self.namespace def _create_namespace(self): """Create this repo's namespace module and insert it into sys.modules. @@ -658,21 +650,15 @@ class Repo(object): self._provider_index = ProviderIndex.from_yaml(f) # Read the old ProviderIndex, or make a new one. - index_existed = os.path.isfile(self.index_file) + key = self._cache_file + index_existed = spack.user_cache.init_entry(key) if index_existed and not self._needs_update: - self._lock.acquire_read() - try: - read() - finally: - self._lock.release_read() - + with spack.user_cache.read_transaction(key) as f: + self._provider_index = ProviderIndex.from_yaml(f) else: - tmp = self.index_file + '.tmp' - self._lock.acquire_write() - try: - if index_existed: - with open(self.index_file) as f: - self._provider_index = ProviderIndex.from_yaml(f) + with spack.user_cache.write_transaction(key) as (old, new): + if old: + self._provider_index = ProviderIndex.from_yaml(old) else: self._provider_index = ProviderIndex() @@ -681,17 +667,7 @@ class Repo(object): self._provider_index.remove_provider(namespaced_name) self._provider_index.update(namespaced_name) - - with open(tmp, 'w') as f: - self._provider_index.to_yaml(f) - os.rename(tmp, self.index_file) - - except: - shutil.rmtree(tmp, ignore_errors=True) - raise - - finally: - self._lock.release_write() + self._provider_index.to_yaml(new) @property @@ -745,7 +721,7 @@ class Repo(object): def _fast_package_check(self): - """List packages in the repo and cehck whether index is up to date. + """List packages in the repo and check whether index is up to date. Both of these opreations require checking all `package.py` files so we do them at the same time. We list the repo @@ -763,10 +739,7 @@ class Repo(object): self._all_package_names = [] # Get index modification time. - index_mtime = 0 - if os.path.exists(self.index_file): - sinfo = os.stat(self.index_file) - index_mtime = sinfo.st_mtime + index_mtime = spack.user_cache.mtime(self._cache_file) for pkg_name in os.listdir(self.packages_path): # Skip non-directories in the package root. @@ -774,8 +747,9 @@ class Repo(object): # Warn about invalid names that look like packages. if not valid_module_name(pkg_name): - tty.warn("Skipping package at %s. '%s' is not a valid Spack module name." - % (pkg_dir, pkg_name)) + msg = ("Skipping package at %s. " + "'%s' is not a valid Spack module name.") + tty.warn(msg % (pkg_dir, pkg_name)) continue # construct the file name from the directory diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 8fcc331482..7676cb9ab6 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -315,7 +315,8 @@ class Stage(object): # Add URL strategies for all the mirrors with the digest for url in urls: fetchers.insert(0, fs.URLFetchStrategy(url, digest)) - fetchers.insert(0, spack.cache.fetcher(self.mirror_path, digest)) + fetchers.insert(0, spack.fetch_cache.fetcher(self.mirror_path, + digest)) # Look for the archive in list_url package_name = os.path.dirname(self.mirror_path) @@ -365,7 +366,7 @@ class Stage(object): self.fetcher.check() def cache_local(self): - spack.cache.store(self.fetcher, self.mirror_path) + spack.fetch_cache.store(self.fetcher, self.mirror_path) def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 7795cb59c7..4969081e63 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -49,6 +49,7 @@ test_names = [ 'database', 'directory_layout', 'environment', + 'file_cache', 'git_fetch', 'hg_fetch', 'install', diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index e1322f2081..a2f09450bc 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -273,3 +273,37 @@ class DatabaseTest(MockDatabase): # mpich ref count updated properly. mpich_rec = self.installed_db.get_record('mpich') self.assertEqual(mpich_rec.ref_count, 0) + + def test_100_no_write_with_exception_on_remove(self): + def fail_while_writing(): + with self.installed_db.write_transaction(): + self._mock_remove('mpileaks ^zmpi') + raise Exception() + + with self.installed_db.read_transaction(): + self.assertEqual( + len(self.installed_db.query('mpileaks ^zmpi', installed=any)), 1) + + self.assertRaises(Exception, fail_while_writing) + + # reload DB and make sure zmpi is still there. + with self.installed_db.read_transaction(): + self.assertEqual( + len(self.installed_db.query('mpileaks ^zmpi', installed=any)), 1) + + def test_110_no_write_with_exception_on_install(self): + def fail_while_writing(): + with self.installed_db.write_transaction(): + self._mock_install('cmake') + raise Exception() + + with self.installed_db.read_transaction(): + self.assertEqual( + self.installed_db.query('cmake', installed=any), []) + + self.assertRaises(Exception, fail_while_writing) + + # reload DB and make sure cmake was not written. + with self.installed_db.read_transaction(): + self.assertEqual( + self.installed_db.query('cmake', installed=any), []) diff --git a/lib/spack/spack/test/file_cache.py b/lib/spack/spack/test/file_cache.py new file mode 100644 index 0000000000..6142b135eb --- /dev/null +++ b/lib/spack/spack/test/file_cache.py @@ -0,0 +1,84 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +""" +Test Spack's FileCache. +""" +import os +import shutil +import tempfile +import unittest + +import spack +from spack.file_cache import FileCache + + +class FileCacheTest(unittest.TestCase): + """Ensure that a file cache can properly write to a file and recover its + contents.""" + + def setUp(self): + self.scratch_dir = tempfile.mkdtemp() + self.cache = FileCache(self.scratch_dir) + + def tearDown(self): + shutil.rmtree(self.scratch_dir) + + def test_write_and_read_cache_file(self): + """Test writing then reading a cached file.""" + with self.cache.write_transaction('test.yaml') as (old, new): + self.assertTrue(old is None) + self.assertTrue(new is not None) + new.write("foobar\n") + + with self.cache.read_transaction('test.yaml') as stream: + text = stream.read() + self.assertEqual("foobar\n", text) + + def test_remove(self): + """Test removing an entry from the cache.""" + self.test_write_and_write_cache_file() + + self.cache.remove('test.yaml') + + self.assertFalse(os.path.exists(self.cache.cache_path('test.yaml'))) + self.assertFalse(os.path.exists(self.cache._lock_path('test.yaml'))) + + def test_write_and_write_cache_file(self): + """Test two write transactions on a cached file.""" + with self.cache.write_transaction('test.yaml') as (old, new): + self.assertTrue(old is None) + self.assertTrue(new is not None) + new.write("foobar\n") + + with self.cache.write_transaction('test.yaml') as (old, new): + self.assertTrue(old is not None) + text = old.read() + self.assertEqual("foobar\n", text) + self.assertTrue(new is not None) + new.write("barbaz\n") + + with self.cache.read_transaction('test.yaml') as stream: + text = stream.read() + self.assertEqual("barbaz\n", text) diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py index 0e9f6daf4d..aaf573241b 100644 --- a/lib/spack/spack/test/lock.py +++ b/lib/spack/spack/test/lock.py @@ -187,7 +187,6 @@ class LockTest(unittest.TestCase): barrier.wait() # ---------------------------------------- 13 lock.release_read() - def p2(barrier): lock = Lock(self.lock_path) @@ -224,7 +223,6 @@ class LockTest(unittest.TestCase): barrier.wait() # ---------------------------------------- 13 lock.release_read() - def p3(barrier): lock = Lock(self.lock_path) @@ -262,3 +260,164 @@ class LockTest(unittest.TestCase): lock.release_read() self.multiproc_test(p1, p2, p3) + + def test_transaction(self): + def enter_fn(): + vals['entered'] = True + + def exit_fn(t, v, tb): + vals['exited'] = True + vals['exception'] = (t or v or tb) + + lock = Lock(self.lock_path) + vals = {'entered': False, 'exited': False, 'exception': False } + with ReadTransaction(lock, enter_fn, exit_fn): pass + self.assertTrue(vals['entered']) + self.assertTrue(vals['exited']) + self.assertFalse(vals['exception']) + + vals = {'entered': False, 'exited': False, 'exception': False } + with WriteTransaction(lock, enter_fn, exit_fn): pass + self.assertTrue(vals['entered']) + self.assertTrue(vals['exited']) + self.assertFalse(vals['exception']) + + def test_transaction_with_exception(self): + def enter_fn(): + vals['entered'] = True + + def exit_fn(t, v, tb): + vals['exited'] = True + vals['exception'] = (t or v or tb) + + lock = Lock(self.lock_path) + + def do_read_with_exception(): + with ReadTransaction(lock, enter_fn, exit_fn): + raise Exception() + + def do_write_with_exception(): + with WriteTransaction(lock, enter_fn, exit_fn): + raise Exception() + + vals = {'entered': False, 'exited': False, 'exception': False } + self.assertRaises(Exception, do_read_with_exception) + self.assertTrue(vals['entered']) + self.assertTrue(vals['exited']) + self.assertTrue(vals['exception']) + + vals = {'entered': False, 'exited': False, 'exception': False } + self.assertRaises(Exception, do_write_with_exception) + self.assertTrue(vals['entered']) + self.assertTrue(vals['exited']) + self.assertTrue(vals['exception']) + + def test_transaction_with_context_manager(self): + class TestContextManager(object): + def __enter__(self): + vals['entered'] = True + + def __exit__(self, t, v, tb): + vals['exited'] = True + vals['exception'] = (t or v or tb) + + def exit_fn(t, v, tb): + vals['exited_fn'] = True + vals['exception_fn'] = (t or v or tb) + + lock = Lock(self.lock_path) + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False } + with ReadTransaction(lock, TestContextManager, exit_fn): pass + self.assertTrue(vals['entered']) + self.assertTrue(vals['exited']) + self.assertFalse(vals['exception']) + self.assertTrue(vals['exited_fn']) + self.assertFalse(vals['exception_fn']) + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False } + with ReadTransaction(lock, TestContextManager): pass + self.assertTrue(vals['entered']) + self.assertTrue(vals['exited']) + self.assertFalse(vals['exception']) + self.assertFalse(vals['exited_fn']) + self.assertFalse(vals['exception_fn']) + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False } + with WriteTransaction(lock, TestContextManager, exit_fn): pass + self.assertTrue(vals['entered']) + self.assertTrue(vals['exited']) + self.assertFalse(vals['exception']) + self.assertTrue(vals['exited_fn']) + self.assertFalse(vals['exception_fn']) + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False } + with WriteTransaction(lock, TestContextManager): pass + self.assertTrue(vals['entered']) + self.assertTrue(vals['exited']) + self.assertFalse(vals['exception']) + self.assertFalse(vals['exited_fn']) + self.assertFalse(vals['exception_fn']) + + def test_transaction_with_context_manager_and_exception(self): + class TestContextManager(object): + def __enter__(self): + vals['entered'] = True + + def __exit__(self, t, v, tb): + vals['exited'] = True + vals['exception'] = (t or v or tb) + + def exit_fn(t, v, tb): + vals['exited_fn'] = True + vals['exception_fn'] = (t or v or tb) + + lock = Lock(self.lock_path) + + def do_read_with_exception(exit_fn): + with ReadTransaction(lock, TestContextManager, exit_fn): + raise Exception() + + def do_write_with_exception(exit_fn): + with WriteTransaction(lock, TestContextManager, exit_fn): + raise Exception() + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False } + self.assertRaises(Exception, do_read_with_exception, exit_fn) + self.assertTrue(vals['entered']) + self.assertTrue(vals['exited']) + self.assertTrue(vals['exception']) + self.assertTrue(vals['exited_fn']) + self.assertTrue(vals['exception_fn']) + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False } + self.assertRaises(Exception, do_read_with_exception, None) + self.assertTrue(vals['entered']) + self.assertTrue(vals['exited']) + self.assertTrue(vals['exception']) + self.assertFalse(vals['exited_fn']) + self.assertFalse(vals['exception_fn']) + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False } + self.assertRaises(Exception, do_write_with_exception, exit_fn) + self.assertTrue(vals['entered']) + self.assertTrue(vals['exited']) + self.assertTrue(vals['exception']) + self.assertTrue(vals['exited_fn']) + self.assertTrue(vals['exception_fn']) + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False } + self.assertRaises(Exception, do_write_with_exception, None) + self.assertTrue(vals['entered']) + self.assertTrue(vals['exited']) + self.assertTrue(vals['exception']) + self.assertFalse(vals['exited_fn']) + self.assertFalse(vals['exception_fn']) diff --git a/lib/spack/spack/test/mock_database.py b/lib/spack/spack/test/mock_database.py index b1194f2451..da01e82bfa 100644 --- a/lib/spack/spack/test/mock_database.py +++ b/lib/spack/spack/test/mock_database.py @@ -95,8 +95,10 @@ class MockDatabase(MockPackagesTest): self._mock_install('mpileaks ^zmpi') def tearDown(self): - for spec in spack.installed_db.query(): - spec.package.do_uninstall(spec) + with spack.installed_db.write_transaction(): + for spec in spack.installed_db.query(): + spec.package.do_uninstall(spec) + super(MockDatabase, self).tearDown() shutil.rmtree(self.install_path) spack.install_path = self.spack_install_path diff --git a/lib/spack/spack/test/provider_index.py b/lib/spack/spack/test/provider_index.py index 7d5f997b0a..861814e0ae 100644 --- a/lib/spack/spack/test/provider_index.py +++ b/lib/spack/spack/test/provider_index.py @@ -94,7 +94,3 @@ class ProviderIndexTest(MockPackagesTest): p = ProviderIndex(spack.repo.all_package_names()) q = p.copy() self.assertEqual(p, q) - - - def test_copy(self): - pass -- cgit v1.2.3-60-g2f50