From eee502cc3b1a13fd801bb16ee8e57b2c85dd38a6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 13 Apr 2018 21:39:39 -0700 Subject: init: Move file cache initialization out of __init__.py --- lib/spack/spack/__init__.py | 22 ---- lib/spack/spack/caches.py | 73 +++++++++++++ lib/spack/spack/cmd/clean.py | 5 +- lib/spack/spack/file_cache.py | 182 -------------------------------- lib/spack/spack/repository.py | 19 ++-- lib/spack/spack/stage.py | 5 +- lib/spack/spack/test/cmd/clean.py | 13 ++- lib/spack/spack/test/conftest.py | 5 +- lib/spack/spack/test/file_cache.py | 75 ------------- lib/spack/spack/test/util/file_cache.py | 75 +++++++++++++ lib/spack/spack/util/file_cache.py | 182 ++++++++++++++++++++++++++++++++ 11 files changed, 358 insertions(+), 298 deletions(-) create mode 100644 lib/spack/spack/caches.py delete mode 100644 lib/spack/spack/file_cache.py delete mode 100644 lib/spack/spack/test/file_cache.py create mode 100644 lib/spack/spack/test/util/file_cache.py create mode 100644 lib/spack/spack/util/file_cache.py (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index e9c4f4191c..863e8f8016 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -23,12 +23,9 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os import sys import multiprocessing -from spack.paths import var_path, user_config_path - #----------------------------------------------------------------------------- # Below code imports spack packages. #----------------------------------------------------------------------------- @@ -44,20 +41,8 @@ import spack.config from spack.util.path import canonicalize_path -# handle basic configuration first _config = spack.config.get_config('config') - -# Path where downloaded source code is cached -cache_path = canonicalize_path( - _config.get('source_cache', os.path.join(var_path, "cache"))) - - -# cache for miscellaneous stuff. -misc_cache_path = canonicalize_path( - _config.get('misc_cache', os.path.join(user_config_path, 'cache'))) - - # TODO: get this out of __init__.py binary_cache_retrieved_specs = set() @@ -102,13 +87,6 @@ from spack.version import Version spack_version = Version("0.11.2") -# set up the caches after getting all config options -import spack.fetch_strategy -from spack.file_cache import FileCache -misc_cache = FileCache(misc_cache_path) -fetch_cache = spack.fetch_strategy.FsCache(cache_path) - - # Set up the default packages database. import spack.error try: diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py new file mode 100644 index 0000000000..ff2bb7a7c6 --- /dev/null +++ b/lib/spack/spack/caches.py @@ -0,0 +1,73 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Caches used by Spack to store data""" +import os +import spack.paths +import spack.config +import spack.fetch_strategy +from spack.util.path import canonicalize_path +from spack.util.file_cache import FileCache + + +_misc_cache = None +_fetch_cache = None + + +def misc_cache(): + """The ``misc_cache`` is Spack's cache for small data. + + Currently the ``misc_cache`` stores indexes for virtual dependency + providers and for which packages provide which tags. + """ + global _misc_cache + + if _misc_cache is None: + config = spack.config.get_config('config') + path = config.get('misc_cache') + if not path: + path = os.path.join(spack.paths.user_config_path, 'cache') + path = canonicalize_path(path) + _misc_cache = FileCache(path) + + return _misc_cache + + +def fetch_cache(): + """Filesystem cache of downloaded archives. + + This prevents Spack from repeatedly fetch the same files when + building the same package different ways or multiple times. + """ + global _fetch_cache + + if _fetch_cache is None: + config = spack.config.get_config('config') + path = config.get('source_cache') + if not path: + path = os.path.join(spack.paths.var_path, "cache") + path = canonicalize_path(path) + _fetch_cache = spack.fetch_strategy.FsCache(path) + + return _fetch_cache diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index c072bac251..83d5116d0e 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -27,6 +27,7 @@ import argparse import llnl.util.tty as tty import spack +import spack.caches import spack.cmd description = "remove temporary build files and/or downloaded archives" @@ -81,8 +82,8 @@ def clean(parser, args): if args.downloads: tty.msg('Removing cached downloads') - spack.fetch_cache.destroy() + spack.caches.fetch_cache().destroy() if args.misc_cache: tty.msg('Removing cached information on repositories') - spack.misc_cache.destroy() + spack.caches.misc_cache().destroy() diff --git a/lib/spack/spack/file_cache.py b/lib/spack/spack/file_cache.py deleted file mode 100644 index 37e2bcf0a6..0000000000 --- a/lib/spack/spack/file_cache.py +++ /dev/null @@ -1,182 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/spack/spack -# Please also see the NOTICE and LICENSE files for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import os -import shutil - -from llnl.util.filesystem import mkdirp, join_path -from llnl.util.lock import Lock, ReadTransaction, WriteTransaction - -from spack.error import SpackError - - -class FileCache(object): - """This class manages cached data in the filesystem. - - - Cache files are fetched and stored by unique keys. Keys can be relative - paths, so that there can be some hierarchy in the cache. - - - The FileCache handles locking cache files for reading and writing, so - client code need not manage locks for cache entries. - - """ - - def __init__(self, root): - """Create a file cache object. - - This will create the cache directory if it does not exist yet. - - """ - self.root = root.rstrip(os.path.sep) - if not os.path.exists(self.root): - mkdirp(self.root) - - self._locks = {} - - def destroy(self): - """Remove all files under the cache root.""" - for f in os.listdir(self.root): - path = join_path(self.root, f) - if os.path.isdir(path): - shutil.rmtree(path, True) - else: - os.remove(path) - - def cache_path(self, key): - """Path to the file in the cache for a particular key.""" - return join_path(self.root, key) - - def _lock_path(self, key): - """Path to the file in the cache for a particular key.""" - keyfile = os.path.basename(key) - keydir = os.path.dirname(key) - - return join_path(self.root, keydir, '.' + keyfile + '.lock') - - def _get_lock(self, key): - """Create a lock for a key, if necessary, and return a lock object.""" - if key not in self._locks: - self._locks[key] = Lock(self._lock_path(key)) - return self._locks[key] - - def init_entry(self, key): - """Ensure we can access a cache file. Create a lock for it if needed. - - Return whether the cache file exists yet or not. - """ - cache_path = self.cache_path(key) - - exists = os.path.exists(cache_path) - if exists: - if not os.path.isfile(cache_path): - raise CacheError("Cache file is not a file: %s" % cache_path) - - if not os.access(cache_path, os.R_OK | os.W_OK): - raise CacheError("Cannot access cache file: %s" % cache_path) - else: - # if the file is hierarchical, make parent directories - parent = os.path.dirname(cache_path) - if parent.rstrip(os.path.sep) != self.root: - mkdirp(parent) - - if not os.access(parent, os.R_OK | os.W_OK): - raise CacheError("Cannot access cache directory: %s" % parent) - - # ensure lock is created for this key - self._get_lock(key) - return exists - - def read_transaction(self, key): - """Get a read transaction on a file cache item. - - Returns a ReadTransaction context manager and opens the cache file for - reading. You can use it like this: - - with file_cache_object.read_transaction(key) as cache_file: - cache_file.read() - - """ - return ReadTransaction( - self._get_lock(key), lambda: open(self.cache_path(key))) - - def write_transaction(self, key): - """Get a write transaction on a file cache item. - - Returns a WriteTransaction context manager that opens a temporary file - for writing. Once the context manager finishes, if nothing went wrong, - moves the file into place on top of the old file atomically. - - """ - class WriteContextManager(object): - - def __enter__(cm): - cm.orig_filename = self.cache_path(key) - cm.orig_file = None - if os.path.exists(cm.orig_filename): - cm.orig_file = open(cm.orig_filename, 'r') - - cm.tmp_filename = self.cache_path(key) + '.tmp' - cm.tmp_file = open(cm.tmp_filename, 'w') - - return cm.orig_file, cm.tmp_file - - def __exit__(cm, type, value, traceback): - if cm.orig_file: - cm.orig_file.close() - cm.tmp_file.close() - - if value: - # remove tmp on exception & raise it - shutil.rmtree(cm.tmp_filename, True) - - else: - os.rename(cm.tmp_filename, cm.orig_filename) - - return WriteTransaction(self._get_lock(key), WriteContextManager) - - def mtime(self, key): - """Return modification time of cache file, or 0 if it does not exist. - - Time is in units returned by os.stat in the mtime field, which is - platform-dependent. - - """ - if not self.init_entry(key): - return 0 - else: - sinfo = os.stat(self.cache_path(key)) - return sinfo.st_mtime - - def remove(self, key): - lock = self._get_lock(key) - try: - lock.acquire_write() - os.unlink(self.cache_path(key)) - finally: - lock.release_write() - os.unlink(self._lock_path(key)) - - -class CacheError(SpackError): - pass diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 8a713b1a39..d22b5e269e 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -48,6 +48,7 @@ import llnl.util.tty as tty from llnl.util.filesystem import mkdirp, join_path, install import spack +import spack.caches import spack.error import spack.spec from spack.provider_index import ProviderIndex @@ -252,7 +253,8 @@ def make_provider_index_cache(packages_path, namespace): cache_filename = 'providers/{0}-index.yaml'.format(namespace) # Compute which packages needs to be updated in the cache - index_mtime = spack.misc_cache.mtime(cache_filename) + misc_cache = spack.caches.misc_cache() + index_mtime = misc_cache.mtime(cache_filename) needs_update = [ x for x, sinfo in fast_package_checker.items() @@ -260,19 +262,19 @@ def make_provider_index_cache(packages_path, namespace): ] # Read the old ProviderIndex, or make a new one. - index_existed = spack.misc_cache.init_entry(cache_filename) + index_existed = misc_cache.init_entry(cache_filename) if index_existed and not needs_update: # If the provider index exists and doesn't need an update # just read from it - with spack.misc_cache.read_transaction(cache_filename) as f: + with misc_cache.read_transaction(cache_filename) as f: index = ProviderIndex.from_yaml(f) else: # Otherwise we need a write transaction to update it - with spack.misc_cache.write_transaction(cache_filename) as (old, new): + with misc_cache.write_transaction(cache_filename) as (old, new): index = ProviderIndex.from_yaml(old) if old else ProviderIndex() @@ -305,7 +307,8 @@ def make_tag_index_cache(packages_path, namespace): cache_filename = 'tags/{0}-index.json'.format(namespace) # Compute which packages needs to be updated in the cache - index_mtime = spack.misc_cache.mtime(cache_filename) + misc_cache = spack.caches.misc_cache() + index_mtime = misc_cache.mtime(cache_filename) needs_update = [ x for x, sinfo in fast_package_checker.items() @@ -313,19 +316,19 @@ def make_tag_index_cache(packages_path, namespace): ] # Read the old ProviderIndex, or make a new one. - index_existed = spack.misc_cache.init_entry(cache_filename) + index_existed = misc_cache.init_entry(cache_filename) if index_existed and not needs_update: # If the provider index exists and doesn't need an update # just read from it - with spack.misc_cache.read_transaction(cache_filename) as f: + with misc_cache.read_transaction(cache_filename) as f: index = TagIndex.from_json(f) else: # Otherwise we need a write transaction to update it - with spack.misc_cache.write_transaction(cache_filename) as (old, new): + with misc_cache.write_transaction(cache_filename) as (old, new): index = TagIndex.from_json(old) if old else TagIndex() diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index d282786370..b119a874bb 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -39,6 +39,7 @@ from llnl.util.filesystem import mkdirp, join_path, can_access from llnl.util.filesystem import remove_if_dead_link, remove_linked_tree import spack.paths +import spack.caches import spack.config import spack.error import spack.fetch_strategy as fs @@ -408,7 +409,7 @@ class Stage(object): url, digest, expand=expand, extension=extension)) if self.default_fetcher.cachable: fetchers.insert( - 0, spack.fetch_cache.fetcher( + 0, spack.caches.fetch_cache().fetcher( self.mirror_path, digest, expand=expand, extension=extension)) @@ -455,7 +456,7 @@ class Stage(object): self.fetcher.check() def cache_local(self): - spack.fetch_cache.store(self.fetcher, self.mirror_path) + spack.caches.fetch_cache().store(self.fetcher, self.mirror_path) def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded diff --git a/lib/spack/spack/test/cmd/clean.py b/lib/spack/spack/test/cmd/clean.py index 9231029c90..3bc5e4c184 100644 --- a/lib/spack/spack/test/cmd/clean.py +++ b/lib/spack/spack/test/cmd/clean.py @@ -23,7 +23,8 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import pytest -import spack +import spack.stage +import spack.caches import spack.main import spack.package @@ -42,8 +43,10 @@ def mock_calls_for_clean(monkeypatch): monkeypatch.setattr(spack.package.PackageBase, 'do_clean', Counter()) monkeypatch.setattr(spack.stage, 'purge', Counter()) - monkeypatch.setattr(spack.fetch_cache, 'destroy', Counter(), raising=False) - monkeypatch.setattr(spack.misc_cache, 'destroy', Counter()) + monkeypatch.setattr( + spack.caches._fetch_cache, 'destroy', Counter(), raising=False) + monkeypatch.setattr( + spack.caches._misc_cache, 'destroy', Counter()) @pytest.mark.usefixtures( @@ -66,5 +69,5 @@ def test_function_calls(command_line, counters): # number of times assert spack.package.PackageBase.do_clean.call_count == counters[0] assert spack.stage.purge.call_count == counters[1] - assert spack.fetch_cache.destroy.call_count == counters[2] - assert spack.misc_cache.destroy.call_count == counters[3] + assert spack.caches.fetch_cache().destroy.call_count == counters[2] + assert spack.caches.misc_cache().destroy.call_count == counters[3] diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 997cdb724c..6b60c31a4f 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -36,10 +36,11 @@ import pytest from llnl.util.filesystem import remove_linked_tree import spack -import spack.paths import spack.architecture +import spack.caches import spack.database import spack.directory_layout +import spack.paths import spack.platforms.test import spack.repository import spack.stage @@ -155,7 +156,7 @@ def mock_fetch_cache(monkeypatch): def __str__(self): return "[mock fetch cache]" - monkeypatch.setattr(spack, 'fetch_cache', MockCache()) + monkeypatch.setattr(spack.caches, '_fetch_cache', MockCache()) # FIXME: The lines below should better be added to a fixture with diff --git a/lib/spack/spack/test/file_cache.py b/lib/spack/spack/test/file_cache.py deleted file mode 100644 index c3c87296d2..0000000000 --- a/lib/spack/spack/test/file_cache.py +++ /dev/null @@ -1,75 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/spack/spack -# Please also see the NOTICE and LICENSE files for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -"""Test Spack's FileCache.""" -import os - -import pytest -from spack.file_cache import FileCache - - -@pytest.fixture() -def file_cache(tmpdir): - """Returns a properly initialized FileCache instance""" - return FileCache(str(tmpdir)) - - -def test_write_and_read_cache_file(file_cache): - """Test writing then reading a cached file.""" - with file_cache.write_transaction('test.yaml') as (old, new): - assert old is None - assert new is not None - new.write("foobar\n") - - with file_cache.read_transaction('test.yaml') as stream: - text = stream.read() - assert text == "foobar\n" - - -def test_write_and_remove_cache_file(file_cache): - """Test two write transactions on a cached file. Then try to remove an - entry from it. - """ - - with file_cache.write_transaction('test.yaml') as (old, new): - assert old is None - assert new is not None - new.write("foobar\n") - - with file_cache.write_transaction('test.yaml') as (old, new): - assert old is not None - text = old.read() - assert text == "foobar\n" - assert new is not None - new.write("barbaz\n") - - with file_cache.read_transaction('test.yaml') as stream: - text = stream.read() - assert text == "barbaz\n" - - file_cache.remove('test.yaml') - - # After removal both the file and the lock file should not exist - assert not os.path.exists(file_cache.cache_path('test.yaml')) - assert not os.path.exists(file_cache._lock_path('test.yaml')) diff --git a/lib/spack/spack/test/util/file_cache.py b/lib/spack/spack/test/util/file_cache.py new file mode 100644 index 0000000000..4079a8d160 --- /dev/null +++ b/lib/spack/spack/test/util/file_cache.py @@ -0,0 +1,75 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Test Spack's FileCache.""" +import os + +import pytest +from spack.util.file_cache import FileCache + + +@pytest.fixture() +def file_cache(tmpdir): + """Returns a properly initialized FileCache instance""" + return FileCache(str(tmpdir)) + + +def test_write_and_read_cache_file(file_cache): + """Test writing then reading a cached file.""" + with file_cache.write_transaction('test.yaml') as (old, new): + assert old is None + assert new is not None + new.write("foobar\n") + + with file_cache.read_transaction('test.yaml') as stream: + text = stream.read() + assert text == "foobar\n" + + +def test_write_and_remove_cache_file(file_cache): + """Test two write transactions on a cached file. Then try to remove an + entry from it. + """ + + with file_cache.write_transaction('test.yaml') as (old, new): + assert old is None + assert new is not None + new.write("foobar\n") + + with file_cache.write_transaction('test.yaml') as (old, new): + assert old is not None + text = old.read() + assert text == "foobar\n" + assert new is not None + new.write("barbaz\n") + + with file_cache.read_transaction('test.yaml') as stream: + text = stream.read() + assert text == "barbaz\n" + + file_cache.remove('test.yaml') + + # After removal both the file and the lock file should not exist + assert not os.path.exists(file_cache.cache_path('test.yaml')) + assert not os.path.exists(file_cache._lock_path('test.yaml')) diff --git a/lib/spack/spack/util/file_cache.py b/lib/spack/spack/util/file_cache.py new file mode 100644 index 0000000000..37e2bcf0a6 --- /dev/null +++ b/lib/spack/spack/util/file_cache.py @@ -0,0 +1,182 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import shutil + +from llnl.util.filesystem import mkdirp, join_path +from llnl.util.lock import Lock, ReadTransaction, WriteTransaction + +from spack.error import SpackError + + +class FileCache(object): + """This class manages cached data in the filesystem. + + - Cache files are fetched and stored by unique keys. Keys can be relative + paths, so that there can be some hierarchy in the cache. + + - The FileCache handles locking cache files for reading and writing, so + client code need not manage locks for cache entries. + + """ + + def __init__(self, root): + """Create a file cache object. + + This will create the cache directory if it does not exist yet. + + """ + self.root = root.rstrip(os.path.sep) + if not os.path.exists(self.root): + mkdirp(self.root) + + self._locks = {} + + def destroy(self): + """Remove all files under the cache root.""" + for f in os.listdir(self.root): + path = join_path(self.root, f) + if os.path.isdir(path): + shutil.rmtree(path, True) + else: + os.remove(path) + + def cache_path(self, key): + """Path to the file in the cache for a particular key.""" + return join_path(self.root, key) + + def _lock_path(self, key): + """Path to the file in the cache for a particular key.""" + keyfile = os.path.basename(key) + keydir = os.path.dirname(key) + + return join_path(self.root, keydir, '.' + keyfile + '.lock') + + def _get_lock(self, key): + """Create a lock for a key, if necessary, and return a lock object.""" + if key not in self._locks: + self._locks[key] = Lock(self._lock_path(key)) + return self._locks[key] + + def init_entry(self, key): + """Ensure we can access a cache file. Create a lock for it if needed. + + Return whether the cache file exists yet or not. + """ + cache_path = self.cache_path(key) + + exists = os.path.exists(cache_path) + if exists: + if not os.path.isfile(cache_path): + raise CacheError("Cache file is not a file: %s" % cache_path) + + if not os.access(cache_path, os.R_OK | os.W_OK): + raise CacheError("Cannot access cache file: %s" % cache_path) + else: + # if the file is hierarchical, make parent directories + parent = os.path.dirname(cache_path) + if parent.rstrip(os.path.sep) != self.root: + mkdirp(parent) + + if not os.access(parent, os.R_OK | os.W_OK): + raise CacheError("Cannot access cache directory: %s" % parent) + + # ensure lock is created for this key + self._get_lock(key) + return exists + + def read_transaction(self, key): + """Get a read transaction on a file cache item. + + Returns a ReadTransaction context manager and opens the cache file for + reading. You can use it like this: + + with file_cache_object.read_transaction(key) as cache_file: + cache_file.read() + + """ + return ReadTransaction( + self._get_lock(key), lambda: open(self.cache_path(key))) + + def write_transaction(self, key): + """Get a write transaction on a file cache item. + + Returns a WriteTransaction context manager that opens a temporary file + for writing. Once the context manager finishes, if nothing went wrong, + moves the file into place on top of the old file atomically. + + """ + class WriteContextManager(object): + + def __enter__(cm): + cm.orig_filename = self.cache_path(key) + cm.orig_file = None + if os.path.exists(cm.orig_filename): + cm.orig_file = open(cm.orig_filename, 'r') + + cm.tmp_filename = self.cache_path(key) + '.tmp' + cm.tmp_file = open(cm.tmp_filename, 'w') + + return cm.orig_file, cm.tmp_file + + def __exit__(cm, type, value, traceback): + if cm.orig_file: + cm.orig_file.close() + cm.tmp_file.close() + + if value: + # remove tmp on exception & raise it + shutil.rmtree(cm.tmp_filename, True) + + else: + os.rename(cm.tmp_filename, cm.orig_filename) + + return WriteTransaction(self._get_lock(key), WriteContextManager) + + def mtime(self, key): + """Return modification time of cache file, or 0 if it does not exist. + + Time is in units returned by os.stat in the mtime field, which is + platform-dependent. + + """ + if not self.init_entry(key): + return 0 + else: + sinfo = os.stat(self.cache_path(key)) + return sinfo.st_mtime + + def remove(self, key): + lock = self._get_lock(key) + try: + lock.acquire_write() + os.unlink(self.cache_path(key)) + finally: + lock.release_write() + os.unlink(self._lock_path(key)) + + +class CacheError(SpackError): + pass -- cgit v1.2.3-70-g09d2