From da760a66de8370ad73eadfe7a38a7cb6782c99bc Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 28 Oct 2016 00:57:51 -0700 Subject: source_cache, misc_cache, verify_ssl, checksum, & dirty in config.yaml - Added new preferences to config.yaml: - source_cache - misc_cache - verify_ssl - checksum - dirty --- lib/spack/spack/__init__.py | 115 ++++++++++++++++++----------- lib/spack/spack/cmd/common/arguments.py | 18 +++++ lib/spack/spack/cmd/diy.py | 7 +- lib/spack/spack/cmd/install.py | 8 +- lib/spack/spack/cmd/purge.py | 10 +-- lib/spack/spack/cmd/setup.py | 7 +- lib/spack/spack/config.py | 3 + lib/spack/spack/file_cache.py | 2 +- lib/spack/spack/modules.py | 2 +- lib/spack/spack/package.py | 6 +- lib/spack/spack/repository.py | 8 +- lib/spack/spack/test/config.py | 3 +- lib/spack/spack/test/mock_packages_test.py | 9 ++- lib/spack/spack/test/modules.py | 2 +- lib/spack/spack/test/stage.py | 28 ++++++- lib/spack/spack/util/path.py | 8 +- 16 files changed, 160 insertions(+), 76 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 782a9b8a9f..0646f5cb32 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -30,7 +30,10 @@ import getpass from llnl.util.filesystem import * import llnl.util.tty as tty -# This lives in $prefix/lib/spack/spack/__file__ +#----------------------------------------------------------------------------- +# Variables describing how Spack is laid out in its prefix. +#----------------------------------------------------------------------------- +# This file lives in $prefix/lib/spack/spack/__file__ spack_root = ancestor(__file__, 4) # The spack script itself @@ -49,82 +52,100 @@ var_path = join_path(spack_root, "var", "spack") stage_path = join_path(var_path, "stage") repos_path = join_path(var_path, "repos") share_path = join_path(spack_root, "share", "spack") -cache_path = join_path(var_path, "cache") + +# Paths to built-in Spack repositories. +packages_path = join_path(repos_path, "builtin") +mock_packages_path = join_path(repos_path, "builtin.mock") # User configuration location user_config_path = os.path.expanduser('~/.spack') -import spack.fetch_strategy -fetch_cache = spack.fetch_strategy.FsCache(cache_path) - -from spack.file_cache import FileCache -user_cache_path = join_path(user_config_path, 'cache') -user_cache = FileCache(user_cache_path) - prefix = spack_root opt_path = join_path(prefix, "opt") etc_path = join_path(prefix, "etc") -# -# Set up the default packages database. -# + +#----------------------------------------------------------------------------- +# Initial imports (only for use in this file -- see __all__ below.) +#----------------------------------------------------------------------------- +# These imports depend on the paths above, or on each other +# Group them here so it's easy to understand the order. +# TODO: refactor this stuff to be more init order agnostic. import spack.repository +import spack.error +import spack.config +import spack.fetch_strategy +from spack.file_cache import FileCache +from spack.preferred_packages import PreferredPackages +from spack.abi import ABI +from spack.concretize import DefaultConcretizer +from spack.version import Version +from spack.util.path import canonicalize_path + + +#----------------------------------------------------------------------------- +# Initialize various data structures & objects at the core of Spack. +#----------------------------------------------------------------------------- +# Version information +spack_version = Version("0.9.1") + + +# Set up the default packages database. try: repo = spack.repository.RepoPath() sys.meta_path.append(repo) except spack.error.SpackError, e: tty.die('while initializing Spack RepoPath:', e.message) -# -# Paths to built-in Spack repositories. -# -packages_path = join_path(repos_path, "builtin") -mock_packages_path = join_path(repos_path, "builtin.mock") -# -# This controls how packages are sorted when trying to choose -# the most preferred package. More preferred packages are sorted -# first. -# -from spack.preferred_packages import PreferredPackages +# PreferredPackages controls preference sort order during concretization. +# More preferred packages are sorted first. pkgsort = PreferredPackages() -# -# This tests ABI compatibility between packages -# -from spack.abi import ABI + +# Tests ABI compatibility between packages abi = ABI() -# + # This controls how things are concretized in spack. # Replace it with a subclass if you want different # policies. -# -from spack.concretize import DefaultConcretizer concretizer = DefaultConcretizer() -# Version information -from spack.version import Version -spack_version = Version("0.9.1") +#----------------------------------------------------------------------------- +# config.yaml options +#----------------------------------------------------------------------------- +_config = spack.config.get_config('config') -# -# Executables used by Spack -# -from spack.util.executable import Executable, which -# User's editor from the environment -editor = Executable(os.environ.get("EDITOR", "vi")) +# Path where downloaded source code is cached +cache_path = canonicalize_path( + _config.get('source_cache', join_path(var_path, "cache"))) +fetch_cache = spack.fetch_strategy.FsCache(cache_path) + + +# cache for miscellaneous stuff. +misc_cache_path = canonicalize_path( + _config.get('misc_cache', join_path(user_config_path, 'cache'))) +misc_cache = FileCache(misc_cache_path) + # If this is enabled, tools that use SSL should not verify # certifiates. e.g., curl should use the -k option. -insecure = False +insecure = not _config.get('verify_ssl', True) -# Whether spack should allow installation of unsafe versions of -# software. "Unsafe" versions are ones it doesn't have a checksum -# for. -do_checksum = True -# +# Whether spack should allow installation of unsafe versions of software. +# "Unsafe" versions are ones it doesn't have a checksum for. +do_checksum = _config.get('checksum', True) + + +# If this is True, spack will not clean the environment to remove +# potentially harmful variables before builds. +dirty = _config.get('dirty', False) + + +#----------------------------------------------------------------------------- # When packages call 'from spack import *', this extra stuff is brought in. # # Spack internal code should call 'import spack' and accesses other @@ -135,6 +156,7 @@ do_checksum = True # packages should live. This file is overloaded for spack core vs. # for packages. # +#----------------------------------------------------------------------------- __all__ = ['PackageBase', 'Package', 'CMakePackage', @@ -165,6 +187,9 @@ import spack.util.executable from spack.util.executable import * __all__ += spack.util.executable.__all__ +# User's editor from the environment +editor = Executable(os.environ.get("EDITOR", "vi")) + from spack.package import \ install_dependency_symlinks, flatten_dependencies, \ DependencyConflictError, InstallError, ExternalPackageError diff --git a/lib/spack/spack/cmd/common/arguments.py b/lib/spack/spack/cmd/common/arguments.py index 5ab54bdb61..8cb4a4b871 100644 --- a/lib/spack/spack/cmd/common/arguments.py +++ b/lib/spack/spack/cmd/common/arguments.py @@ -95,3 +95,21 @@ parms = Bunch( 'help': 'Recursively traverse spec dependencies' }) _arguments['recurse_dependencies'] = parms + +parms = Bunch( + flags=('--clean',), + kwargs={ + 'action': 'store_false', + 'dest': 'dirty', + 'help': 'Clean environment before installing package.' + }) +_arguments['clean'] = parms + +parms = Bunch( + flags=('--dirty',), + kwargs={ + 'action': 'store_true', + 'dest': 'dirty', + 'help': 'Do NOT clean environment before installing.' + }) +_arguments['dirty'] = parms diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 08386cac07..22966a26eb 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -30,6 +30,7 @@ import llnl.util.tty as tty import spack import spack.cmd +import spack.cmd.common.arguments as arguments from spack.cmd.edit import edit_package from spack.stage import DIYStage @@ -52,9 +53,9 @@ def setup_parser(subparser): subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="specs to use for install. Must contain package AND version.") - subparser.add_argument( - '--dirty', action='store_true', dest='dirty', - help="Install a package *without* cleaning the environment.") + + cd_group = subparser.add_mutually_exclusive_group() + arguments.add_common_arguments(cd_group, ['clean', 'dirty']) def diy(self, args): diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 417e07e9c4..79af35d3a0 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -34,6 +34,7 @@ import llnl.util.filesystem as fs import llnl.util.tty as tty import spack import spack.cmd +import spack.cmd.common.arguments as arguments from spack.build_environment import InstallError from spack.fetch_strategy import FetchError from spack.package import PackageBase @@ -70,9 +71,10 @@ the dependencies.""" subparser.add_argument( '--fake', action='store_true', dest='fake', help="Fake install. Just remove prefix and create a fake file.") - subparser.add_argument( - '--dirty', action='store_true', dest='dirty', - help="Install a package *without* cleaning the environment.") + + cd_group = subparser.add_mutually_exclusive_group() + arguments.add_common_arguments(cd_group, ['clean', 'dirty']) + subparser.add_argument( 'package', nargs=argparse.REMAINDER, diff --git a/lib/spack/spack/cmd/purge.py b/lib/spack/spack/cmd/purge.py index 26d160635c..66cfc2af29 100644 --- a/lib/spack/spack/cmd/purge.py +++ b/lib/spack/spack/cmd/purge.py @@ -36,8 +36,8 @@ def setup_parser(subparser): '-d', '--downloads', action='store_true', help="Remove cached downloads.") subparser.add_argument( - '-u', '--user-cache', action='store_true', - help="Remove caches in user home directory. Includes virtual indices.") + '-m', '--misc-cache', action='store_true', + help="Remove long-lived caches, like the virtual package index.") subparser.add_argument( '-a', '--all', action='store_true', help="Remove all of the above.") @@ -45,7 +45,7 @@ def setup_parser(subparser): def purge(parser, args): # Special case: no flags. - if not any((args.stage, args.downloads, args.user_cache, args.all)): + if not any((args.stage, args.downloads, args.misc_cache, args.all)): stage.purge() return @@ -54,5 +54,5 @@ def purge(parser, args): stage.purge() if args.downloads or args.all: spack.fetch_cache.destroy() - if args.user_cache or args.all: - spack.user_cache.destroy() + if args.misc_cache or args.all: + spack.misc_cache.destroy() diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py index 6509228a98..953906975e 100644 --- a/lib/spack/spack/cmd/setup.py +++ b/lib/spack/spack/cmd/setup.py @@ -32,6 +32,7 @@ import llnl.util.tty as tty import spack import spack.cmd import spack.cmd.install as install +import spack.cmd.common.arguments as arguments from llnl.util.filesystem import set_executable from spack import which from spack.cmd.edit import edit_package @@ -50,9 +51,9 @@ def setup_parser(subparser): subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="specs to use for install. Must contain package AND version.") - subparser.add_argument( - '--dirty', action='store_true', dest='dirty', - help="Install a package *without* cleaning the environment.") + + cd_group = subparser.add_mutually_exclusive_group() + arguments.add_common_arguments(cd_group, ['clean', 'dirty']) def spack_transitive_include_path(): diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 1bd1689150..904195e47e 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -298,6 +298,9 @@ class ConfigScope(object): """Empty cached config information.""" self.sections = {} + def __repr__(self): + return '' % (self.name, self.path) + # # Below are configuration scopes. # diff --git a/lib/spack/spack/file_cache.py b/lib/spack/spack/file_cache.py index 31ae009836..e37f77d68d 100644 --- a/lib/spack/spack/file_cache.py +++ b/lib/spack/spack/file_cache.py @@ -113,7 +113,7 @@ class FileCache(object): Returns a ReadTransaction context manager and opens the cache file for reading. You can use it like this: - with spack.user_cache.read_transaction(key) as cache_file: + with file_cache_object.read_transaction(key) as cache_file: cache_file.read() """ diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 2c2804aea1..5e2a840e14 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -675,7 +675,7 @@ class LmodModule(EnvModule): def __init__(self, spec=None): super(LmodModule, self).__init__(spec) - self.configuration = CONFIGURATION.get('lmod', {}) + self.configuration = _module_config.get('lmod', {}) hierarchy_tokens = self.configuration.get('hierarchical_scheme', []) # TODO : Check that the extra hierarchy tokens specified in the # TODO : configuration file are actually virtual dependencies diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 6bc3362639..491e21bf95 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1118,7 +1118,7 @@ class PackageBase(object): run_tests=False, fake=False, explicit=False, - dirty=False, + dirty=None, **kwargs): """Called by commands to install a package and its dependencies. @@ -1165,6 +1165,10 @@ class PackageBase(object): rec.explicit = True return + # Dirty argument takes precedence over dirty config setting. + if dirty is None: + dirty = spack.dirty + self._do_install_pop_kwargs(kwargs) tty.msg("Installing %s" % self.name) diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 47d6df85b3..94b79accdb 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -620,12 +620,12 @@ class Repo(object): # Read the old ProviderIndex, or make a new one. key = self._cache_file - index_existed = spack.user_cache.init_entry(key) + index_existed = spack.misc_cache.init_entry(key) if index_existed and not self._needs_update: - with spack.user_cache.read_transaction(key) as f: + with spack.misc_cache.read_transaction(key) as f: self._provider_index = ProviderIndex.from_yaml(f) else: - with spack.user_cache.write_transaction(key) as (old, new): + with spack.misc_cache.write_transaction(key) as (old, new): if old: self._provider_index = ProviderIndex.from_yaml(old) else: @@ -701,7 +701,7 @@ class Repo(object): self._all_package_names = [] # Get index modification time. - index_mtime = spack.user_cache.mtime(self._cache_file) + index_mtime = spack.misc_cache.mtime(self._cache_file) for pkg_name in os.listdir(self.packages_path): # Skip non-directories in the package root. diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index de6cd79594..d5e1791b40 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -221,7 +221,8 @@ class ConfigTest(MockPackagesTest): self.check_config(b_comps, *self.b_comp_specs) def check_canonical(self, var, expected): - """ensure things are substituted properly and canonicalized.""" + """Ensure that is substituted properly for in strings + containing in various positions.""" path = '/foo/bar/baz' self.assertEqual(canonicalize_path(var + path), diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 4e1f243c82..a1919cd437 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -191,7 +191,7 @@ config: misc_cache: ~/.spack/cache verify_ssl: true checksum: true - dirty: false + dirty: True """ # these are written out to mock config files. @@ -211,9 +211,6 @@ class MockPackagesTest(unittest.TestCase): self.db = RepoPath(spack.mock_packages_path) spack.repo.swap(self.db) - spack.config.clear_config_caches() - self.real_scopes = spack.config.config_scopes - # Mock up temporary configuration directories self.temp_config = tempfile.mkdtemp() self.mock_site_config = os.path.join(self.temp_config, 'site') @@ -227,6 +224,9 @@ class MockPackagesTest(unittest.TestCase): # TODO: Mocking this up is kind of brittle b/c ConfigScope # TODO: constructor modifies config_scopes. Make it cleaner. + spack.config.clear_config_caches() + self.real_scopes = spack.config.config_scopes + spack.config.config_scopes = OrderedDict() spack.config.ConfigScope('site', self.mock_site_config) spack.config.ConfigScope('user', self.mock_user_config) @@ -261,6 +261,7 @@ class MockPackagesTest(unittest.TestCase): """Restore the real packages path after any test.""" spack.repo.swap(self.db) spack.config.config_scopes = self.real_scopes + shutil.rmtree(self.temp_config, ignore_errors=True) spack.config.clear_config_caches() diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index cb3a26e62b..42f072debb 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -471,7 +471,7 @@ class LmodTests(ModuleFileGeneratorTests): # Make sure that virtual providers (in the hierarchy) always # include a hash. Make sure that the module file for the spec # does not include a hash if hash_length is 0. - spack.modules.CONFIGURATION = self.configuration_no_hash + spack.modules._module_config = self.configuration_no_hash spec = spack.spec.Spec(mpileaks_spec_string) spec.concretize() module = spack.modules.LmodModule(spec) diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index a21142c2cb..64cfa222db 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -46,9 +46,16 @@ def use_tmp(use_tmp): not use temporary space for stages. """ # mock up config - path = _test_tmp_path if use_tmp else spack.stage_path + assert(_test_tmp_path is not None) + + if use_tmp: + path = _test_tmp_path # use temporary stage + else: + path = spack.stage_path # Use Spack's stage dir (no links) + spack.config.update_config( 'config', {'build_stage': [path]}, scope='user') + yield @@ -59,13 +66,28 @@ class StageTest(MockPackagesTest): by the Stage class. It doesn't actually create the Stage -- that is done by individual tests. """ + super(StageTest, self).setUp() + global _test_tmp_path + # + # Mock up a stage area that looks like this: + # + # TMPDIR/ test_files_dir + # tmp/ test_tmp_path (where stage should be) + # test-files/ archive_dir_path + # README.txt test_readme (contains "hello world!\n") + # test-files.tar.gz archive_url = file:///path/to/this + # self.test_files_dir = tempfile.mkdtemp() self.test_tmp_path = os.path.realpath( os.path.join(self.test_files_dir, 'tmp')) _test_tmp_path = self.test_tmp_path + # set _test_tmp_path as the default test directory to use for stages. + spack.config.update_config( + 'config', {'build_stage': [_test_tmp_path]}, scope='user') + self.archive_dir = 'test-files' self.archive_name = self.archive_dir + '.tar.gz' archive_dir_path = os.path.join(self.test_files_dir, @@ -99,6 +121,8 @@ class StageTest(MockPackagesTest): def tearDown(self): """Blows away the test environment directory.""" + super(StageTest, self).tearDown() + shutil.rmtree(self.test_files_dir, ignore_errors=True) # chdir back to original working dir @@ -138,7 +162,7 @@ class StageTest(MockPackagesTest): self.assertFalse(os.path.islink(target)) # Make sure the directory is in the place we asked it to - # be (see setUp and tearDown) + # be (see setUp, tearDown, and use_tmp) self.assertTrue(target.startswith(self.test_tmp_path)) else: diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 5332115ae9..7235f6b756 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -50,18 +50,22 @@ def substitute_config_variables(path): - $spack The Spack instance's prefix - $user The current user's username - $tempdir Default temporary directory returned by tempfile.gettempdir() + + These are substituted case-insensitively into the path, and users can + use either ``$var`` or ``${var}`` syntax for the variables. + """ # Look up replacements for re.sub in the replacements dict. def repl(match): m = match.group(0).strip('${}') - return replacements.get(m, match.group(0)) + return replacements.get(m.lower(), match.group(0)) # Replace $var or ${var}. return re.sub(r'(\$\w+\b|\$\{\w+\})', repl, path) def canonicalize_path(path): - """Substitute $spack, expand user home, take abspath.""" + """Substitute config vars, expand user home, take abspath.""" path = substitute_config_variables(path) path = os.path.expanduser(path) path = os.path.abspath(path) -- cgit v1.2.3-70-g09d2