summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorTodd Gamblin <tgamblin@llnl.gov>2016-10-28 00:57:51 -0700
committerTodd Gamblin <tgamblin@llnl.gov>2016-10-30 23:55:00 -0700
commitda760a66de8370ad73eadfe7a38a7cb6782c99bc (patch)
tree3661bb8e3555839fed59f757e9f398503dd40113 /lib
parent22b14e0f235864a636919911f134aee9fe4def8b (diff)
downloadspack-da760a66de8370ad73eadfe7a38a7cb6782c99bc.tar.gz
spack-da760a66de8370ad73eadfe7a38a7cb6782c99bc.tar.bz2
spack-da760a66de8370ad73eadfe7a38a7cb6782c99bc.tar.xz
spack-da760a66de8370ad73eadfe7a38a7cb6782c99bc.zip
source_cache, misc_cache, verify_ssl, checksum, & dirty in config.yaml
- Added new preferences to config.yaml: - source_cache - misc_cache - verify_ssl - checksum - dirty
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/spack/__init__.py115
-rw-r--r--lib/spack/spack/cmd/common/arguments.py18
-rw-r--r--lib/spack/spack/cmd/diy.py7
-rw-r--r--lib/spack/spack/cmd/install.py8
-rw-r--r--lib/spack/spack/cmd/purge.py10
-rw-r--r--lib/spack/spack/cmd/setup.py7
-rw-r--r--lib/spack/spack/config.py3
-rw-r--r--lib/spack/spack/file_cache.py2
-rw-r--r--lib/spack/spack/modules.py2
-rw-r--r--lib/spack/spack/package.py6
-rw-r--r--lib/spack/spack/repository.py8
-rw-r--r--lib/spack/spack/test/config.py3
-rw-r--r--lib/spack/spack/test/mock_packages_test.py9
-rw-r--r--lib/spack/spack/test/modules.py2
-rw-r--r--lib/spack/spack/test/stage.py28
-rw-r--r--lib/spack/spack/util/path.py8
16 files changed, 160 insertions, 76 deletions
diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index 782a9b8a9f..0646f5cb32 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -30,7 +30,10 @@ import getpass
from llnl.util.filesystem import *
import llnl.util.tty as tty
-# This lives in $prefix/lib/spack/spack/__file__
+#-----------------------------------------------------------------------------
+# Variables describing how Spack is laid out in its prefix.
+#-----------------------------------------------------------------------------
+# This file lives in $prefix/lib/spack/spack/__file__
spack_root = ancestor(__file__, 4)
# The spack script itself
@@ -49,82 +52,100 @@ var_path = join_path(spack_root, "var", "spack")
stage_path = join_path(var_path, "stage")
repos_path = join_path(var_path, "repos")
share_path = join_path(spack_root, "share", "spack")
-cache_path = join_path(var_path, "cache")
+
+# Paths to built-in Spack repositories.
+packages_path = join_path(repos_path, "builtin")
+mock_packages_path = join_path(repos_path, "builtin.mock")
# User configuration location
user_config_path = os.path.expanduser('~/.spack')
-import spack.fetch_strategy
-fetch_cache = spack.fetch_strategy.FsCache(cache_path)
-
-from spack.file_cache import FileCache
-user_cache_path = join_path(user_config_path, 'cache')
-user_cache = FileCache(user_cache_path)
-
prefix = spack_root
opt_path = join_path(prefix, "opt")
etc_path = join_path(prefix, "etc")
-#
-# Set up the default packages database.
-#
+
+#-----------------------------------------------------------------------------
+# Initial imports (only for use in this file -- see __all__ below.)
+#-----------------------------------------------------------------------------
+# These imports depend on the paths above, or on each other
+# Group them here so it's easy to understand the order.
+# TODO: refactor this stuff to be more init order agnostic.
import spack.repository
+import spack.error
+import spack.config
+import spack.fetch_strategy
+from spack.file_cache import FileCache
+from spack.preferred_packages import PreferredPackages
+from spack.abi import ABI
+from spack.concretize import DefaultConcretizer
+from spack.version import Version
+from spack.util.path import canonicalize_path
+
+
+#-----------------------------------------------------------------------------
+# Initialize various data structures & objects at the core of Spack.
+#-----------------------------------------------------------------------------
+# Version information
+spack_version = Version("0.9.1")
+
+
+# Set up the default packages database.
try:
repo = spack.repository.RepoPath()
sys.meta_path.append(repo)
except spack.error.SpackError, e:
tty.die('while initializing Spack RepoPath:', e.message)
-#
-# Paths to built-in Spack repositories.
-#
-packages_path = join_path(repos_path, "builtin")
-mock_packages_path = join_path(repos_path, "builtin.mock")
-#
-# This controls how packages are sorted when trying to choose
-# the most preferred package. More preferred packages are sorted
-# first.
-#
-from spack.preferred_packages import PreferredPackages
+# PreferredPackages controls preference sort order during concretization.
+# More preferred packages are sorted first.
pkgsort = PreferredPackages()
-#
-# This tests ABI compatibility between packages
-#
-from spack.abi import ABI
+
+# Tests ABI compatibility between packages
abi = ABI()
-#
+
# This controls how things are concretized in spack.
# Replace it with a subclass if you want different
# policies.
-#
-from spack.concretize import DefaultConcretizer
concretizer = DefaultConcretizer()
-# Version information
-from spack.version import Version
-spack_version = Version("0.9.1")
+#-----------------------------------------------------------------------------
+# config.yaml options
+#-----------------------------------------------------------------------------
+_config = spack.config.get_config('config')
-#
-# Executables used by Spack
-#
-from spack.util.executable import Executable, which
-# User's editor from the environment
-editor = Executable(os.environ.get("EDITOR", "vi"))
+# Path where downloaded source code is cached
+cache_path = canonicalize_path(
+ _config.get('source_cache', join_path(var_path, "cache")))
+fetch_cache = spack.fetch_strategy.FsCache(cache_path)
+
+
+# cache for miscellaneous stuff.
+misc_cache_path = canonicalize_path(
+ _config.get('misc_cache', join_path(user_config_path, 'cache')))
+misc_cache = FileCache(misc_cache_path)
+
# If this is enabled, tools that use SSL should not verify
# certifiates. e.g., curl should use the -k option.
-insecure = False
+insecure = not _config.get('verify_ssl', True)
-# Whether spack should allow installation of unsafe versions of
-# software. "Unsafe" versions are ones it doesn't have a checksum
-# for.
-do_checksum = True
-#
+# Whether spack should allow installation of unsafe versions of software.
+# "Unsafe" versions are ones it doesn't have a checksum for.
+do_checksum = _config.get('checksum', True)
+
+
+# If this is True, spack will not clean the environment to remove
+# potentially harmful variables before builds.
+dirty = _config.get('dirty', False)
+
+
+#-----------------------------------------------------------------------------
# When packages call 'from spack import *', this extra stuff is brought in.
#
# Spack internal code should call 'import spack' and accesses other
@@ -135,6 +156,7 @@ do_checksum = True
# packages should live. This file is overloaded for spack core vs.
# for packages.
#
+#-----------------------------------------------------------------------------
__all__ = ['PackageBase',
'Package',
'CMakePackage',
@@ -165,6 +187,9 @@ import spack.util.executable
from spack.util.executable import *
__all__ += spack.util.executable.__all__
+# User's editor from the environment
+editor = Executable(os.environ.get("EDITOR", "vi"))
+
from spack.package import \
install_dependency_symlinks, flatten_dependencies, \
DependencyConflictError, InstallError, ExternalPackageError
diff --git a/lib/spack/spack/cmd/common/arguments.py b/lib/spack/spack/cmd/common/arguments.py
index 5ab54bdb61..8cb4a4b871 100644
--- a/lib/spack/spack/cmd/common/arguments.py
+++ b/lib/spack/spack/cmd/common/arguments.py
@@ -95,3 +95,21 @@ parms = Bunch(
'help': 'Recursively traverse spec dependencies'
})
_arguments['recurse_dependencies'] = parms
+
+parms = Bunch(
+ flags=('--clean',),
+ kwargs={
+ 'action': 'store_false',
+ 'dest': 'dirty',
+ 'help': 'Clean environment before installing package.'
+ })
+_arguments['clean'] = parms
+
+parms = Bunch(
+ flags=('--dirty',),
+ kwargs={
+ 'action': 'store_true',
+ 'dest': 'dirty',
+ 'help': 'Do NOT clean environment before installing.'
+ })
+_arguments['dirty'] = parms
diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py
index 08386cac07..22966a26eb 100644
--- a/lib/spack/spack/cmd/diy.py
+++ b/lib/spack/spack/cmd/diy.py
@@ -30,6 +30,7 @@ import llnl.util.tty as tty
import spack
import spack.cmd
+import spack.cmd.common.arguments as arguments
from spack.cmd.edit import edit_package
from spack.stage import DIYStage
@@ -52,9 +53,9 @@ def setup_parser(subparser):
subparser.add_argument(
'spec', nargs=argparse.REMAINDER,
help="specs to use for install. Must contain package AND version.")
- subparser.add_argument(
- '--dirty', action='store_true', dest='dirty',
- help="Install a package *without* cleaning the environment.")
+
+ cd_group = subparser.add_mutually_exclusive_group()
+ arguments.add_common_arguments(cd_group, ['clean', 'dirty'])
def diy(self, args):
diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py
index 417e07e9c4..79af35d3a0 100644
--- a/lib/spack/spack/cmd/install.py
+++ b/lib/spack/spack/cmd/install.py
@@ -34,6 +34,7 @@ import llnl.util.filesystem as fs
import llnl.util.tty as tty
import spack
import spack.cmd
+import spack.cmd.common.arguments as arguments
from spack.build_environment import InstallError
from spack.fetch_strategy import FetchError
from spack.package import PackageBase
@@ -70,9 +71,10 @@ the dependencies."""
subparser.add_argument(
'--fake', action='store_true', dest='fake',
help="Fake install. Just remove prefix and create a fake file.")
- subparser.add_argument(
- '--dirty', action='store_true', dest='dirty',
- help="Install a package *without* cleaning the environment.")
+
+ cd_group = subparser.add_mutually_exclusive_group()
+ arguments.add_common_arguments(cd_group, ['clean', 'dirty'])
+
subparser.add_argument(
'package',
nargs=argparse.REMAINDER,
diff --git a/lib/spack/spack/cmd/purge.py b/lib/spack/spack/cmd/purge.py
index 26d160635c..66cfc2af29 100644
--- a/lib/spack/spack/cmd/purge.py
+++ b/lib/spack/spack/cmd/purge.py
@@ -36,8 +36,8 @@ def setup_parser(subparser):
'-d', '--downloads', action='store_true',
help="Remove cached downloads.")
subparser.add_argument(
- '-u', '--user-cache', action='store_true',
- help="Remove caches in user home directory. Includes virtual indices.")
+ '-m', '--misc-cache', action='store_true',
+ help="Remove long-lived caches, like the virtual package index.")
subparser.add_argument(
'-a', '--all', action='store_true',
help="Remove all of the above.")
@@ -45,7 +45,7 @@ def setup_parser(subparser):
def purge(parser, args):
# Special case: no flags.
- if not any((args.stage, args.downloads, args.user_cache, args.all)):
+ if not any((args.stage, args.downloads, args.misc_cache, args.all)):
stage.purge()
return
@@ -54,5 +54,5 @@ def purge(parser, args):
stage.purge()
if args.downloads or args.all:
spack.fetch_cache.destroy()
- if args.user_cache or args.all:
- spack.user_cache.destroy()
+ if args.misc_cache or args.all:
+ spack.misc_cache.destroy()
diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py
index 6509228a98..953906975e 100644
--- a/lib/spack/spack/cmd/setup.py
+++ b/lib/spack/spack/cmd/setup.py
@@ -32,6 +32,7 @@ import llnl.util.tty as tty
import spack
import spack.cmd
import spack.cmd.install as install
+import spack.cmd.common.arguments as arguments
from llnl.util.filesystem import set_executable
from spack import which
from spack.cmd.edit import edit_package
@@ -50,9 +51,9 @@ def setup_parser(subparser):
subparser.add_argument(
'spec', nargs=argparse.REMAINDER,
help="specs to use for install. Must contain package AND version.")
- subparser.add_argument(
- '--dirty', action='store_true', dest='dirty',
- help="Install a package *without* cleaning the environment.")
+
+ cd_group = subparser.add_mutually_exclusive_group()
+ arguments.add_common_arguments(cd_group, ['clean', 'dirty'])
def spack_transitive_include_path():
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index 1bd1689150..904195e47e 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -298,6 +298,9 @@ class ConfigScope(object):
"""Empty cached config information."""
self.sections = {}
+ def __repr__(self):
+ return '<ConfigScope: %s: %s>' % (self.name, self.path)
+
#
# Below are configuration scopes.
#
diff --git a/lib/spack/spack/file_cache.py b/lib/spack/spack/file_cache.py
index 31ae009836..e37f77d68d 100644
--- a/lib/spack/spack/file_cache.py
+++ b/lib/spack/spack/file_cache.py
@@ -113,7 +113,7 @@ class FileCache(object):
Returns a ReadTransaction context manager and opens the cache file for
reading. You can use it like this:
- with spack.user_cache.read_transaction(key) as cache_file:
+ with file_cache_object.read_transaction(key) as cache_file:
cache_file.read()
"""
diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py
index 2c2804aea1..5e2a840e14 100644
--- a/lib/spack/spack/modules.py
+++ b/lib/spack/spack/modules.py
@@ -675,7 +675,7 @@ class LmodModule(EnvModule):
def __init__(self, spec=None):
super(LmodModule, self).__init__(spec)
- self.configuration = CONFIGURATION.get('lmod', {})
+ self.configuration = _module_config.get('lmod', {})
hierarchy_tokens = self.configuration.get('hierarchical_scheme', [])
# TODO : Check that the extra hierarchy tokens specified in the
# TODO : configuration file are actually virtual dependencies
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 6bc3362639..491e21bf95 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -1118,7 +1118,7 @@ class PackageBase(object):
run_tests=False,
fake=False,
explicit=False,
- dirty=False,
+ dirty=None,
**kwargs):
"""Called by commands to install a package and its dependencies.
@@ -1165,6 +1165,10 @@ class PackageBase(object):
rec.explicit = True
return
+ # Dirty argument takes precedence over dirty config setting.
+ if dirty is None:
+ dirty = spack.dirty
+
self._do_install_pop_kwargs(kwargs)
tty.msg("Installing %s" % self.name)
diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py
index 47d6df85b3..94b79accdb 100644
--- a/lib/spack/spack/repository.py
+++ b/lib/spack/spack/repository.py
@@ -620,12 +620,12 @@ class Repo(object):
# Read the old ProviderIndex, or make a new one.
key = self._cache_file
- index_existed = spack.user_cache.init_entry(key)
+ index_existed = spack.misc_cache.init_entry(key)
if index_existed and not self._needs_update:
- with spack.user_cache.read_transaction(key) as f:
+ with spack.misc_cache.read_transaction(key) as f:
self._provider_index = ProviderIndex.from_yaml(f)
else:
- with spack.user_cache.write_transaction(key) as (old, new):
+ with spack.misc_cache.write_transaction(key) as (old, new):
if old:
self._provider_index = ProviderIndex.from_yaml(old)
else:
@@ -701,7 +701,7 @@ class Repo(object):
self._all_package_names = []
# Get index modification time.
- index_mtime = spack.user_cache.mtime(self._cache_file)
+ index_mtime = spack.misc_cache.mtime(self._cache_file)
for pkg_name in os.listdir(self.packages_path):
# Skip non-directories in the package root.
diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py
index de6cd79594..d5e1791b40 100644
--- a/lib/spack/spack/test/config.py
+++ b/lib/spack/spack/test/config.py
@@ -221,7 +221,8 @@ class ConfigTest(MockPackagesTest):
self.check_config(b_comps, *self.b_comp_specs)
def check_canonical(self, var, expected):
- """ensure things are substituted properly and canonicalized."""
+ """Ensure that <expected> is substituted properly for <var> in strings
+ containing <var> in various positions."""
path = '/foo/bar/baz'
self.assertEqual(canonicalize_path(var + path),
diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py
index 4e1f243c82..a1919cd437 100644
--- a/lib/spack/spack/test/mock_packages_test.py
+++ b/lib/spack/spack/test/mock_packages_test.py
@@ -191,7 +191,7 @@ config:
misc_cache: ~/.spack/cache
verify_ssl: true
checksum: true
- dirty: false
+ dirty: True
"""
# these are written out to mock config files.
@@ -211,9 +211,6 @@ class MockPackagesTest(unittest.TestCase):
self.db = RepoPath(spack.mock_packages_path)
spack.repo.swap(self.db)
- spack.config.clear_config_caches()
- self.real_scopes = spack.config.config_scopes
-
# Mock up temporary configuration directories
self.temp_config = tempfile.mkdtemp()
self.mock_site_config = os.path.join(self.temp_config, 'site')
@@ -227,6 +224,9 @@ class MockPackagesTest(unittest.TestCase):
# TODO: Mocking this up is kind of brittle b/c ConfigScope
# TODO: constructor modifies config_scopes. Make it cleaner.
+ spack.config.clear_config_caches()
+ self.real_scopes = spack.config.config_scopes
+
spack.config.config_scopes = OrderedDict()
spack.config.ConfigScope('site', self.mock_site_config)
spack.config.ConfigScope('user', self.mock_user_config)
@@ -261,6 +261,7 @@ class MockPackagesTest(unittest.TestCase):
"""Restore the real packages path after any test."""
spack.repo.swap(self.db)
spack.config.config_scopes = self.real_scopes
+
shutil.rmtree(self.temp_config, ignore_errors=True)
spack.config.clear_config_caches()
diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py
index cb3a26e62b..42f072debb 100644
--- a/lib/spack/spack/test/modules.py
+++ b/lib/spack/spack/test/modules.py
@@ -471,7 +471,7 @@ class LmodTests(ModuleFileGeneratorTests):
# Make sure that virtual providers (in the hierarchy) always
# include a hash. Make sure that the module file for the spec
# does not include a hash if hash_length is 0.
- spack.modules.CONFIGURATION = self.configuration_no_hash
+ spack.modules._module_config = self.configuration_no_hash
spec = spack.spec.Spec(mpileaks_spec_string)
spec.concretize()
module = spack.modules.LmodModule(spec)
diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py
index a21142c2cb..64cfa222db 100644
--- a/lib/spack/spack/test/stage.py
+++ b/lib/spack/spack/test/stage.py
@@ -46,9 +46,16 @@ def use_tmp(use_tmp):
not use temporary space for stages.
"""
# mock up config
- path = _test_tmp_path if use_tmp else spack.stage_path
+ assert(_test_tmp_path is not None)
+
+ if use_tmp:
+ path = _test_tmp_path # use temporary stage
+ else:
+ path = spack.stage_path # Use Spack's stage dir (no links)
+
spack.config.update_config(
'config', {'build_stage': [path]}, scope='user')
+
yield
@@ -59,13 +66,28 @@ class StageTest(MockPackagesTest):
by the Stage class. It doesn't actually create the Stage -- that
is done by individual tests.
"""
+ super(StageTest, self).setUp()
+
global _test_tmp_path
+ #
+ # Mock up a stage area that looks like this:
+ #
+ # TMPDIR/ test_files_dir
+ # tmp/ test_tmp_path (where stage should be)
+ # test-files/ archive_dir_path
+ # README.txt test_readme (contains "hello world!\n")
+ # test-files.tar.gz archive_url = file:///path/to/this
+ #
self.test_files_dir = tempfile.mkdtemp()
self.test_tmp_path = os.path.realpath(
os.path.join(self.test_files_dir, 'tmp'))
_test_tmp_path = self.test_tmp_path
+ # set _test_tmp_path as the default test directory to use for stages.
+ spack.config.update_config(
+ 'config', {'build_stage': [_test_tmp_path]}, scope='user')
+
self.archive_dir = 'test-files'
self.archive_name = self.archive_dir + '.tar.gz'
archive_dir_path = os.path.join(self.test_files_dir,
@@ -99,6 +121,8 @@ class StageTest(MockPackagesTest):
def tearDown(self):
"""Blows away the test environment directory."""
+ super(StageTest, self).tearDown()
+
shutil.rmtree(self.test_files_dir, ignore_errors=True)
# chdir back to original working dir
@@ -138,7 +162,7 @@ class StageTest(MockPackagesTest):
self.assertFalse(os.path.islink(target))
# Make sure the directory is in the place we asked it to
- # be (see setUp and tearDown)
+ # be (see setUp, tearDown, and use_tmp)
self.assertTrue(target.startswith(self.test_tmp_path))
else:
diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py
index 5332115ae9..7235f6b756 100644
--- a/lib/spack/spack/util/path.py
+++ b/lib/spack/spack/util/path.py
@@ -50,18 +50,22 @@ def substitute_config_variables(path):
- $spack The Spack instance's prefix
- $user The current user's username
- $tempdir Default temporary directory returned by tempfile.gettempdir()
+
+ These are substituted case-insensitively into the path, and users can
+ use either ``$var`` or ``${var}`` syntax for the variables.
+
"""
# Look up replacements for re.sub in the replacements dict.
def repl(match):
m = match.group(0).strip('${}')
- return replacements.get(m, match.group(0))
+ return replacements.get(m.lower(), match.group(0))
# Replace $var or ${var}.
return re.sub(r'(\$\w+\b|\$\{\w+\})', repl, path)
def canonicalize_path(path):
- """Substitute $spack, expand user home, take abspath."""
+ """Substitute config vars, expand user home, take abspath."""
path = substitute_config_variables(path)
path = os.path.expanduser(path)
path = os.path.abspath(path)