diff options
| -rw-r--r-- | etc/spack/defaults/bootstrap.yaml | 2 | ||||
| -rw-r--r-- | etc/spack/defaults/config.yaml | 10 | ||||
| -rw-r--r-- | lib/spack/docs/configuration.rst | 41 | ||||
| -rw-r--r-- | lib/spack/spack/bootstrap.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/caches.py | 9 | ||||
| -rw-r--r-- | lib/spack/spack/config.py | 51 | ||||
| -rw-r--r-- | lib/spack/spack/paths.py | 90 | ||||
| -rw-r--r-- | lib/spack/spack/test/config.py | 70 | ||||
| -rw-r--r-- | lib/spack/spack/util/path.py | 12 |
9 files changed, 234 insertions, 53 deletions
diff --git a/etc/spack/defaults/bootstrap.yaml b/etc/spack/defaults/bootstrap.yaml index 392c48b7bb..7a72bdfe6a 100644 --- a/etc/spack/defaults/bootstrap.yaml +++ b/etc/spack/defaults/bootstrap.yaml @@ -4,7 +4,7 @@ bootstrap: enable: true # Root directory for bootstrapping work. The software bootstrapped # by Spack is installed in a "store" subfolder of this root directory - root: ~/.spack/bootstrap + root: $user_config_path/bootstrap # Methods that can be used to bootstrap software. Each method may or # may not be able to bootstrap all of the software that Spack needs, # depending on its type. diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 3487d33162..eb0d4fc409 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -42,8 +42,8 @@ config: # (i.e., ``$TMP` or ``$TMPDIR``). # # Another option that prevents conflicts and potential permission issues is - # to specify `~/.spack/stage`, which ensures each user builds in their home - # directory. + # to specify `$user_cache_path/stage`, which ensures each user builds in their + # home directory. # # A more traditional path uses the value of `$spack/var/spack/stage`, which # builds directly inside Spack's instance without staging them in a @@ -60,13 +60,13 @@ config: # identifies Spack staging to avoid accidentally wiping out non-Spack work. build_stage: - $tempdir/$user/spack-stage - - ~/.spack/stage + - $user_cache_path/stage # - $spack/var/spack/stage # Directory in which to run tests and store test results. # Tests will be stored in directories named by date/time and package # name/hash. - test_stage: ~/.spack/test + test_stage: $user_cache_path/test # Cache directory for already downloaded source tarballs and archived # repositories. This can be purged with `spack clean --downloads`. @@ -75,7 +75,7 @@ config: # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` - misc_cache: ~/.spack/cache + misc_cache: $user_cache_path/cache # Timeout in seconds used for downloading sources etc. This only applies diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 10b442ab74..5d41f86997 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -402,12 +402,17 @@ Spack-specific variables Spack understands several special variables. These are: +* ``$env``: name of the currently active :ref:`environment <environments>` * ``$spack``: path to the prefix of this Spack installation * ``$tempdir``: default system temporary directory (as specified in Python's `tempfile.tempdir <https://docs.python.org/2/library/tempfile.html#tempfile.tempdir>`_ variable. * ``$user``: name of the current user +* ``$user_config_path``: user configuration directory (``~/.spack`` unless + :ref:`overridden <local-config-overrides>`) +* ``$user_cache_path``: user cache directory (``~/.spack`` unless + :ref:`overridden <local-config-overrides>`) Note that, as with shell variables, you can write these as ``$varname`` or with braces to distinguish the variable from surrounding characters: @@ -562,3 +567,39 @@ built in and are not overridden by a configuration file. The command line. ``dirty`` and ``install_tree`` come from the custom scopes ``./my-scope`` and ``./my-scope-2``, and all other configuration options come from the default configuration files that ship with Spack. + +.. _local-config-overrides: + +------------------------------ +Overriding Local Configuration +------------------------------ + +Spack's ``system`` and ``user`` scopes provide ways for administrators and users to set +global defaults for all Spack instances, but for use cases where one wants a clean Spack +installation, these scopes can be undesirable. For example, users may want to opt out of +global system configuration, or they may want to ignore their own home directory +settings when running in a continuous integration environment. + +Spack also, by default, keeps various caches and user data in ``~/.spack``, but +users may want to override these locations. + +Spack provides three environment variables that allow you to override or opt out of +configuration locations: + +* ``SPACK_USER_CONFIG_PATH``: Override the path to use for the + ``user`` (``~/.spack``) scope. +* ``SPACK_SYSTEM_CONFIG_PATH``: Override the path to use for the ``system`` + (``/etc/spack``) scope. +* ``SPACK_DISABLE_LOCAL_CONFIG``: set this environment variable to completely disable + **both** the system and user configuration directories. Spack will only consider its + own defaults and ``site`` configuration locations. + +And one that allows you to move the default cache location: + +* ``SPACK_USER_CACHE_PATH``: Override the default path to use for user data + (misc_cache, tests, reports, etc.) + +With these settings, if you want to isolate Spack in a CI environment, you can do this:: + + export SPACK_DISABLE_LOCAL_CONFIG=true + export SPACK_USER_CACHE_PATH=/tmp/spack diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py index 0bdf49c5e8..78ed54429f 100644 --- a/lib/spack/spack/bootstrap.py +++ b/lib/spack/spack/bootstrap.py @@ -582,7 +582,7 @@ def store_path(): def _root_path(): """Root of all the bootstrap related folders""" return spack.config.get( - 'bootstrap:root', spack.paths.user_bootstrap_path + 'bootstrap:root', spack.paths.default_user_bootstrap_path ) diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py index 67c93cb306..71695aaf5c 100644 --- a/lib/spack/spack/caches.py +++ b/lib/spack/spack/caches.py @@ -23,11 +23,8 @@ def misc_cache_location(): Currently the ``misc_cache`` stores indexes for virtual dependency providers and for which packages provide which tags. """ - path = spack.config.get('config:misc_cache') - if not path: - path = os.path.join(spack.paths.user_config_path, 'cache') - path = spack.util.path.canonicalize_path(path) - return path + path = spack.config.get('config:misc_cache', spack.paths.default_misc_cache_path) + return spack.util.path.canonicalize_path(path) def _misc_cache(): @@ -47,7 +44,7 @@ def fetch_cache_location(): """ path = spack.config.get('config:source_cache') if not path: - path = os.path.join(spack.paths.var_path, "cache") + path = spack.paths.default_fetch_cache_path path = spack.util.path.canonicalize_path(path) return path diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 470a2b958c..c82729b007 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -89,24 +89,6 @@ configuration_defaults_path = ( 'defaults', os.path.join(spack.paths.etc_path, 'spack', 'defaults') ) -#: Builtin paths to configuration files in Spack -configuration_paths = ( - # Default configuration scope is the lowest-level scope. These are - # versioned with Spack and can be overridden by systems, sites or users - configuration_defaults_path, - - # System configuration is per machine. - # No system-level configs should be checked into spack by default - ('system', os.path.join(spack.paths.system_etc_path, 'spack')), - - # Site configuration is per spack instance, for sites or projects - # No site-level configs should be checked into spack by default. - ('site', os.path.join(spack.paths.etc_path, 'spack')), - - # User configuration can override both spack defaults and site config - ('user', spack.paths.user_config_path) -) - #: Hard-coded default values for some key configuration options. #: This ensures that Spack will still work even if config.yaml in #: the defaults scope is removed. @@ -808,8 +790,37 @@ def _config(): cfg = Configuration() # first do the builtin, hardcoded defaults - defaults = InternalConfigScope('_builtin', config_defaults) - cfg.push_scope(defaults) + builtin = InternalConfigScope('_builtin', config_defaults) + cfg.push_scope(builtin) + + # Builtin paths to configuration files in Spack + configuration_paths = [ + # Default configuration scope is the lowest-level scope. These are + # versioned with Spack and can be overridden by systems, sites or users + configuration_defaults_path, + ] + + disable_local_config = "SPACK_DISABLE_LOCAL_CONFIG" in os.environ + + # System configuration is per machine. + # This is disabled if user asks for no local configuration. + if not disable_local_config: + configuration_paths.append( + ('system', spack.paths.system_config_path), + ) + + # Site configuration is per spack instance, for sites or projects + # No site-level configs should be checked into spack by default. + configuration_paths.append( + ('site', os.path.join(spack.paths.etc_path, 'spack')), + ) + + # User configuration can override both spack defaults and site config + # This is disabled if user asks for no local configuration. + if not disable_local_config: + configuration_paths.append( + ('user', spack.paths.user_config_path) + ) # add each scope and its platform-specific directory for name, path in configuration_paths: diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 627be54bd7..70d138a932 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -41,30 +41,88 @@ build_systems_path = os.path.join(module_path, 'build_systems') operating_system_path = os.path.join(module_path, 'operating_systems') test_path = os.path.join(module_path, "test") hooks_path = os.path.join(module_path, "hooks") -var_path = os.path.join(prefix, "var", "spack") -repos_path = os.path.join(var_path, "repos") -tests_path = os.path.join(var_path, "tests") +opt_path = os.path.join(prefix, "opt") share_path = os.path.join(prefix, "share", "spack") +etc_path = os.path.join(prefix, "etc") -# Paths to built-in Spack repositories. -packages_path = os.path.join(repos_path, "builtin") -mock_packages_path = os.path.join(repos_path, "builtin.mock") -#: User configuration location -user_config_path = os.path.expanduser('~/.spack') -user_bootstrap_path = os.path.join(user_config_path, 'bootstrap') -reports_path = os.path.join(user_config_path, "reports") -monitor_path = os.path.join(reports_path, "monitor") +# +# Things in $spack/var/spack +# +var_path = os.path.join(prefix, "var", "spack") -# We cache repositories (git) in first, extracted metadata in second -user_repos_cache_path = os.path.join(user_config_path, 'git_repos') +# read-only things in $spack/var/spack +repos_path = os.path.join(var_path, "repos") +packages_path = os.path.join(repos_path, "builtin") +mock_packages_path = os.path.join(repos_path, "builtin.mock") -opt_path = os.path.join(prefix, "opt") -etc_path = os.path.join(prefix, "etc") -system_etc_path = '/etc' +# +# Writable things in $spack/var/spack +# TODO: Deprecate these, as we want a read-only spack prefix by default. +# TODO: These should probably move to user cache, or some other location. +# +# fetch cache for downloaded files +default_fetch_cache_path = os.path.join(var_path, "cache") # GPG paths. gpg_keys_path = os.path.join(var_path, "gpg") mock_gpg_data_path = os.path.join(var_path, "gpg.mock", "data") mock_gpg_keys_path = os.path.join(var_path, "gpg.mock", "keys") gpg_path = os.path.join(opt_path, "spack", "gpg") + + +# Below paths are where Spack can write information for the user. +# Some are caches, some are not exactly caches. +# +# The options that start with `default_` below are overridable in +# `config.yaml`, but they default to use `user_cache_path/<location>`. +# +# You can override the top-level directory (the user cache path) by +# setting `SPACK_USER_CACHE_PATH`. Otherwise it defaults to ~/.spack. +# +def _get_user_cache_path(): + return os.path.expanduser(os.getenv('SPACK_USER_CACHE_PATH') or "~/.spack") + + +user_cache_path = _get_user_cache_path() + +#: junit, cdash, etc. reports about builds +reports_path = os.path.join(user_cache_path, "reports") + +#: spack monitor analysis directories +monitor_path = os.path.join(reports_path, "monitor") + +#: git repositories fetched to compare commits to versions +user_repos_cache_path = os.path.join(user_cache_path, 'git_repos') + +#: bootstrap store for bootstrapping clingo and other tools +default_user_bootstrap_path = os.path.join(user_cache_path, 'bootstrap') + +#: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) +default_misc_cache_path = os.path.join(user_cache_path, 'cache') + + +# Below paths pull configuration from the host environment. +# +# There are three environment variables you can use to isolate spack from +# the host environment: +# - `SPACK_USER_CONFIG_PATH`: override `~/.spack` location (for config and caches) +# - `SPACK_SYSTEM_CONFIG_PATH`: override `/etc/spack` configuration scope. +# - `SPACK_DISABLE_LOCAL_CONFIG`: disable both of these locations. + + +# User configuration and caches in $HOME/.spack +def _get_user_config_path(): + return os.path.expanduser(os.getenv('SPACK_USER_CONFIG_PATH') or "~/.spack") + + +# Configuration in /etc/spack on the system +def _get_system_config_path(): + return os.path.expanduser(os.getenv('SPACK_SYSTEM_CONFIG_PATH') or "/etc/spack") + + +#: User configuration location +user_config_path = _get_user_config_path() + +#: System configuration location +system_config_path = _get_system_config_path() diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 704e9994ed..f2f8626b4f 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -432,6 +432,20 @@ def test_substitute_user(mock_low_high_config): ) +def test_substitute_user_config(mock_low_high_config): + user_config_path = spack.paths.user_config_path + assert user_config_path + '/baz' == spack_path.canonicalize_path( + '$user_cache_path/baz' + ) + + +def test_substitute_user_cache(mock_low_high_config): + user_cache_path = spack.paths.user_cache_path + assert user_cache_path + '/baz' == spack_path.canonicalize_path( + '$user_cache_path/baz' + ) + + def test_substitute_tempdir(mock_low_high_config): tempdir = tempfile.gettempdir() assert tempdir == spack_path.canonicalize_path('$tempdir') @@ -1167,3 +1181,59 @@ def test_internal_config_scope_cache_clearing(): internal_scope.clear() # Check that this didn't affect the scope object assert internal_scope.sections['config'] == data + + +def test_system_config_path_is_overridable(working_env): + p = "/some/path" + os.environ['SPACK_SYSTEM_CONFIG_PATH'] = p + assert spack.paths._get_system_config_path() == p + + +def test_system_config_path_is_default_when_env_var_is_empty(working_env): + os.environ['SPACK_SYSTEM_CONFIG_PATH'] = '' + assert "/etc/spack" == spack.paths._get_system_config_path() + + +def test_user_config_path_is_overridable(working_env): + p = "/some/path" + os.environ['SPACK_USER_CONFIG_PATH'] = p + assert p == spack.paths._get_user_config_path() + + +def test_user_config_path_is_default_when_env_var_is_empty(working_env): + os.environ['SPACK_USER_CONFIG_PATH'] = '' + assert os.path.expanduser("~/.spack") == spack.paths._get_user_config_path() + + +def test_local_config_can_be_disabled(working_env): + os.environ['SPACK_DISABLE_LOCAL_CONFIG'] = 'true' + cfg = spack.config._config() + assert "defaults" in cfg.scopes + assert "system" not in cfg.scopes + assert "site" in cfg.scopes + assert "user" not in cfg.scopes + + os.environ['SPACK_DISABLE_LOCAL_CONFIG'] = '' + cfg = spack.config._config() + assert "defaults" in cfg.scopes + assert "system" not in cfg.scopes + assert "site" in cfg.scopes + assert "user" not in cfg.scopes + + del os.environ['SPACK_DISABLE_LOCAL_CONFIG'] + cfg = spack.config._config() + assert "defaults" in cfg.scopes + assert "system" in cfg.scopes + assert "site" in cfg.scopes + assert "user" in cfg.scopes + + +def test_user_cache_path_is_overridable(working_env): + p = "/some/path" + os.environ['SPACK_USER_CACHE_PATH'] = p + assert spack.paths._get_user_cache_path() == p + + +def test_user_cache_path_is_default_when_env_var_is_empty(working_env): + os.environ['SPACK_USER_CACHE_PATH'] = '' + assert os.path.expanduser("~/.spack") == spack.paths._get_user_cache_path() diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 49bb65f7d3..8f76db93bb 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -30,6 +30,8 @@ replacements = { 'spack': spack.paths.prefix, 'user': getpass.getuser(), 'tempdir': tempfile.gettempdir(), + 'user_config_path': spack.paths.user_config_path, + 'user_cache_path': spack.paths.user_cache_path, } # This is intended to be longer than the part of the install path @@ -74,10 +76,12 @@ def substitute_config_variables(path): Spack allows paths in configs to have some placeholders, as follows: - - $spack The Spack instance's prefix - - $user The current user's username - - $tempdir Default temporary directory returned by tempfile.gettempdir() - - $env The active Spack environment. + - $env The active Spack environment. + - $spack The Spack instance's prefix + - $tempdir Default temporary directory returned by tempfile.gettempdir() + - $user The current user's username + - $user_config_path The user configuration directory (~/.spack, unless overridden) + - $user_cache_path The user cache directory (~/.spack, unless overridden) These are substituted case-insensitively into the path, and users can use either ``$var`` or ``${var}`` syntax for the variables. $env is only |