diff options
| -rw-r--r-- | etc/spack/defaults/bootstrap.yaml | 7 | ||||
| -rw-r--r-- | lib/spack/spack/bootstrap.py | 21 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/bootstrap.py | 110 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/clean.py | 5 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/find.py | 6 | ||||
| -rw-r--r-- | lib/spack/spack/config.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/paths.py | 5 | ||||
| -rw-r--r-- | lib/spack/spack/schema/bootstrap.py | 26 | ||||
| -rw-r--r-- | lib/spack/spack/schema/merged.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/store.py | 4 | ||||
| -rw-r--r-- | lib/spack/spack/test/bootstrap.py | 27 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/bootstrap.py | 101 | ||||
| -rwxr-xr-x | share/spack/spack-completion.bash | 32 |
13 files changed, 337 insertions, 11 deletions
diff --git a/etc/spack/defaults/bootstrap.yaml b/etc/spack/defaults/bootstrap.yaml new file mode 100644 index 0000000000..78bb43f4b1 --- /dev/null +++ b/etc/spack/defaults/bootstrap.yaml @@ -0,0 +1,7 @@ +bootstrap: + # If set to false Spack will not bootstrap missing software, + # but will instead raise an error. + enable: true + # Root directory for bootstrapping work. The software bootstrapped + # by Spack is installed in a "store" subfolder of this root directory + root: ~/.spack/bootstrap diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py index 86c3110983..96f11b631d 100644 --- a/lib/spack/spack/bootstrap.py +++ b/lib/spack/spack/bootstrap.py @@ -25,6 +25,7 @@ import spack.spec import spack.store import spack.user_environment as uenv import spack.util.executable +import spack.util.path from spack.util.environment import EnvironmentModifications @@ -216,9 +217,10 @@ def _bootstrap_config_scopes(): @contextlib.contextmanager def ensure_bootstrap_configuration(): + bootstrap_store_path = store_path() with spack.architecture.use_platform(spack.architecture.real_platform()): with spack.repo.use_repositories(spack.paths.packages_path): - with spack.store.use_store(spack.paths.user_bootstrap_store): + with spack.store.use_store(bootstrap_store_path): # Default configuration scopes excluding command line # and builtin but accounting for platform specific scopes config_scopes = _bootstrap_config_scopes() @@ -227,6 +229,23 @@ def ensure_bootstrap_configuration(): yield +def store_path(): + """Path to the store used for bootstrapped software""" + enabled = spack.config.get('bootstrap:enable', True) + if not enabled: + msg = ('bootstrapping is currently disabled. ' + 'Use "spack bootstrap enable" to enable it') + raise RuntimeError(msg) + + bootstrap_root_path = spack.config.get( + 'bootstrap:root', spack.paths.user_bootstrap_path + ) + bootstrap_store_path = spack.util.path.canonicalize_path( + os.path.join(bootstrap_root_path, 'store') + ) + return bootstrap_store_path + + def clingo_root_spec(): # Construct the root spec that will be used to bootstrap clingo spec_str = 'clingo-bootstrap@spack+python' diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py new file mode 100644 index 0000000000..2c95da117c --- /dev/null +++ b/lib/spack/spack/cmd/bootstrap.py @@ -0,0 +1,110 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os.path +import shutil + +import llnl.util.tty + +import spack.cmd.common.arguments +import spack.config +import spack.main +import spack.util.path + +description = "manage bootstrap configuration" +section = "system" +level = "long" + + +def _add_scope_option(parser): + scopes = spack.config.scopes() + scopes_metavar = spack.config.scopes_metavar + parser.add_argument( + '--scope', choices=scopes, metavar=scopes_metavar, + help="configuration scope to read/modify" + ) + + +def setup_parser(subparser): + sp = subparser.add_subparsers(dest='subcommand') + + enable = sp.add_parser('enable', help='enable bootstrapping') + _add_scope_option(enable) + + disable = sp.add_parser('disable', help='disable bootstrapping') + _add_scope_option(disable) + + reset = sp.add_parser( + 'reset', help='reset bootstrapping configuration to Spack defaults' + ) + spack.cmd.common.arguments.add_common_arguments( + reset, ['yes_to_all'] + ) + + root = sp.add_parser( + 'root', help='get/set the root bootstrap directory' + ) + _add_scope_option(root) + root.add_argument( + 'path', nargs='?', default=None, + help='set the bootstrap directory to this value' + ) + + +def _enable_or_disable(args): + # Set to True if we called "enable", otherwise set to false + value = args.subcommand == 'enable' + spack.config.set('bootstrap:enable', value, scope=args.scope) + + +def _reset(args): + if not args.yes_to_all: + msg = [ + "Bootstrapping configuration is being reset to Spack's defaults. " + "Current configuration will be lost.\n", + "Do you want to continue?" + ] + ok_to_continue = llnl.util.tty.get_yes_or_no( + ''.join(msg), default=True + ) + if not ok_to_continue: + raise RuntimeError('Aborting') + + for scope in spack.config.config.file_scopes: + # The default scope should stay untouched + if scope.name == 'defaults': + continue + + # If we are in an env scope we can't delete a file, but the best we + # can do is nullify the corresponding configuration + if (scope.name.startswith('env') and + spack.config.get('bootstrap', scope=scope.name)): + spack.config.set('bootstrap', {}, scope=scope.name) + continue + + # If we are outside of an env scope delete the bootstrap.yaml file + bootstrap_yaml = os.path.join(scope.path, 'bootstrap.yaml') + backup_file = bootstrap_yaml + '.bkp' + if os.path.exists(bootstrap_yaml): + shutil.move(bootstrap_yaml, backup_file) + + +def _root(args): + if args.path: + spack.config.set('bootstrap:root', args.path, scope=args.scope) + + root = spack.config.get('bootstrap:root', default=None, scope=args.scope) + if root: + root = spack.util.path.canonicalize_path(root) + print(root) + + +def bootstrap(parser, args): + callbacks = { + 'enable': _enable_or_disable, + 'disable': _enable_or_disable, + 'reset': _reset, + 'root': _root + } + callbacks[args.subcommand](args) diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 273b1789ad..3b66358293 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -9,6 +9,7 @@ import shutil import llnl.util.tty as tty +import spack.bootstrap import spack.caches import spack.cmd.common.arguments as arguments import spack.cmd.test @@ -102,7 +103,7 @@ def clean(parser, args): if args.bootstrap: msg = 'Removing software in "{0}"' - tty.msg(msg.format(spack.paths.user_bootstrap_store)) - with spack.store.use_store(spack.paths.user_bootstrap_store): + tty.msg(msg.format(spack.bootstrap.store_path())) + with spack.store.use_store(spack.bootstrap.store_path()): uninstall = spack.main.SpackCommand('uninstall') uninstall('-a', '-y') diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index f532b684bd..875c236164 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -13,6 +13,7 @@ import llnl.util.lang import llnl.util.tty as tty import llnl.util.tty.color as color +import spack.bootstrap import spack.cmd as cmd import spack.cmd.common.arguments as arguments import spack.environment as ev @@ -207,9 +208,10 @@ def find(parser, args): q_args = query_arguments(args) # Query the current store or the internal bootstrap store if required if args.bootstrap: + bootstrap_store_path = spack.bootstrap.store_path() msg = 'Showing internal bootstrap store at "{0}"' - tty.msg(msg.format(spack.paths.user_bootstrap_store)) - with spack.store.use_store(spack.paths.user_bootstrap_store): + tty.msg(msg.format(bootstrap_store_path)) + with spack.store.use_store(bootstrap_store_path): results = args.specs(**q_args) else: results = args.specs(**q_args) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 038a879a81..5ceaef8e7b 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -51,6 +51,7 @@ import spack.architecture import spack.compilers import spack.paths import spack.schema +import spack.schema.bootstrap import spack.schema.compilers import spack.schema.config import spack.schema.env @@ -74,6 +75,7 @@ section_schemas = { 'modules': spack.schema.modules.schema, 'config': spack.schema.config.schema, 'upstreams': spack.schema.upstreams.schema, + 'bootstrap': spack.schema.bootstrap.schema } # Same as above, but including keys for environments diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 6b41049de8..76eb9dfdac 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -11,10 +11,10 @@ dependencies. """ import os -from llnl.util.filesystem import ancestor +import llnl.util.filesystem #: This file lives in $prefix/lib/spack/spack/__file__ -prefix = ancestor(__file__, 4) +prefix = llnl.util.filesystem.ancestor(__file__, 4) #: synonym for prefix spack_root = prefix @@ -53,7 +53,6 @@ mock_packages_path = os.path.join(repos_path, "builtin.mock") #: User configuration location user_config_path = os.path.expanduser('~/.spack') user_bootstrap_path = os.path.join(user_config_path, 'bootstrap') -user_bootstrap_store = os.path.join(user_bootstrap_path, 'store') reports_path = os.path.join(user_config_path, "reports") monitor_path = os.path.join(reports_path, "monitor") diff --git a/lib/spack/spack/schema/bootstrap.py b/lib/spack/spack/schema/bootstrap.py new file mode 100644 index 0000000000..0505f09003 --- /dev/null +++ b/lib/spack/spack/schema/bootstrap.py @@ -0,0 +1,26 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Schema for bootstrap.yaml configuration file.""" + +properties = { + 'bootstrap': { + 'type': 'object', + 'properties': { + 'enable': {'type': 'boolean'}, + 'root': { + 'type': 'string' + }, + } + } +} + +#: Full schema with metadata +schema = { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack bootstrap configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'properties': properties, +} diff --git a/lib/spack/spack/schema/merged.py b/lib/spack/spack/schema/merged.py index 9555f7b194..11db4f78df 100644 --- a/lib/spack/spack/schema/merged.py +++ b/lib/spack/spack/schema/merged.py @@ -10,6 +10,7 @@ """ from llnl.util.lang import union_dicts +import spack.schema.bootstrap import spack.schema.cdash import spack.schema.compilers import spack.schema.config @@ -23,6 +24,7 @@ import spack.schema.upstreams #: Properties for inclusion in other schemas properties = union_dicts( + spack.schema.bootstrap.properties, spack.schema.cdash.properties, spack.schema.compilers.properties, spack.schema.config.properties, diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 4fcd96cdd8..a04ece2027 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -193,6 +193,7 @@ class Store(object): def _store(): """Get the singleton store instance.""" + import spack.bootstrap config_dict = spack.config.get('config') root, unpadded_root, projections = parse_install_tree(config_dict) hash_length = spack.config.get('config:install_hash_length') @@ -201,7 +202,8 @@ def _store(): # reserved by Spack to bootstrap its own dependencies, since this would # lead to bizarre behaviors (e.g. cleaning the bootstrap area would wipe # user installed software) - if spack.paths.user_bootstrap_store == root: + enable_bootstrap = spack.config.get('bootstrap:enable', True) + if enable_bootstrap and spack.bootstrap.store_path() == root: msg = ('please change the install tree root "{0}" in your ' 'configuration [path reserved for Spack internal use]') raise ValueError(msg.format(root)) diff --git a/lib/spack/spack/test/bootstrap.py b/lib/spack/spack/test/bootstrap.py index 97687ddb4d..fccf67e569 100644 --- a/lib/spack/spack/test/bootstrap.py +++ b/lib/spack/spack/test/bootstrap.py @@ -6,6 +6,7 @@ import pytest import spack.bootstrap import spack.store +import spack.util.path @pytest.mark.regression('22294') @@ -22,5 +23,29 @@ def test_store_is_restored_correctly_after_bootstrap(mutable_config, tmpdir): # Test that within the context manager we use the bootstrap store # and that outside we restore the correct location with spack.bootstrap.ensure_bootstrap_configuration(): - assert spack.store.root == spack.paths.user_bootstrap_store + assert spack.store.root == spack.bootstrap.store_path() assert spack.store.root == user_path + + +@pytest.mark.parametrize('config_value,expected', [ + # Absolute path without expansion + ('/opt/spack/bootstrap', '/opt/spack/bootstrap/store'), + # Path with placeholder + ('$spack/opt/bootstrap', '$spack/opt/bootstrap/store'), +]) +def test_store_path_customization(config_value, expected, mutable_config): + # Set the current configuration to a specific value + spack.config.set('bootstrap:root', config_value) + + # Check the store path + current = spack.bootstrap.store_path() + assert current == spack.util.path.canonicalize_path(expected) + + +def test_raising_exception_if_bootstrap_disabled(mutable_config): + # Disable bootstrapping in config.yaml + spack.config.set('bootstrap:enable', False) + + # Check the correct exception is raised + with pytest.raises(RuntimeError, match='bootstrapping is currently disabled'): + spack.bootstrap.store_path() diff --git a/lib/spack/spack/test/cmd/bootstrap.py b/lib/spack/spack/test/cmd/bootstrap.py new file mode 100644 index 0000000000..0537d85faa --- /dev/null +++ b/lib/spack/spack/test/cmd/bootstrap.py @@ -0,0 +1,101 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os.path + +import pytest + +import spack.config +import spack.environment +import spack.main + +_bootstrap = spack.main.SpackCommand('bootstrap') + + +@pytest.mark.parametrize('scope', [ + None, 'site', 'system', 'user' +]) +def test_enable_and_disable(mutable_config, scope): + scope_args = [] + if scope: + scope_args = ['--scope={0}'.format(scope)] + + _bootstrap('enable', *scope_args) + assert spack.config.get('bootstrap:enable', scope=scope) is True + + _bootstrap('disable', *scope_args) + assert spack.config.get('bootstrap:enable', scope=scope) is False + + +@pytest.mark.parametrize('scope', [ + None, 'site', 'system', 'user' +]) +def test_root_get_and_set(mutable_config, scope): + scope_args, path = [], '/scratch/spack/bootstrap' + if scope: + scope_args = ['--scope={0}'.format(scope)] + + _bootstrap('root', path, *scope_args) + out = _bootstrap('root', *scope_args, output=str) + assert out.strip() == path + + +@pytest.mark.parametrize('scopes', [ + ('site',), + ('system', 'user') +]) +def test_reset_in_file_scopes(mutable_config, scopes): + # Assert files are created in the right scopes + bootstrap_yaml_files = [] + for s in scopes: + _bootstrap('disable', '--scope={0}'.format(s)) + scope_path = spack.config.config.scopes[s].path + bootstrap_yaml = os.path.join( + scope_path, 'bootstrap.yaml' + ) + assert os.path.exists(bootstrap_yaml) + bootstrap_yaml_files.append(bootstrap_yaml) + + _bootstrap('reset', '-y') + for bootstrap_yaml in bootstrap_yaml_files: + assert not os.path.exists(bootstrap_yaml) + + +def test_reset_in_environment(mutable_mock_env_path, mutable_config): + env = spack.main.SpackCommand('env') + env('create', 'bootstrap-test') + current_environment = spack.environment.read('bootstrap-test') + + with current_environment: + _bootstrap('disable') + assert spack.config.get('bootstrap:enable') is False + _bootstrap('reset', '-y') + # We have no default settings in tests + assert spack.config.get('bootstrap:enable') is None + + # Check that reset didn't delete the entire file + spack_yaml = os.path.join(current_environment.path, 'spack.yaml') + assert os.path.exists(spack_yaml) + + +def test_reset_in_file_scopes_overwrites_backup_files(mutable_config): + # Create a bootstrap.yaml with some config + _bootstrap('disable', '--scope=site') + scope_path = spack.config.config.scopes['site'].path + bootstrap_yaml = os.path.join(scope_path, 'bootstrap.yaml') + assert os.path.exists(bootstrap_yaml) + + # Reset the bootstrap configuration + _bootstrap('reset', '-y') + backup_file = bootstrap_yaml + '.bkp' + assert not os.path.exists(bootstrap_yaml) + assert os.path.exists(backup_file) + + # Iterate another time + _bootstrap('disable', '--scope=site') + assert os.path.exists(bootstrap_yaml) + assert os.path.exists(backup_file) + _bootstrap('reset', '-y') + assert not os.path.exists(bootstrap_yaml) + assert os.path.exists(backup_file) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index cd5a01c42d..44cc7945fd 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -333,7 +333,7 @@ _spack() { then SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars" else - SPACK_COMPREPLY="activate add analyze arch audit blame build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config containerize create deactivate debug dependencies dependents deprecate dev-build develop docs edit env extensions external fetch find flake8 gc gpg graph help info install license list load location log-parse maintainers mark mirror module monitor patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style test test-env tutorial undevelop uninstall unit-test unload url verify versions view" + SPACK_COMPREPLY="activate add analyze arch audit blame bootstrap build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config containerize create deactivate debug dependencies dependents deprecate dev-build develop docs edit env extensions external fetch find flake8 gc gpg graph help info install license list load location log-parse maintainers mark mirror module monitor patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style test test-env tutorial undevelop uninstall unit-test unload url verify versions view" fi } @@ -416,6 +416,36 @@ _spack_blame() { fi } +_spack_bootstrap() { + if $list_options + then + SPACK_COMPREPLY="-h --help" + else + SPACK_COMPREPLY="enable disable reset root" + fi +} + +_spack_bootstrap_enable() { + SPACK_COMPREPLY="-h --help --scope" +} + +_spack_bootstrap_disable() { + SPACK_COMPREPLY="-h --help --scope" +} + +_spack_bootstrap_reset() { + SPACK_COMPREPLY="-h --help -y --yes-to-all" +} + +_spack_bootstrap_root() { + if $list_options + then + SPACK_COMPREPLY="-h --help --scope" + else + SPACK_COMPREPLY="" + fi +} + _spack_build_env() { if $list_options then |