diff options
| -rw-r--r-- | lib/spack/docs/getting_started.rst | 11 | ||||
| -rw-r--r-- | lib/spack/spack/bootstrap.py | 64 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/clean.py | 22 | ||||
| -rw-r--r-- | lib/spack/spack/config.py | 7 | ||||
| -rw-r--r-- | lib/spack/spack/test/bootstrap.py | 19 |
5 files changed, 96 insertions, 27 deletions
diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index ba5b93a567..aa56890847 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -197,17 +197,6 @@ Spack will build the required software on the first request to concretize a spec [ ... ] zlib@1.2.11%gcc@10.1.0+optimize+pic+shared arch=linux-ubuntu18.04-broadwell -.. tip:: - - If you want to speed-up bootstrapping ``clingo`` from sources, you may try to - search for ``cmake`` and ``bison`` on your system: - - .. code-block:: console - - $ spack external find cmake bison - ==> The following specs have been detected on this system and added to /home/spack/.spack/packages.yaml - bison@3.0.4 cmake@3.19.4 - """"""""""""""""""" The Bootstrap Store """"""""""""""""""" diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py index ca92b6fad0..8f4119a04d 100644 --- a/lib/spack/spack/bootstrap.py +++ b/lib/spack/spack/bootstrap.py @@ -26,6 +26,7 @@ import llnl.util.tty as tty import spack.architecture import spack.binary_distribution import spack.config +import spack.detection import spack.environment import spack.main import spack.modules @@ -209,7 +210,7 @@ class _BuildcacheBootstrapper(object): buildcache = spack.main.SpackCommand('buildcache') # Ensure we see only the buildcache being used to bootstrap mirror_scope = spack.config.InternalConfigScope( - 'bootstrap', {'mirrors:': {self.name: self.url}} + 'bootstrap_buildcache', {'mirrors:': {self.name: self.url}} ) with spack.config.override(mirror_scope): # This index is currently needed to get the compiler used to build some @@ -218,7 +219,7 @@ class _BuildcacheBootstrapper(object): index = spack.binary_distribution.update_cache_and_get_specs() if not index: - raise RuntimeError("Could not populate the binary index") + raise RuntimeError("could not populate the binary index") for item in data['verified']: candidate_spec = item['spec'] @@ -279,6 +280,10 @@ class _SourceBootstrapper(object): tty.info("Bootstrapping {0} from sources".format(module)) + # If we compile code from sources detecting a few build tools + # might reduce compilation time by a fair amount + _add_externals_if_missing() + # Try to build and install from sources with spack_python_interpreter(): # Add hint to use frontend operating system on Cray @@ -492,7 +497,11 @@ def _bootstrap_config_scopes(): config_scopes = [ spack.config.InternalConfigScope('_builtin', spack.config.config_defaults) ] - for name, path in spack.config.configuration_paths: + configuration_paths = ( + spack.config.configuration_defaults_path, + ('bootstrap', _config_path()) + ) + for name, path in configuration_paths: platform = spack.architecture.platform().name platform_scope = spack.config.ConfigScope( '/'.join([name, platform]), os.path.join(path, platform) @@ -517,9 +526,19 @@ def _add_compilers_if_missing(): spack.compilers.add_compilers_to_config(new_compilers, init_config=False) +def _add_externals_if_missing(): + search_list = [ + spack.repo.path.get('cmake'), + spack.repo.path.get('bison') + ] + detected_packages = spack.detection.by_executable(search_list) + spack.detection.update_configuration(detected_packages, scope='bootstrap') + + @contextlib.contextmanager def ensure_bootstrap_configuration(): bootstrap_store_path = store_path() + user_configuration = _read_and_sanitize_configuration() with spack.environment.deactivate_environment(): with spack.architecture.use_platform(spack.architecture.real_platform()): with spack.repo.use_repositories(spack.paths.packages_path): @@ -531,11 +550,29 @@ def ensure_bootstrap_configuration(): # We may need to compile code from sources, so ensure we have # compilers for the current platform before switching parts. _add_compilers_if_missing() + spack.config.set('bootstrap', user_configuration['bootstrap']) + spack.config.set('config', user_configuration['config']) with spack.modules.disable_modules(): with spack_python_interpreter(): yield +def _read_and_sanitize_configuration(): + """Read the user configuration that needs to be reused for bootstrapping + and remove the entries that should not be copied over. + """ + # Read the "config" section but pop the install tree (the entry will not be + # considered due to the use_store context manager, so it will be confusing + # to have it in the configuration). + config_yaml = spack.config.get('config') + config_yaml.pop('install_tree', None) + user_configuration = { + 'bootstrap': spack.config.get('bootstrap'), + 'config': config_yaml + } + return user_configuration + + def store_path(): """Path to the store used for bootstrapped software""" enabled = spack.config.get('bootstrap:enable', True) @@ -544,13 +581,28 @@ def store_path(): 'Use "spack bootstrap enable" to enable it') raise RuntimeError(msg) - bootstrap_root_path = spack.config.get( + return _store_path() + + +def _root_path(): + """Root of all the bootstrap related folders""" + return spack.config.get( 'bootstrap:root', spack.paths.user_bootstrap_path ) - bootstrap_store_path = spack.util.path.canonicalize_path( + + +def _store_path(): + bootstrap_root_path = _root_path() + return spack.util.path.canonicalize_path( os.path.join(bootstrap_root_path, 'store') ) - return bootstrap_store_path + + +def _config_path(): + bootstrap_root_path = _root_path() + return spack.util.path.canonicalize_path( + os.path.join(bootstrap_root_path, 'config') + ) def clingo_root_spec(): diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index ebcf7cc862..c4012f5032 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -7,6 +7,7 @@ import argparse import os import shutil +import llnl.util.filesystem import llnl.util.tty as tty import spack.bootstrap @@ -14,9 +15,9 @@ import spack.caches import spack.cmd.common.arguments as arguments import spack.cmd.test import spack.config -import spack.main import spack.repo import spack.stage +import spack.util.path from spack.paths import lib_path, var_path description = "remove temporary build files and/or downloaded archives" @@ -27,7 +28,7 @@ level = "long" class AllClean(argparse.Action): """Activates flags -s -d -f -m and -p simultaneously""" def __call__(self, parser, namespace, values, option_string=None): - parser.parse_args(['-sdfmpb'], namespace=namespace) + parser.parse_args(['-sdfmp'], namespace=namespace) def setup_parser(subparser): @@ -48,9 +49,11 @@ def setup_parser(subparser): help="remove .pyc, .pyo files and __pycache__ folders") subparser.add_argument( '-b', '--bootstrap', action='store_true', - help="remove software needed to bootstrap Spack") + help="remove software and configuration needed to bootstrap Spack") subparser.add_argument( - '-a', '--all', action=AllClean, help="equivalent to -sdfmpb", nargs=0 + '-a', '--all', action=AllClean, + help="equivalent to -sdfmp (does not include --bootstrap)", + nargs=0 ) arguments.add_common_arguments(subparser, ['specs']) @@ -102,8 +105,9 @@ def clean(parser, args): shutil.rmtree(dname) if args.bootstrap: - msg = 'Removing software in "{0}"' - tty.msg(msg.format(spack.bootstrap.store_path())) - with spack.bootstrap.ensure_bootstrap_configuration(): - uninstall = spack.main.SpackCommand('uninstall') - uninstall('-a', '-y') + bootstrap_prefix = spack.util.path.canonicalize_path( + spack.config.get('bootstrap:root') + ) + msg = 'Removing bootstrapped software and configuration in "{0}"' + tty.msg(msg.format(bootstrap_prefix)) + llnl.util.filesystem.remove_directory_contents(bootstrap_prefix) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index fa519afec6..8d028ce0cc 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -84,11 +84,16 @@ all_schemas = copy.deepcopy(section_schemas) all_schemas.update(dict((key, spack.schema.env.schema) for key in spack.schema.env.keys)) +#: Path to the default configuration +configuration_defaults_path = ( + 'defaults', os.path.join(spack.paths.etc_path, 'spack', 'defaults') +) + #: Builtin paths to configuration files in Spack configuration_paths = ( # Default configuration scope is the lowest-level scope. These are # versioned with Spack and can be overridden by systems, sites or users - ('defaults', os.path.join(spack.paths.etc_path, 'spack', 'defaults')), + configuration_defaults_path, # System configuration is per machine. # No system-level configs should be checked into spack by default diff --git a/lib/spack/spack/test/bootstrap.py b/lib/spack/spack/test/bootstrap.py index 3aed7231f3..fb84921c7a 100644 --- a/lib/spack/spack/test/bootstrap.py +++ b/lib/spack/spack/test/bootstrap.py @@ -99,3 +99,22 @@ def test_bootstrap_search_for_compilers_with_environment_active( with spack.bootstrap.ensure_bootstrap_configuration(): assert spack.compilers.all_compiler_specs(init_config=False) assert not spack.compilers.all_compiler_specs(init_config=False) + + +@pytest.mark.regression('26189') +def test_config_yaml_is_preserved_during_bootstrap(mutable_config): + # Mock the command line scope + expected_dir = '/tmp/test' + internal_scope = spack.config.InternalConfigScope( + name='command_line', data={ + 'config': { + 'test_stage': expected_dir + } + } + ) + spack.config.config.push_scope(internal_scope) + + assert spack.config.get('config:test_stage') == expected_dir + with spack.bootstrap.ensure_bootstrap_configuration(): + assert spack.config.get('config:test_stage') == expected_dir + assert spack.config.get('config:test_stage') == expected_dir |