From 337b54fab045c3fa189570e5b5cf222b4d305442 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 5 Oct 2021 09:16:09 +0200 Subject: Isolate bootstrap configuration from user configuration (#26071) * Isolate bootstrap configuration from user configuration * Search for build dependencies automatically if bootstrapping from sources The bootstrapping logic will search for build dependencies automatically if bootstrapping anything form sources. Any external spec, if found, is written in a scope that is specific to bootstrapping. * Don't clean the bootstrap store with "spack clean -a" * Copy bootstrap.yaml and config.yaml in the bootstrap area --- lib/spack/docs/getting_started.rst | 11 ------- lib/spack/spack/bootstrap.py | 64 ++++++++++++++++++++++++++++++++++---- lib/spack/spack/cmd/clean.py | 22 +++++++------ lib/spack/spack/config.py | 7 ++++- lib/spack/spack/test/bootstrap.py | 19 +++++++++++ 5 files changed, 96 insertions(+), 27 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index ba5b93a567..aa56890847 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -197,17 +197,6 @@ Spack will build the required software on the first request to concretize a spec [ ... ] zlib@1.2.11%gcc@10.1.0+optimize+pic+shared arch=linux-ubuntu18.04-broadwell -.. tip:: - - If you want to speed-up bootstrapping ``clingo`` from sources, you may try to - search for ``cmake`` and ``bison`` on your system: - - .. code-block:: console - - $ spack external find cmake bison - ==> The following specs have been detected on this system and added to /home/spack/.spack/packages.yaml - bison@3.0.4 cmake@3.19.4 - """"""""""""""""""" The Bootstrap Store """"""""""""""""""" diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py index ca92b6fad0..8f4119a04d 100644 --- a/lib/spack/spack/bootstrap.py +++ b/lib/spack/spack/bootstrap.py @@ -26,6 +26,7 @@ import llnl.util.tty as tty import spack.architecture import spack.binary_distribution import spack.config +import spack.detection import spack.environment import spack.main import spack.modules @@ -209,7 +210,7 @@ class _BuildcacheBootstrapper(object): buildcache = spack.main.SpackCommand('buildcache') # Ensure we see only the buildcache being used to bootstrap mirror_scope = spack.config.InternalConfigScope( - 'bootstrap', {'mirrors:': {self.name: self.url}} + 'bootstrap_buildcache', {'mirrors:': {self.name: self.url}} ) with spack.config.override(mirror_scope): # This index is currently needed to get the compiler used to build some @@ -218,7 +219,7 @@ class _BuildcacheBootstrapper(object): index = spack.binary_distribution.update_cache_and_get_specs() if not index: - raise RuntimeError("Could not populate the binary index") + raise RuntimeError("could not populate the binary index") for item in data['verified']: candidate_spec = item['spec'] @@ -279,6 +280,10 @@ class _SourceBootstrapper(object): tty.info("Bootstrapping {0} from sources".format(module)) + # If we compile code from sources detecting a few build tools + # might reduce compilation time by a fair amount + _add_externals_if_missing() + # Try to build and install from sources with spack_python_interpreter(): # Add hint to use frontend operating system on Cray @@ -492,7 +497,11 @@ def _bootstrap_config_scopes(): config_scopes = [ spack.config.InternalConfigScope('_builtin', spack.config.config_defaults) ] - for name, path in spack.config.configuration_paths: + configuration_paths = ( + spack.config.configuration_defaults_path, + ('bootstrap', _config_path()) + ) + for name, path in configuration_paths: platform = spack.architecture.platform().name platform_scope = spack.config.ConfigScope( '/'.join([name, platform]), os.path.join(path, platform) @@ -517,9 +526,19 @@ def _add_compilers_if_missing(): spack.compilers.add_compilers_to_config(new_compilers, init_config=False) +def _add_externals_if_missing(): + search_list = [ + spack.repo.path.get('cmake'), + spack.repo.path.get('bison') + ] + detected_packages = spack.detection.by_executable(search_list) + spack.detection.update_configuration(detected_packages, scope='bootstrap') + + @contextlib.contextmanager def ensure_bootstrap_configuration(): bootstrap_store_path = store_path() + user_configuration = _read_and_sanitize_configuration() with spack.environment.deactivate_environment(): with spack.architecture.use_platform(spack.architecture.real_platform()): with spack.repo.use_repositories(spack.paths.packages_path): @@ -531,11 +550,29 @@ def ensure_bootstrap_configuration(): # We may need to compile code from sources, so ensure we have # compilers for the current platform before switching parts. _add_compilers_if_missing() + spack.config.set('bootstrap', user_configuration['bootstrap']) + spack.config.set('config', user_configuration['config']) with spack.modules.disable_modules(): with spack_python_interpreter(): yield +def _read_and_sanitize_configuration(): + """Read the user configuration that needs to be reused for bootstrapping + and remove the entries that should not be copied over. + """ + # Read the "config" section but pop the install tree (the entry will not be + # considered due to the use_store context manager, so it will be confusing + # to have it in the configuration). + config_yaml = spack.config.get('config') + config_yaml.pop('install_tree', None) + user_configuration = { + 'bootstrap': spack.config.get('bootstrap'), + 'config': config_yaml + } + return user_configuration + + def store_path(): """Path to the store used for bootstrapped software""" enabled = spack.config.get('bootstrap:enable', True) @@ -544,13 +581,28 @@ def store_path(): 'Use "spack bootstrap enable" to enable it') raise RuntimeError(msg) - bootstrap_root_path = spack.config.get( + return _store_path() + + +def _root_path(): + """Root of all the bootstrap related folders""" + return spack.config.get( 'bootstrap:root', spack.paths.user_bootstrap_path ) - bootstrap_store_path = spack.util.path.canonicalize_path( + + +def _store_path(): + bootstrap_root_path = _root_path() + return spack.util.path.canonicalize_path( os.path.join(bootstrap_root_path, 'store') ) - return bootstrap_store_path + + +def _config_path(): + bootstrap_root_path = _root_path() + return spack.util.path.canonicalize_path( + os.path.join(bootstrap_root_path, 'config') + ) def clingo_root_spec(): diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index ebcf7cc862..c4012f5032 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -7,6 +7,7 @@ import argparse import os import shutil +import llnl.util.filesystem import llnl.util.tty as tty import spack.bootstrap @@ -14,9 +15,9 @@ import spack.caches import spack.cmd.common.arguments as arguments import spack.cmd.test import spack.config -import spack.main import spack.repo import spack.stage +import spack.util.path from spack.paths import lib_path, var_path description = "remove temporary build files and/or downloaded archives" @@ -27,7 +28,7 @@ level = "long" class AllClean(argparse.Action): """Activates flags -s -d -f -m and -p simultaneously""" def __call__(self, parser, namespace, values, option_string=None): - parser.parse_args(['-sdfmpb'], namespace=namespace) + parser.parse_args(['-sdfmp'], namespace=namespace) def setup_parser(subparser): @@ -48,9 +49,11 @@ def setup_parser(subparser): help="remove .pyc, .pyo files and __pycache__ folders") subparser.add_argument( '-b', '--bootstrap', action='store_true', - help="remove software needed to bootstrap Spack") + help="remove software and configuration needed to bootstrap Spack") subparser.add_argument( - '-a', '--all', action=AllClean, help="equivalent to -sdfmpb", nargs=0 + '-a', '--all', action=AllClean, + help="equivalent to -sdfmp (does not include --bootstrap)", + nargs=0 ) arguments.add_common_arguments(subparser, ['specs']) @@ -102,8 +105,9 @@ def clean(parser, args): shutil.rmtree(dname) if args.bootstrap: - msg = 'Removing software in "{0}"' - tty.msg(msg.format(spack.bootstrap.store_path())) - with spack.bootstrap.ensure_bootstrap_configuration(): - uninstall = spack.main.SpackCommand('uninstall') - uninstall('-a', '-y') + bootstrap_prefix = spack.util.path.canonicalize_path( + spack.config.get('bootstrap:root') + ) + msg = 'Removing bootstrapped software and configuration in "{0}"' + tty.msg(msg.format(bootstrap_prefix)) + llnl.util.filesystem.remove_directory_contents(bootstrap_prefix) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index fa519afec6..8d028ce0cc 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -84,11 +84,16 @@ all_schemas = copy.deepcopy(section_schemas) all_schemas.update(dict((key, spack.schema.env.schema) for key in spack.schema.env.keys)) +#: Path to the default configuration +configuration_defaults_path = ( + 'defaults', os.path.join(spack.paths.etc_path, 'spack', 'defaults') +) + #: Builtin paths to configuration files in Spack configuration_paths = ( # Default configuration scope is the lowest-level scope. These are # versioned with Spack and can be overridden by systems, sites or users - ('defaults', os.path.join(spack.paths.etc_path, 'spack', 'defaults')), + configuration_defaults_path, # System configuration is per machine. # No system-level configs should be checked into spack by default diff --git a/lib/spack/spack/test/bootstrap.py b/lib/spack/spack/test/bootstrap.py index 3aed7231f3..fb84921c7a 100644 --- a/lib/spack/spack/test/bootstrap.py +++ b/lib/spack/spack/test/bootstrap.py @@ -99,3 +99,22 @@ def test_bootstrap_search_for_compilers_with_environment_active( with spack.bootstrap.ensure_bootstrap_configuration(): assert spack.compilers.all_compiler_specs(init_config=False) assert not spack.compilers.all_compiler_specs(init_config=False) + + +@pytest.mark.regression('26189') +def test_config_yaml_is_preserved_during_bootstrap(mutable_config): + # Mock the command line scope + expected_dir = '/tmp/test' + internal_scope = spack.config.InternalConfigScope( + name='command_line', data={ + 'config': { + 'test_stage': expected_dir + } + } + ) + spack.config.config.push_scope(internal_scope) + + assert spack.config.get('config:test_stage') == expected_dir + with spack.bootstrap.ensure_bootstrap_configuration(): + assert spack.config.get('config:test_stage') == expected_dir + assert spack.config.get('config:test_stage') == expected_dir -- cgit v1.2.3-60-g2f50