From 4381cb5957f71e36ce5175413084c3c3fa56b7dc Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Thu, 23 Dec 2021 19:34:04 +0100 Subject: New subcommand: spack bootstrap status (#28004) This command pokes the environment, Python interpreter and bootstrap store to check if dependencies needed by Spack are available. If any are missing, it shows a comprehensible message. --- lib/spack/spack/bootstrap.py | 140 ++++++++++++++++++++++++++++++++++++++ lib/spack/spack/cmd/bootstrap.py | 43 ++++++++++++ lib/spack/spack/test/bootstrap.py | 17 +++++ share/spack/qa/run-unit-tests | 1 + share/spack/spack-completion.bash | 6 +- 5 files changed, 206 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py index ee9ec118e4..3cb649789d 100644 --- a/lib/spack/spack/bootstrap.py +++ b/lib/spack/spack/bootstrap.py @@ -10,6 +10,7 @@ import functools import json import os import os.path +import platform import re import sys import sysconfig @@ -837,3 +838,142 @@ def ensure_flake8_in_path_or_raise(): """Ensure that flake8 is in the PATH or raise.""" executable, root_spec = 'flake8', flake8_root_spec() return ensure_executables_in_path_or_raise([executable], abstract_spec=root_spec) + + +def _missing(name, purpose, system_only=True): + """Message to be printed if an executable is not found""" + msg = '[{2}] MISSING "{0}": {1}' + if not system_only: + return msg.format(name, purpose, '@*y{{B}}') + return msg.format(name, purpose, '@*y{{-}}') + + +def _required_system_executable(exes, msg): + """Search for an executable is the system path only.""" + if isinstance(exes, six.string_types): + exes = (exes,) + if spack.util.executable.which_string(*exes): + return True, None + return False, msg + + +def _required_python_module(module, query_spec, msg): + """Check if a Python module is available in the current interpreter or + if it can be loaded from the bootstrap store + """ + if _python_import(module) or _try_import_from_store(module, query_spec): + return True, None + return False, msg + + +def _required_executable(exes, query_spec, msg): + """Search for an executable in the system path or in the bootstrap store.""" + if isinstance(exes, six.string_types): + exes = (exes,) + if (spack.util.executable.which_string(*exes) or + _executables_in_store(exes, query_spec)): + return True, None + return False, msg + + +def _core_requirements(): + _core_system_exes = { + 'make': _missing('make', 'required to build software from sources'), + 'patch': _missing('patch', 'required to patch source code before building'), + 'bash': _missing('bash', 'required for Spack compiler wrapper'), + 'tar': _missing('tar', 'required to manage code archives'), + 'gzip': _missing('gzip', 'required to compress/decompress code archives'), + 'unzip': _missing('unzip', 'required to compress/decompress code archives'), + 'bzip2': _missing('bzip2', 'required to compress/decompress code archives'), + 'git': _missing('git', 'required to fetch/manage git repositories') + } + if platform.system().lower() == 'linux': + _core_system_exes['xz'] = _missing( + 'xz', 'required to compress/decompress code archives' + ) + + # Executables that are not bootstrapped yet + result = [_required_system_executable(exe, msg) + for exe, msg in _core_system_exes.items()] + # Python modules + result.append(_required_python_module( + 'clingo', clingo_root_spec(), + _missing('clingo', 'required to concretize specs', False) + )) + return result + + +def _buildcache_requirements(): + _buildcache_exes = { + 'file': _missing('file', 'required to analyze files for buildcaches'), + ('gpg2', 'gpg'): _missing('gpg2', 'required to sign/verify buildcaches', False) + } + if platform.system().lower() == 'darwin': + _buildcache_exes['otool'] = _missing('otool', 'required to relocate binaries') + + # Executables that are not bootstrapped yet + result = [_required_system_executable(exe, msg) + for exe, msg in _buildcache_exes.items()] + + if platform.system().lower() == 'linux': + result.append(_required_executable( + 'patchelf', patchelf_root_spec(), + _missing('patchelf', 'required to relocate binaries', False) + )) + + return result + + +def _optional_requirements(): + _optional_exes = { + 'zstd': _missing('zstd', 'required to compress/decompress code archives'), + 'svn': _missing('svn', 'required to manage subversion repositories'), + 'hg': _missing('hg', 'required to manage mercurial repositories') + } + # Executables that are not bootstrapped yet + result = [_required_system_executable(exe, msg) + for exe, msg in _optional_exes.items()] + return result + + +def _development_requirements(): + return [ + _required_executable('isort', isort_root_spec(), + _missing('isort', 'required for style checks', False)), + _required_executable('mypy', mypy_root_spec(), + _missing('mypy', 'required for style checks', False)), + _required_executable('flake8', flake8_root_spec(), + _missing('flake8', 'required for style checks', False)), + _required_executable('black', black_root_spec(), + _missing('black', 'required for code formatting', False)) + ] + + +def status_message(section): + """Return a status message to be printed to screen that refers to the + section passed as argument and a bool which is True if there are missing + dependencies. + + Args: + section (str): either 'core' or 'buildcache' or 'optional' or 'develop' + """ + pass_token, fail_token = '@*g{[PASS]}', '@*r{[FAIL]}' + + # Contain the header of the section and a list of requirements + spack_sections = { + 'core': ("{0} @*{{Core Functionalities}}", _core_requirements), + 'buildcache': ("{0} @*{{Binary packages}}", _buildcache_requirements), + 'optional': ("{0} @*{{Optional Features}}", _optional_requirements), + 'develop': ("{0} @*{{Development Dependencies}}", _development_requirements) + } + msg, required_software = spack_sections[section] + + with ensure_bootstrap_configuration(): + missing_software = False + for found, err_msg in required_software(): + if not found: + missing_software = True + msg += "\n " + err_msg + msg += '\n' + msg = msg.format(pass_token if not missing_software else fail_token) + return msg, missing_software diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index ae3e1b7639..7446650403 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -10,6 +10,8 @@ import shutil import llnl.util.tty import llnl.util.tty.color +import spack +import spack.bootstrap import spack.cmd.common.arguments import spack.config import spack.main @@ -32,6 +34,16 @@ def _add_scope_option(parser): def setup_parser(subparser): sp = subparser.add_subparsers(dest='subcommand') + status = sp.add_parser('status', help='get the status of Spack') + status.add_argument( + '--optional', action='store_true', default=False, + help='show the status of rarely used optional dependencies' + ) + status.add_argument( + '--dev', action='store_true', default=False, + help='show the status of dependencies needed to develop Spack' + ) + enable = sp.add_parser('enable', help='enable bootstrapping') _add_scope_option(enable) @@ -207,8 +219,39 @@ def _untrust(args): llnl.util.tty.msg(msg.format(args.name)) +def _status(args): + sections = ['core', 'buildcache'] + if args.optional: + sections.append('optional') + if args.dev: + sections.append('develop') + + header = "@*b{{Spack v{0} - {1}}}".format( + spack.spack_version, spack.bootstrap.spec_for_current_python() + ) + print(llnl.util.tty.color.colorize(header)) + print() + # Use the context manager here to avoid swapping between user and + # bootstrap config many times + missing = False + with spack.bootstrap.ensure_bootstrap_configuration(): + for current_section in sections: + status_msg, fail = spack.bootstrap.status_message(section=current_section) + missing = missing or fail + if status_msg: + print(llnl.util.tty.color.colorize(status_msg)) + print() + legend = ('Spack will take care of bootstrapping any missing dependency marked' + ' as [@*y{B}]. Dependencies marked as [@*y{-}] are instead required' + ' to be found on the system.') + if missing: + print(llnl.util.tty.color.colorize(legend)) + print() + + def bootstrap(parser, args): callbacks = { + 'status': _status, 'enable': _enable_or_disable, 'disable': _enable_or_disable, 'reset': _reset, diff --git a/lib/spack/spack/test/bootstrap.py b/lib/spack/spack/test/bootstrap.py index fdfd1f9610..9ae4c85c6a 100644 --- a/lib/spack/spack/test/bootstrap.py +++ b/lib/spack/spack/test/bootstrap.py @@ -150,3 +150,20 @@ def test_nested_use_of_context_manager(mutable_config): with spack.bootstrap.ensure_bootstrap_configuration(): assert spack.config.config != user_config assert spack.config.config == user_config + + +@pytest.mark.parametrize('expected_missing', [False, True]) +def test_status_function_find_files( + mutable_config, mock_executable, tmpdir, monkeypatch, expected_missing +): + if not expected_missing: + mock_executable('foo', 'echo Hello WWorld!') + + monkeypatch.setattr( + spack.bootstrap, '_optional_requirements', + lambda: [spack.bootstrap._required_system_executable('foo', 'NOT FOUND')] + ) + monkeypatch.setenv('PATH', str(tmpdir.join('bin'))) + + _, missing = spack.bootstrap.status_message('optional') + assert missing is expected_missing diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests index b71103ea31..a0c6e402c6 100755 --- a/share/spack/qa/run-unit-tests +++ b/share/spack/qa/run-unit-tests @@ -38,6 +38,7 @@ bin/spack help -a # Profile and print top 20 lines for a simple call to spack spec spack -p --lines 20 spec mpileaks%gcc ^dyninst@10.0.0 ^elfutils@0.170 +$coverage_run $(which spack) bootstrap status --dev --optional #----------------------------------------------------------- # Run unit tests with code coverage diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index acd8867bd9..753767a3de 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -434,10 +434,14 @@ _spack_bootstrap() { then SPACK_COMPREPLY="-h --help" else - SPACK_COMPREPLY="enable disable reset root list trust untrust" + SPACK_COMPREPLY="status enable disable reset root list trust untrust" fi } +_spack_bootstrap_status() { + SPACK_COMPREPLY="-h --help --optional --dev" +} + _spack_bootstrap_enable() { SPACK_COMPREPLY="-h --help --scope" } -- cgit v1.2.3-60-g2f50