From e1a104e3a2ba3e6b7daf71f221ea983c69da99d1 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Thu, 30 Mar 2023 22:12:18 +0200 Subject: Add type-hints to spack.bootstrap (#36491) --- lib/spack/spack/bootstrap/_common.py | 21 +++++--- lib/spack/spack/bootstrap/config.py | 29 ++++++----- lib/spack/spack/bootstrap/core.py | 83 ++++++++++++++++++++------------ lib/spack/spack/bootstrap/environment.py | 35 +++++++------- lib/spack/spack/bootstrap/status.py | 27 +++++++---- 5 files changed, 117 insertions(+), 78 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/bootstrap/_common.py b/lib/spack/spack/bootstrap/_common.py index 3e464d1d86..555c11d809 100644 --- a/lib/spack/spack/bootstrap/_common.py +++ b/lib/spack/spack/bootstrap/_common.py @@ -9,6 +9,7 @@ import re import sys import sysconfig import warnings +from typing import Dict, Optional, Sequence, Union import archspec.cpu @@ -21,8 +22,10 @@ import spack.util.executable from .config import spec_for_current_python +QueryInfo = Dict[str, "spack.spec.Spec"] -def _python_import(module): + +def _python_import(module: str) -> bool: try: __import__(module) except ImportError: @@ -30,7 +33,9 @@ def _python_import(module): return True -def _try_import_from_store(module, query_spec, query_info=None): +def _try_import_from_store( + module: str, query_spec: Union[str, "spack.spec.Spec"], query_info: Optional[QueryInfo] = None +) -> bool: """Return True if the module can be imported from an already installed spec, False otherwise. @@ -52,7 +57,7 @@ def _try_import_from_store(module, query_spec, query_info=None): module_paths = [ os.path.join(candidate_spec.prefix, pkg.purelib), os.path.join(candidate_spec.prefix, pkg.platlib), - ] # type: list[str] + ] path_before = list(sys.path) # NOTE: try module_paths first and last, last allows an existing version in path @@ -89,7 +94,7 @@ def _try_import_from_store(module, query_spec, query_info=None): return False -def _fix_ext_suffix(candidate_spec): +def _fix_ext_suffix(candidate_spec: "spack.spec.Spec"): """Fix the external suffixes of Python extensions on the fly for platforms that may need it @@ -157,7 +162,11 @@ def _fix_ext_suffix(candidate_spec): os.symlink(abs_path, link_name) -def _executables_in_store(executables, query_spec, query_info=None): +def _executables_in_store( + executables: Sequence[str], + query_spec: Union["spack.spec.Spec", str], + query_info: Optional[QueryInfo] = None, +) -> bool: """Return True if at least one of the executables can be retrieved from a spec in store, False otherwise. @@ -193,7 +202,7 @@ def _executables_in_store(executables, query_spec, query_info=None): return False -def _root_spec(spec_str): +def _root_spec(spec_str: str) -> str: """Add a proper compiler and target to a spec used during bootstrapping. Args: diff --git a/lib/spack/spack/bootstrap/config.py b/lib/spack/spack/bootstrap/config.py index 20017e5d26..1753810a93 100644 --- a/lib/spack/spack/bootstrap/config.py +++ b/lib/spack/spack/bootstrap/config.py @@ -7,6 +7,7 @@ import contextlib import os.path import sys +from typing import Any, Dict, Generator, MutableSequence, Sequence from llnl.util import tty @@ -24,12 +25,12 @@ import spack.util.path _REF_COUNT = 0 -def is_bootstrapping(): +def is_bootstrapping() -> bool: """Return True if we are in a bootstrapping context, False otherwise.""" return _REF_COUNT > 0 -def spec_for_current_python(): +def spec_for_current_python() -> str: """For bootstrapping purposes we are just interested in the Python minor version (all patches are ABI compatible with the same minor). @@ -41,14 +42,14 @@ def spec_for_current_python(): return f"python@{version_str}" -def root_path(): +def root_path() -> str: """Root of all the bootstrap related folders""" return spack.util.path.canonicalize_path( spack.config.get("bootstrap:root", spack.paths.default_user_bootstrap_path) ) -def store_path(): +def store_path() -> str: """Path to the store used for bootstrapped software""" enabled = spack.config.get("bootstrap:enable", True) if not enabled: @@ -59,7 +60,7 @@ def store_path(): @contextlib.contextmanager -def spack_python_interpreter(): +def spack_python_interpreter() -> Generator: """Override the current configuration to set the interpreter under which Spack is currently running as the only Python external spec available. @@ -76,18 +77,18 @@ def spack_python_interpreter(): yield -def _store_path(): +def _store_path() -> str: bootstrap_root_path = root_path() return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, "store")) -def _config_path(): +def _config_path() -> str: bootstrap_root_path = root_path() return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, "config")) @contextlib.contextmanager -def ensure_bootstrap_configuration(): +def ensure_bootstrap_configuration() -> Generator: """Swap the current configuration for the one used to bootstrap Spack. The context manager is reference counted to ensure we don't swap multiple @@ -107,7 +108,7 @@ def ensure_bootstrap_configuration(): _REF_COUNT -= 1 -def _read_and_sanitize_configuration(): +def _read_and_sanitize_configuration() -> Dict[str, Any]: """Read the user configuration that needs to be reused for bootstrapping and remove the entries that should not be copied over. """ @@ -120,9 +121,11 @@ def _read_and_sanitize_configuration(): return user_configuration -def _bootstrap_config_scopes(): +def _bootstrap_config_scopes() -> Sequence["spack.config.ConfigScope"]: tty.debug("[BOOTSTRAP CONFIG SCOPE] name=_builtin") - config_scopes = [spack.config.InternalConfigScope("_builtin", spack.config.config_defaults)] + config_scopes: MutableSequence["spack.config.ConfigScope"] = [ + spack.config.InternalConfigScope("_builtin", spack.config.config_defaults) + ] configuration_paths = (spack.config.configuration_defaults_path, ("bootstrap", _config_path())) for name, path in configuration_paths: platform = spack.platforms.host().name @@ -137,7 +140,7 @@ def _bootstrap_config_scopes(): return config_scopes -def _add_compilers_if_missing(): +def _add_compilers_if_missing() -> None: arch = spack.spec.ArchSpec.frontend_arch() if not spack.compilers.compilers_for_arch(arch): new_compilers = spack.compilers.find_new_compilers() @@ -146,7 +149,7 @@ def _add_compilers_if_missing(): @contextlib.contextmanager -def _ensure_bootstrap_configuration(): +def _ensure_bootstrap_configuration() -> Generator: bootstrap_store_path = store_path() user_configuration = _read_and_sanitize_configuration() with spack.environment.no_active_environment(): diff --git a/lib/spack/spack/bootstrap/core.py b/lib/spack/spack/bootstrap/core.py index 3fd1efbc3f..90263075cb 100644 --- a/lib/spack/spack/bootstrap/core.py +++ b/lib/spack/spack/bootstrap/core.py @@ -29,7 +29,7 @@ import os import os.path import sys import uuid -from typing import Callable, List, Optional +from typing import Any, Callable, Dict, List, Optional, Tuple from llnl.util import tty from llnl.util.lang import GroupedExceptionHandler @@ -66,6 +66,9 @@ IS_WINDOWS = sys.platform == "win32" _bootstrap_methods = {} +ConfigDictionary = Dict[str, Any] + + def bootstrapper(bootstrapper_type: str): """Decorator to register classes implementing bootstrapping methods. @@ -86,7 +89,7 @@ class Bootstrapper: config_scope_name = "" - def __init__(self, conf): + def __init__(self, conf: ConfigDictionary) -> None: self.conf = conf self.name = conf["name"] self.metadata_dir = spack.util.path.canonicalize_path(conf["metadata"]) @@ -100,7 +103,7 @@ class Bootstrapper: self.url = url @property - def mirror_scope(self): + def mirror_scope(self) -> spack.config.InternalConfigScope: """Mirror scope to be pushed onto the bootstrapping configuration when using this bootstrapper. """ @@ -121,7 +124,7 @@ class Bootstrapper: """ return False - def try_search_path(self, executables: List[str], abstract_spec_str: str) -> bool: + def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bool: """Try to search some executables in the prefix of specs satisfying the abstract spec passed as argument. @@ -139,13 +142,15 @@ class Bootstrapper: class BuildcacheBootstrapper(Bootstrapper): """Install the software needed during bootstrapping from a buildcache.""" - def __init__(self, conf): + def __init__(self, conf) -> None: super().__init__(conf) - self.last_search = None + self.last_search: Optional[ConfigDictionary] = None self.config_scope_name = f"bootstrap_buildcache-{uuid.uuid4()}" @staticmethod - def _spec_and_platform(abstract_spec_str): + def _spec_and_platform( + abstract_spec_str: str, + ) -> Tuple[spack.spec.Spec, spack.platforms.Platform]: """Return the spec object and platform we need to use when querying the buildcache. @@ -158,7 +163,7 @@ class BuildcacheBootstrapper(Bootstrapper): bincache_platform = spack.platforms.real_host() return abstract_spec, bincache_platform - def _read_metadata(self, package_name): + def _read_metadata(self, package_name: str) -> Any: """Return metadata about the given package.""" json_filename = f"{package_name}.json" json_dir = self.metadata_dir @@ -167,7 +172,13 @@ class BuildcacheBootstrapper(Bootstrapper): data = json.load(stream) return data - def _install_by_hash(self, pkg_hash, pkg_sha256, index, bincache_platform): + def _install_by_hash( + self, + pkg_hash: str, + pkg_sha256: str, + index: List[spack.spec.Spec], + bincache_platform: spack.platforms.Platform, + ) -> None: index_spec = next(x for x in index if x.dag_hash() == pkg_hash) # Reconstruct the compiler that we need to use for bootstrapping compiler_entry = { @@ -192,7 +203,13 @@ class BuildcacheBootstrapper(Bootstrapper): match, allow_root=True, unsigned=True, force=True, sha256=pkg_sha256 ) - def _install_and_test(self, abstract_spec, bincache_platform, bincache_data, test_fn): + def _install_and_test( + self, + abstract_spec: spack.spec.Spec, + bincache_platform: spack.platforms.Platform, + bincache_data, + test_fn, + ) -> bool: # Ensure we see only the buildcache being used to bootstrap with spack.config.override(self.mirror_scope): # This index is currently needed to get the compiler used to build some @@ -217,13 +234,14 @@ class BuildcacheBootstrapper(Bootstrapper): for _, pkg_hash, pkg_sha256 in item["binaries"]: self._install_by_hash(pkg_hash, pkg_sha256, index, bincache_platform) - info = {} + info: ConfigDictionary = {} if test_fn(query_spec=abstract_spec, query_info=info): self.last_search = info return True return False - def try_import(self, module, abstract_spec_str): + def try_import(self, module: str, abstract_spec_str: str) -> bool: + info: ConfigDictionary test_fn, info = functools.partial(_try_import_from_store, module), {} if test_fn(query_spec=abstract_spec_str, query_info=info): return True @@ -235,7 +253,8 @@ class BuildcacheBootstrapper(Bootstrapper): data = self._read_metadata(module) return self._install_and_test(abstract_spec, bincache_platform, data, test_fn) - def try_search_path(self, executables, abstract_spec_str): + def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bool: + info: ConfigDictionary test_fn, info = functools.partial(_executables_in_store, executables), {} if test_fn(query_spec=abstract_spec_str, query_info=info): self.last_search = info @@ -251,13 +270,13 @@ class BuildcacheBootstrapper(Bootstrapper): class SourceBootstrapper(Bootstrapper): """Install the software needed during bootstrapping from sources.""" - def __init__(self, conf): + def __init__(self, conf) -> None: super().__init__(conf) - self.last_search = None + self.last_search: Optional[ConfigDictionary] = None self.config_scope_name = f"bootstrap_source-{uuid.uuid4()}" - def try_import(self, module, abstract_spec_str): - info = {} + def try_import(self, module: str, abstract_spec_str: str) -> bool: + info: ConfigDictionary = {} if _try_import_from_store(module, abstract_spec_str, query_info=info): self.last_search = info return True @@ -293,8 +312,8 @@ class SourceBootstrapper(Bootstrapper): return True return False - def try_search_path(self, executables, abstract_spec_str): - info = {} + def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bool: + info: ConfigDictionary = {} if _executables_in_store(executables, abstract_spec_str, query_info=info): self.last_search = info return True @@ -323,13 +342,13 @@ class SourceBootstrapper(Bootstrapper): return False -def create_bootstrapper(conf): +def create_bootstrapper(conf: ConfigDictionary): """Return a bootstrap object built according to the configuration argument""" btype = conf["type"] return _bootstrap_methods[btype](conf) -def source_is_enabled_or_raise(conf): +def source_is_enabled_or_raise(conf: ConfigDictionary): """Raise ValueError if the source is not enabled for bootstrapping""" trusted, name = spack.config.get("bootstrap:trusted"), conf["name"] if not trusted.get(name, False): @@ -454,7 +473,7 @@ def ensure_executables_in_path_or_raise( raise RuntimeError(msg) -def _add_externals_if_missing(): +def _add_externals_if_missing() -> None: search_list = [ # clingo spack.repo.path.get_pkg_class("cmake"), @@ -468,41 +487,41 @@ def _add_externals_if_missing(): spack.detection.update_configuration(detected_packages, scope="bootstrap") -def clingo_root_spec(): +def clingo_root_spec() -> str: """Return the root spec used to bootstrap clingo""" return _root_spec("clingo-bootstrap@spack+python") -def ensure_clingo_importable_or_raise(): +def ensure_clingo_importable_or_raise() -> None: """Ensure that the clingo module is available for import.""" ensure_module_importable_or_raise(module="clingo", abstract_spec=clingo_root_spec()) -def gnupg_root_spec(): +def gnupg_root_spec() -> str: """Return the root spec used to bootstrap GnuPG""" return _root_spec("gnupg@2.3:") -def ensure_gpg_in_path_or_raise(): +def ensure_gpg_in_path_or_raise() -> None: """Ensure gpg or gpg2 are in the PATH or raise.""" return ensure_executables_in_path_or_raise( executables=["gpg2", "gpg"], abstract_spec=gnupg_root_spec() ) -def patchelf_root_spec(): +def patchelf_root_spec() -> str: """Return the root spec used to bootstrap patchelf""" # 0.13.1 is the last version not to require C++17. return _root_spec("patchelf@0.13.1:") -def verify_patchelf(patchelf): +def verify_patchelf(patchelf: "spack.util.executable.Executable") -> bool: """Older patchelf versions can produce broken binaries, so we verify the version here. Arguments: - patchelf (spack.util.executable.Executable): patchelf executable + patchelf: patchelf executable """ out = patchelf("--version", output=str, error=os.devnull, fail_on_error=False).strip() if patchelf.returncode != 0: @@ -517,7 +536,7 @@ def verify_patchelf(patchelf): return version >= spack.version.Version("0.13.1") -def ensure_patchelf_in_path_or_raise(): +def ensure_patchelf_in_path_or_raise() -> None: """Ensure patchelf is in the PATH or raise.""" # The old concretizer is not smart and we're doing its job: if the latest patchelf # does not concretize because the compiler doesn't support C++17, we try to @@ -534,7 +553,7 @@ def ensure_patchelf_in_path_or_raise(): ) -def ensure_core_dependencies(): +def ensure_core_dependencies() -> None: """Ensure the presence of all the core dependencies.""" if sys.platform.lower() == "linux": ensure_patchelf_in_path_or_raise() @@ -543,7 +562,7 @@ def ensure_core_dependencies(): ensure_clingo_importable_or_raise() -def all_core_root_specs(): +def all_core_root_specs() -> List[str]: """Return a list of all the core root specs that may be used to bootstrap Spack""" return [clingo_root_spec(), gnupg_root_spec(), patchelf_root_spec()] diff --git a/lib/spack/spack/bootstrap/environment.py b/lib/spack/spack/bootstrap/environment.py index cc8d6c09b4..32753e9520 100644 --- a/lib/spack/spack/bootstrap/environment.py +++ b/lib/spack/spack/bootstrap/environment.py @@ -9,6 +9,7 @@ import os import pathlib import sys import warnings +from typing import List import archspec.cpu @@ -27,7 +28,7 @@ class BootstrapEnvironment(spack.environment.Environment): """Environment to install dependencies of Spack for a given interpreter and architecture""" @classmethod - def spack_dev_requirements(cls): + def spack_dev_requirements(cls) -> List[str]: """Spack development requirements""" return [ isort_root_spec(), @@ -38,7 +39,7 @@ class BootstrapEnvironment(spack.environment.Environment): ] @classmethod - def environment_root(cls): + def environment_root(cls) -> pathlib.Path: """Environment root directory""" bootstrap_root_path = root_path() python_part = spec_for_current_python().replace("@", "") @@ -52,12 +53,12 @@ class BootstrapEnvironment(spack.environment.Environment): ) @classmethod - def view_root(cls): + def view_root(cls) -> pathlib.Path: """Location of the view""" return cls.environment_root().joinpath("view") @classmethod - def pythonpaths(cls): + def pythonpaths(cls) -> List[str]: """Paths to be added to sys.path or PYTHONPATH""" python_dir_part = f"python{'.'.join(str(x) for x in sys.version_info[:2])}" glob_expr = str(cls.view_root().joinpath("**", python_dir_part, "**")) @@ -68,21 +69,21 @@ class BootstrapEnvironment(spack.environment.Environment): return result @classmethod - def bin_dirs(cls): + def bin_dirs(cls) -> List[pathlib.Path]: """Paths to be added to PATH""" return [cls.view_root().joinpath("bin")] @classmethod - def spack_yaml(cls): + def spack_yaml(cls) -> pathlib.Path: """Environment spack.yaml file""" return cls.environment_root().joinpath("spack.yaml") - def __init__(self): + def __init__(self) -> None: if not self.spack_yaml().exists(): self._write_spack_yaml_file() super().__init__(self.environment_root()) - def update_installations(self): + def update_installations(self) -> None: """Update the installations of this environment. The update is done using a depfile on Linux and macOS, and using the ``install_all`` @@ -103,7 +104,7 @@ class BootstrapEnvironment(spack.environment.Environment): self._install_with_depfile() self.write(regenerate=True) - def update_syspath_and_environ(self): + def update_syspath_and_environ(self) -> None: """Update ``sys.path`` and the PATH, PYTHONPATH environment variables to point to the environment view. """ @@ -119,7 +120,7 @@ class BootstrapEnvironment(spack.environment.Environment): + [str(x) for x in self.pythonpaths()] ) - def _install_with_depfile(self): + def _install_with_depfile(self) -> None: spackcmd = spack.util.executable.which("spack") spackcmd( "-e", @@ -141,7 +142,7 @@ class BootstrapEnvironment(spack.environment.Environment): **kwargs, ) - def _write_spack_yaml_file(self): + def _write_spack_yaml_file(self) -> None: tty.msg( "[BOOTSTRAPPING] Spack has missing dependencies, creating a bootstrapping environment" ) @@ -159,32 +160,32 @@ class BootstrapEnvironment(spack.environment.Environment): self.spack_yaml().write_text(template.render(context), encoding="utf-8") -def isort_root_spec(): +def isort_root_spec() -> str: """Return the root spec used to bootstrap isort""" return _root_spec("py-isort@4.3.5:") -def mypy_root_spec(): +def mypy_root_spec() -> str: """Return the root spec used to bootstrap mypy""" return _root_spec("py-mypy@0.900:") -def black_root_spec(): +def black_root_spec() -> str: """Return the root spec used to bootstrap black""" return _root_spec("py-black@:23.1.0") -def flake8_root_spec(): +def flake8_root_spec() -> str: """Return the root spec used to bootstrap flake8""" return _root_spec("py-flake8") -def pytest_root_spec(): +def pytest_root_spec() -> str: """Return the root spec used to bootstrap flake8""" return _root_spec("py-pytest") -def ensure_environment_dependencies(): +def ensure_environment_dependencies() -> None: """Ensure Spack dependencies from the bootstrap environment are installed and ready to use""" with BootstrapEnvironment() as env: env.update_installations() diff --git a/lib/spack/spack/bootstrap/status.py b/lib/spack/spack/bootstrap/status.py index 40137b57ca..39caa9eac2 100644 --- a/lib/spack/spack/bootstrap/status.py +++ b/lib/spack/spack/bootstrap/status.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Query the status of bootstrapping on this machine""" import platform +from typing import List, Optional, Sequence, Tuple, Union import spack.util.executable @@ -19,8 +20,12 @@ from .environment import ( pytest_root_spec, ) +ExecutablesType = Union[str, Sequence[str]] +RequiredResponseType = Tuple[bool, Optional[str]] +SpecLike = Union["spack.spec.Spec", str] -def _required_system_executable(exes, msg): + +def _required_system_executable(exes: ExecutablesType, msg: str) -> RequiredResponseType: """Search for an executable is the system path only.""" if isinstance(exes, str): exes = (exes,) @@ -29,7 +34,9 @@ def _required_system_executable(exes, msg): return False, msg -def _required_executable(exes, query_spec, msg): +def _required_executable( + exes: ExecutablesType, query_spec: SpecLike, msg: str +) -> RequiredResponseType: """Search for an executable in the system path or in the bootstrap store.""" if isinstance(exes, str): exes = (exes,) @@ -38,7 +45,7 @@ def _required_executable(exes, query_spec, msg): return False, msg -def _required_python_module(module, query_spec, msg): +def _required_python_module(module: str, query_spec: SpecLike, msg: str) -> RequiredResponseType: """Check if a Python module is available in the current interpreter or if it can be loaded from the bootstrap store """ @@ -47,7 +54,7 @@ def _required_python_module(module, query_spec, msg): return False, msg -def _missing(name, purpose, system_only=True): +def _missing(name: str, purpose: str, system_only: bool = True) -> str: """Message to be printed if an executable is not found""" msg = '[{2}] MISSING "{0}": {1}' if not system_only: @@ -55,7 +62,7 @@ def _missing(name, purpose, system_only=True): return msg.format(name, purpose, "@*y{{-}}") -def _core_requirements(): +def _core_requirements() -> List[RequiredResponseType]: _core_system_exes = { "make": _missing("make", "required to build software from sources"), "patch": _missing("patch", "required to patch source code before building"), @@ -80,7 +87,7 @@ def _core_requirements(): return result -def _buildcache_requirements(): +def _buildcache_requirements() -> List[RequiredResponseType]: _buildcache_exes = { "file": _missing("file", "required to analyze files for buildcaches"), ("gpg2", "gpg"): _missing("gpg2", "required to sign/verify buildcaches", False), @@ -103,7 +110,7 @@ def _buildcache_requirements(): return result -def _optional_requirements(): +def _optional_requirements() -> List[RequiredResponseType]: _optional_exes = { "zstd": _missing("zstd", "required to compress/decompress code archives"), "svn": _missing("svn", "required to manage subversion repositories"), @@ -114,7 +121,7 @@ def _optional_requirements(): return result -def _development_requirements(): +def _development_requirements() -> List[RequiredResponseType]: # Ensure we trigger environment modifications if we have an environment if BootstrapEnvironment.spack_yaml().exists(): with BootstrapEnvironment() as env: @@ -139,7 +146,7 @@ def _development_requirements(): ] -def status_message(section): +def status_message(section) -> Tuple[str, bool]: """Return a status message to be printed to screen that refers to the section passed as argument and a bool which is True if there are missing dependencies. @@ -161,7 +168,7 @@ def status_message(section): with ensure_bootstrap_configuration(): missing_software = False for found, err_msg in required_software(): - if not found: + if not found and err_msg: missing_software = True msg += "\n " + err_msg msg += "\n" -- cgit v1.2.3-60-g2f50