From 59c7ff8683ab411fc53d47b10cfe5b5c4a326a67 Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Fri, 15 Mar 2024 03:01:49 -0700 Subject: Allow compilers to be configured in packages.yaml (#42016) Co-authored-by: becker33 --- lib/spack/docs/getting_started.rst | 49 ++++++++++- lib/spack/spack/bootstrap/config.py | 2 +- lib/spack/spack/cmd/compiler.py | 2 +- lib/spack/spack/compiler.py | 34 ++++++++ lib/spack/spack/compilers/__init__.py | 125 ++++++++++++++++++++++++++--- lib/spack/spack/cray_manifest.py | 2 +- lib/spack/spack/environment/environment.py | 2 +- lib/spack/spack/test/cmd/compiler.py | 74 +++++++++++++++++ 8 files changed, 270 insertions(+), 20 deletions(-) diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index d7f913d646..ab9c274e01 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -250,9 +250,10 @@ Compiler configuration Spack has the ability to build packages with multiple compilers and compiler versions. Compilers can be made available to Spack by -specifying them manually in ``compilers.yaml``, or automatically by -running ``spack compiler find``, but for convenience Spack will -automatically detect compilers the first time it needs them. +specifying them manually in ``compilers.yaml`` or ``packages.yaml``, +or automatically by running ``spack compiler find``, but for +convenience Spack will automatically detect compilers the first time +it needs them. .. _cmd-spack-compilers: @@ -457,6 +458,48 @@ specification. The operations available to modify the environment are ``set``, ` prepend_path: # Similar for append|remove_path LD_LIBRARY_PATH: /ld/paths/added/by/setvars/sh +.. note:: + + Spack is in the process of moving compilers from a separate + attribute to be handled like all other packages. As part of this + process, the ``compilers.yaml`` section will eventually be replaced + by configuration in the ``packages.yaml`` section. This new + configuration is now available, although it is not yet the default + behavior. + +Compilers can also be configured as external packages in the +``packages.yaml`` config file. Any external package for a compiler +(e.g. ``gcc`` or ``llvm``) will be treated as a configured compiler +assuming the paths to the compiler executables are determinable from +the prefix. + +If the paths to the compiler executable are not determinable from the +prefix, you can add them to the ``extra_attributes`` field. Similarly, +all other fields from the compilers config can be added to the +``extra_attributes`` field for an external representing a compiler. + +.. code-block:: yaml + + packages: + gcc: + external: + - spec: gcc@12.2.0 arch=linux-rhel8-skylake + prefix: /usr + extra_attributes: + environment: + set: + GCC_ROOT: /usr + external: + - spec: llvm+clang@15.0.0 arch=linux-rhel8-skylake + prefix: /usr + extra_attributes: + paths: + cc: /usr/bin/clang-with-suffix + cxx: /usr/bin/clang++-with-extra-info + fc: /usr/bin/gfortran + f77: /usr/bin/gfortran + extra_rpaths: + - /usr/lib/llvm/ ^^^^^^^^^^^^^^^^^^^^^^^ Build Your Own Compiler diff --git a/lib/spack/spack/bootstrap/config.py b/lib/spack/spack/bootstrap/config.py index 10c5a3db4b..8cba750fc5 100644 --- a/lib/spack/spack/bootstrap/config.py +++ b/lib/spack/spack/bootstrap/config.py @@ -147,7 +147,7 @@ def _add_compilers_if_missing() -> None: mixed_toolchain=sys.platform == "darwin" ) if new_compilers: - spack.compilers.add_compilers_to_config(new_compilers, init_config=False) + spack.compilers.add_compilers_to_config(new_compilers) @contextlib.contextmanager diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 006c6a79a7..860f0a9ee0 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -89,7 +89,7 @@ def compiler_find(args): paths, scope=None, mixed_toolchain=args.mixed_toolchain ) if new_compilers: - spack.compilers.add_compilers_to_config(new_compilers, scope=args.scope, init_config=False) + spack.compilers.add_compilers_to_config(new_compilers, scope=args.scope) n = len(new_compilers) s = "s" if n > 1 else "" diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index d735845d86..15c11995a7 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -334,6 +334,40 @@ class Compiler: # used for version checks for API, e.g. C++11 flag self._real_version = None + def __eq__(self, other): + return ( + self.cc == other.cc + and self.cxx == other.cxx + and self.fc == other.fc + and self.f77 == other.f77 + and self.spec == other.spec + and self.operating_system == other.operating_system + and self.target == other.target + and self.flags == other.flags + and self.modules == other.modules + and self.environment == other.environment + and self.extra_rpaths == other.extra_rpaths + and self.enable_implicit_rpaths == other.enable_implicit_rpaths + ) + + def __hash__(self): + return hash( + ( + self.cc, + self.cxx, + self.fc, + self.f77, + self.spec, + self.operating_system, + self.target, + str(self.flags), + str(self.modules), + str(self.environment), + str(self.extra_rpaths), + self.enable_implicit_rpaths, + ) + ) + def verify_executables(self): """Raise an error if any of the compiler executables is not valid. diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 1aa0b6a74e..9b73028b12 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -109,7 +109,7 @@ def _to_dict(compiler): return {"compiler": d} -def get_compiler_config(scope=None, init_config=True): +def get_compiler_config(scope=None, init_config=False): """Return the compiler configuration for the specified architecture.""" config = spack.config.get("compilers", scope=scope) or [] @@ -118,6 +118,8 @@ def get_compiler_config(scope=None, init_config=True): merged_config = spack.config.get("compilers") if merged_config: + # Config is empty for this scope + # Do not init config because there is a non-empty scope return config _init_compiler_config(scope=scope) @@ -125,6 +127,95 @@ def get_compiler_config(scope=None, init_config=True): return config +def get_compiler_config_from_packages(scope=None): + """Return the compiler configuration from packages.yaml""" + config = spack.config.get("packages", scope=scope) + if not config: + return [] + + packages = [] + compiler_package_names = supported_compilers() + list(package_name_to_compiler_name.keys()) + for name, entry in config.items(): + if name not in compiler_package_names: + continue + externals_config = entry.get("externals", None) + if not externals_config: + continue + packages.extend(_compiler_config_from_package_config(externals_config)) + + return packages + + +def _compiler_config_from_package_config(config): + compilers = [] + for entry in config: + compiler = _compiler_config_from_external(entry) + if compiler: + compilers.append(compiler) + + return compilers + + +def _compiler_config_from_external(config): + spec = spack.spec.parse_with_version_concrete(config["spec"]) + # use str(spec.versions) to allow `@x.y.z` instead of `@=x.y.z` + compiler_spec = spack.spec.CompilerSpec( + package_name_to_compiler_name.get(spec.name, spec.name), spec.version + ) + + extra_attributes = config.get("extra_attributes", {}) + prefix = config.get("prefix", None) + + compiler_class = class_for_compiler_name(compiler_spec.name) + paths = extra_attributes.get("paths", {}) + compiler_langs = ["cc", "cxx", "fc", "f77"] + for lang in compiler_langs: + if paths.setdefault(lang, None): + continue + + if not prefix: + continue + + # Check for files that satisfy the naming scheme for this compiler + bindir = os.path.join(prefix, "bin") + for f, regex in itertools.product(os.listdir(bindir), compiler_class.search_regexps(lang)): + if regex.match(f): + paths[lang] = os.path.join(bindir, f) + + if all(v is None for v in paths.values()): + return None + + if not spec.architecture: + host_platform = spack.platforms.host() + operating_system = host_platform.operating_system("default_os") + target = host_platform.target("default_target").microarchitecture + else: + target = spec.target + if not target: + host_platform = spack.platforms.host() + target = host_platform.target("default_target").microarchitecture + + operating_system = spec.os + if not operating_system: + host_platform = spack.platforms.host() + operating_system = host_platform.operating_system("default_os") + + compiler_entry = { + "compiler": { + "spec": str(compiler_spec), + "paths": paths, + "flags": extra_attributes.get("flags", {}), + "operating_system": str(operating_system), + "target": str(target.family), + "modules": config.get("modules", []), + "environment": extra_attributes.get("environment", {}), + "extra_rpaths": extra_attributes.get("extra_rpaths", []), + "implicit_rpaths": extra_attributes.get("implicit_rpaths", None), + } + } + return compiler_entry + + def _init_compiler_config(*, scope): """Compiler search used when Spack has no compilers.""" compilers = find_compilers() @@ -142,17 +233,20 @@ def compiler_config_files(): compiler_config = config.get("compilers", scope=name) if compiler_config: config_files.append(config.get_config_filename(name, "compilers")) + compiler_config_from_packages = get_compiler_config_from_packages(scope=name) + if compiler_config_from_packages: + config_files.append(config.get_config_filename(name, "packages")) return config_files -def add_compilers_to_config(compilers, scope=None, init_config=True): +def add_compilers_to_config(compilers, scope=None): """Add compilers to the config for the specified architecture. Arguments: compilers: a list of Compiler objects. scope: configuration scope to modify. """ - compiler_config = get_compiler_config(scope, init_config) + compiler_config = get_compiler_config(scope, init_config=False) for compiler in compilers: if not compiler.cc: tty.debug(f"{compiler.spec} does not have a C compiler") @@ -184,6 +278,9 @@ def remove_compiler_from_config(compiler_spec, scope=None): for current_scope in candidate_scopes: removal_happened |= _remove_compiler_from_scope(compiler_spec, scope=current_scope) + msg = "`spack compiler remove` will not remove compilers defined in packages.yaml" + msg += "\nTo remove these compilers, either edit the config or use `spack external remove`" + tty.debug(msg) return removal_happened @@ -198,7 +295,7 @@ def _remove_compiler_from_scope(compiler_spec, scope): True if one or more compiler entries were actually removed, False otherwise """ assert scope is not None, "a specific scope is needed when calling this function" - compiler_config = get_compiler_config(scope) + compiler_config = get_compiler_config(scope, init_config=False) filtered_compiler_config = [ compiler_entry for compiler_entry in compiler_config @@ -221,7 +318,14 @@ def all_compilers_config(scope=None, init_config=True): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ - return get_compiler_config(scope, init_config) + from_packages_yaml = get_compiler_config_from_packages(scope) + if from_packages_yaml: + init_config = False + from_compilers_yaml = get_compiler_config(scope, init_config) + + result = from_compilers_yaml + from_packages_yaml + key = lambda c: _compiler_from_config_entry(c["compiler"]) + return list(llnl.util.lang.dedupe(result, key=key)) def all_compiler_specs(scope=None, init_config=True): @@ -388,7 +492,7 @@ def find_specs_by_arch(compiler_spec, arch_spec, scope=None, init_config=True): def all_compilers(scope=None, init_config=True): - config = get_compiler_config(scope, init_config=init_config) + config = all_compilers_config(scope, init_config=init_config) compilers = list() for items in config: items = items["compiler"] @@ -403,10 +507,7 @@ def compilers_for_spec( """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ - if use_cache: - config = all_compilers_config(scope, init_config) - else: - config = get_compiler_config(scope, init_config) + config = all_compilers_config(scope, init_config) matches = set(find(compiler_spec, scope, init_config)) compilers = [] @@ -583,9 +684,7 @@ def get_compiler_duplicates(compiler_spec, arch_spec): scope_to_compilers = {} for scope in config.scopes: - compilers = compilers_for_spec( - compiler_spec, arch_spec=arch_spec, scope=scope, use_cache=False - ) + compilers = compilers_for_spec(compiler_spec, arch_spec=arch_spec, scope=scope) if compilers: scope_to_compilers[scope] = compilers diff --git a/lib/spack/spack/cray_manifest.py b/lib/spack/spack/cray_manifest.py index eb26b3e6b9..22371f68f2 100644 --- a/lib/spack/spack/cray_manifest.py +++ b/lib/spack/spack/cray_manifest.py @@ -227,7 +227,7 @@ def read(path, apply_updates): if apply_updates and compilers: for compiler in compilers: try: - spack.compilers.add_compilers_to_config([compiler], init_config=False) + spack.compilers.add_compilers_to_config([compiler]) except Exception: warnings.warn( f"Could not add compiler {str(compiler.spec)}: " diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index 727b46d048..ed59b5cdf1 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -1427,7 +1427,7 @@ class Environment: # Ensure we have compilers in compilers.yaml to avoid that # processes try to write the config file in parallel - _ = spack.compilers.get_compiler_config() + _ = spack.compilers.get_compiler_config(init_config=True) # Early return if there is nothing to do if len(args) == 0: diff --git a/lib/spack/spack/test/cmd/compiler.py b/lib/spack/spack/test/cmd/compiler.py index 3a8c662a5e..849b9e7018 100644 --- a/lib/spack/spack/test/cmd/compiler.py +++ b/lib/spack/spack/test/cmd/compiler.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import shutil +import sys import pytest @@ -247,3 +248,76 @@ def test_compiler_list_empty(no_compilers_yaml, working_env, compilers_dir): out = compiler("list") assert not out assert compiler.returncode == 0 + + +@pytest.mark.parametrize( + "external,expected", + [ + ( + { + "spec": "gcc@=7.7.7 os=foobar target=x86_64", + "prefix": "/path/to/fake", + "modules": ["gcc/7.7.7", "foobar"], + "extra_attributes": { + "paths": { + "cc": "/path/to/fake/gcc", + "cxx": "/path/to/fake/g++", + "fc": "/path/to/fake/gfortran", + "f77": "/path/to/fake/gfortran", + }, + "flags": {"fflags": "-ffree-form"}, + }, + }, + """gcc@7.7.7: +\tpaths: +\t\tcc = /path/to/fake/gcc +\t\tcxx = /path/to/fake/g++ +\t\tf77 = /path/to/fake/gfortran +\t\tfc = /path/to/fake/gfortran +\tflags: +\t\tfflags = ['-ffree-form'] +\tmodules = ['gcc/7.7.7', 'foobar'] +\toperating system = foobar +""", + ), + ( + { + "spec": "gcc@7.7.7", + "prefix": "{prefix}", + "modules": ["gcc/7.7.7", "foobar"], + "extra_attributes": {"flags": {"fflags": "-ffree-form"}}, + }, + """gcc@7.7.7: +\tpaths: +\t\tcc = {compilers_dir}{sep}gcc-8{suffix} +\t\tcxx = {compilers_dir}{sep}g++-8{suffix} +\t\tf77 = {compilers_dir}{sep}gfortran-8{suffix} +\t\tfc = {compilers_dir}{sep}gfortran-8{suffix} +\tflags: +\t\tfflags = ['-ffree-form'] +\tmodules = ['gcc/7.7.7', 'foobar'] +\toperating system = debian6 +""", + ), + ], +) +def test_compilers_shows_packages_yaml( + external, expected, no_compilers_yaml, working_env, compilers_dir +): + """Spack should see a single compiler defined from packages.yaml""" + external["prefix"] = external["prefix"].format(prefix=os.path.dirname(compilers_dir)) + gcc_entry = {"externals": [external]} + + packages = spack.config.get("packages") + packages["gcc"] = gcc_entry + spack.config.set("packages", packages) + + out = compiler("list") + assert out.count("gcc@7.7.7") == 1 + + out = compiler("info", "gcc@7.7.7") + assert out == expected.format( + compilers_dir=str(compilers_dir), + sep=os.sep, + suffix=".bat" if sys.platform == "win32" else "", + ) -- cgit v1.2.3-70-g09d2