From 59c7ff8683ab411fc53d47b10cfe5b5c4a326a67 Mon Sep 17 00:00:00 2001
From: Greg Becker <becker33@llnl.gov>
Date: Fri, 15 Mar 2024 03:01:49 -0700
Subject: Allow compilers to be configured in packages.yaml (#42016)
Co-authored-by: becker33 <becker33@users.noreply.github.com>
---
lib/spack/docs/getting_started.rst | 49 ++++++++++-
lib/spack/spack/bootstrap/config.py | 2 +-
lib/spack/spack/cmd/compiler.py | 2 +-
lib/spack/spack/compiler.py | 34 ++++++++
lib/spack/spack/compilers/__init__.py | 125 ++++++++++++++++++++++++++---
lib/spack/spack/cray_manifest.py | 2 +-
lib/spack/spack/environment/environment.py | 2 +-
lib/spack/spack/test/cmd/compiler.py | 74 +++++++++++++++++
8 files changed, 270 insertions(+), 20 deletions(-)
diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst
index d7f913d646..ab9c274e01 100644
--- a/lib/spack/docs/getting_started.rst
+++ b/lib/spack/docs/getting_started.rst
@@ -250,9 +250,10 @@ Compiler configuration
Spack has the ability to build packages with multiple compilers and
compiler versions. Compilers can be made available to Spack by
-specifying them manually in ``compilers.yaml``, or automatically by
-running ``spack compiler find``, but for convenience Spack will
-automatically detect compilers the first time it needs them.
+specifying them manually in ``compilers.yaml`` or ``packages.yaml``,
+or automatically by running ``spack compiler find``, but for
+convenience Spack will automatically detect compilers the first time
+it needs them.
.. _cmd-spack-compilers:
@@ -457,6 +458,48 @@ specification. The operations available to modify the environment are ``set``, `
prepend_path: # Similar for append|remove_path
LD_LIBRARY_PATH: /ld/paths/added/by/setvars/sh
+.. note::
+
+ Spack is in the process of moving compilers from a separate
+ attribute to be handled like all other packages. As part of this
+ process, the ``compilers.yaml`` section will eventually be replaced
+ by configuration in the ``packages.yaml`` section. This new
+ configuration is now available, although it is not yet the default
+ behavior.
+
+Compilers can also be configured as external packages in the
+``packages.yaml`` config file. Any external package for a compiler
+(e.g. ``gcc`` or ``llvm``) will be treated as a configured compiler
+assuming the paths to the compiler executables are determinable from
+the prefix.
+
+If the paths to the compiler executable are not determinable from the
+prefix, you can add them to the ``extra_attributes`` field. Similarly,
+all other fields from the compilers config can be added to the
+``extra_attributes`` field for an external representing a compiler.
+
+.. code-block:: yaml
+
+ packages:
+ gcc:
+ external:
+ - spec: gcc@12.2.0 arch=linux-rhel8-skylake
+ prefix: /usr
+ extra_attributes:
+ environment:
+ set:
+ GCC_ROOT: /usr
+ external:
+ - spec: llvm+clang@15.0.0 arch=linux-rhel8-skylake
+ prefix: /usr
+ extra_attributes:
+ paths:
+ cc: /usr/bin/clang-with-suffix
+ cxx: /usr/bin/clang++-with-extra-info
+ fc: /usr/bin/gfortran
+ f77: /usr/bin/gfortran
+ extra_rpaths:
+ - /usr/lib/llvm/
^^^^^^^^^^^^^^^^^^^^^^^
Build Your Own Compiler
diff --git a/lib/spack/spack/bootstrap/config.py b/lib/spack/spack/bootstrap/config.py
index 10c5a3db4b..8cba750fc5 100644
--- a/lib/spack/spack/bootstrap/config.py
+++ b/lib/spack/spack/bootstrap/config.py
@@ -147,7 +147,7 @@ def _add_compilers_if_missing() -> None:
mixed_toolchain=sys.platform == "darwin"
)
if new_compilers:
- spack.compilers.add_compilers_to_config(new_compilers, init_config=False)
+ spack.compilers.add_compilers_to_config(new_compilers)
@contextlib.contextmanager
diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py
index 006c6a79a7..860f0a9ee0 100644
--- a/lib/spack/spack/cmd/compiler.py
+++ b/lib/spack/spack/cmd/compiler.py
@@ -89,7 +89,7 @@ def compiler_find(args):
paths, scope=None, mixed_toolchain=args.mixed_toolchain
)
if new_compilers:
- spack.compilers.add_compilers_to_config(new_compilers, scope=args.scope, init_config=False)
+ spack.compilers.add_compilers_to_config(new_compilers, scope=args.scope)
n = len(new_compilers)
s = "s" if n > 1 else ""
diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py
index d735845d86..15c11995a7 100644
--- a/lib/spack/spack/compiler.py
+++ b/lib/spack/spack/compiler.py
@@ -334,6 +334,40 @@ class Compiler:
# used for version checks for API, e.g. C++11 flag
self._real_version = None
+ def __eq__(self, other):
+ return (
+ self.cc == other.cc
+ and self.cxx == other.cxx
+ and self.fc == other.fc
+ and self.f77 == other.f77
+ and self.spec == other.spec
+ and self.operating_system == other.operating_system
+ and self.target == other.target
+ and self.flags == other.flags
+ and self.modules == other.modules
+ and self.environment == other.environment
+ and self.extra_rpaths == other.extra_rpaths
+ and self.enable_implicit_rpaths == other.enable_implicit_rpaths
+ )
+
+ def __hash__(self):
+ return hash(
+ (
+ self.cc,
+ self.cxx,
+ self.fc,
+ self.f77,
+ self.spec,
+ self.operating_system,
+ self.target,
+ str(self.flags),
+ str(self.modules),
+ str(self.environment),
+ str(self.extra_rpaths),
+ self.enable_implicit_rpaths,
+ )
+ )
+
def verify_executables(self):
"""Raise an error if any of the compiler executables is not valid.
diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py
index 1aa0b6a74e..9b73028b12 100644
--- a/lib/spack/spack/compilers/__init__.py
+++ b/lib/spack/spack/compilers/__init__.py
@@ -109,7 +109,7 @@ def _to_dict(compiler):
return {"compiler": d}
-def get_compiler_config(scope=None, init_config=True):
+def get_compiler_config(scope=None, init_config=False):
"""Return the compiler configuration for the specified architecture."""
config = spack.config.get("compilers", scope=scope) or []
@@ -118,6 +118,8 @@ def get_compiler_config(scope=None, init_config=True):
merged_config = spack.config.get("compilers")
if merged_config:
+ # Config is empty for this scope
+ # Do not init config because there is a non-empty scope
return config
_init_compiler_config(scope=scope)
@@ -125,6 +127,95 @@ def get_compiler_config(scope=None, init_config=True):
return config
+def get_compiler_config_from_packages(scope=None):
+ """Return the compiler configuration from packages.yaml"""
+ config = spack.config.get("packages", scope=scope)
+ if not config:
+ return []
+
+ packages = []
+ compiler_package_names = supported_compilers() + list(package_name_to_compiler_name.keys())
+ for name, entry in config.items():
+ if name not in compiler_package_names:
+ continue
+ externals_config = entry.get("externals", None)
+ if not externals_config:
+ continue
+ packages.extend(_compiler_config_from_package_config(externals_config))
+
+ return packages
+
+
+def _compiler_config_from_package_config(config):
+ compilers = []
+ for entry in config:
+ compiler = _compiler_config_from_external(entry)
+ if compiler:
+ compilers.append(compiler)
+
+ return compilers
+
+
+def _compiler_config_from_external(config):
+ spec = spack.spec.parse_with_version_concrete(config["spec"])
+ # use str(spec.versions) to allow `@x.y.z` instead of `@=x.y.z`
+ compiler_spec = spack.spec.CompilerSpec(
+ package_name_to_compiler_name.get(spec.name, spec.name), spec.version
+ )
+
+ extra_attributes = config.get("extra_attributes", {})
+ prefix = config.get("prefix", None)
+
+ compiler_class = class_for_compiler_name(compiler_spec.name)
+ paths = extra_attributes.get("paths", {})
+ compiler_langs = ["cc", "cxx", "fc", "f77"]
+ for lang in compiler_langs:
+ if paths.setdefault(lang, None):
+ continue
+
+ if not prefix:
+ continue
+
+ # Check for files that satisfy the naming scheme for this compiler
+ bindir = os.path.join(prefix, "bin")
+ for f, regex in itertools.product(os.listdir(bindir), compiler_class.search_regexps(lang)):
+ if regex.match(f):
+ paths[lang] = os.path.join(bindir, f)
+
+ if all(v is None for v in paths.values()):
+ return None
+
+ if not spec.architecture:
+ host_platform = spack.platforms.host()
+ operating_system = host_platform.operating_system("default_os")
+ target = host_platform.target("default_target").microarchitecture
+ else:
+ target = spec.target
+ if not target:
+ host_platform = spack.platforms.host()
+ target = host_platform.target("default_target").microarchitecture
+
+ operating_system = spec.os
+ if not operating_system:
+ host_platform = spack.platforms.host()
+ operating_system = host_platform.operating_system("default_os")
+
+ compiler_entry = {
+ "compiler": {
+ "spec": str(compiler_spec),
+ "paths": paths,
+ "flags": extra_attributes.get("flags", {}),
+ "operating_system": str(operating_system),
+ "target": str(target.family),
+ "modules": config.get("modules", []),
+ "environment": extra_attributes.get("environment", {}),
+ "extra_rpaths": extra_attributes.get("extra_rpaths", []),
+ "implicit_rpaths": extra_attributes.get("implicit_rpaths", None),
+ }
+ }
+ return compiler_entry
+
+
def _init_compiler_config(*, scope):
"""Compiler search used when Spack has no compilers."""
compilers = find_compilers()
@@ -142,17 +233,20 @@ def compiler_config_files():
compiler_config = config.get("compilers", scope=name)
if compiler_config:
config_files.append(config.get_config_filename(name, "compilers"))
+ compiler_config_from_packages = get_compiler_config_from_packages(scope=name)
+ if compiler_config_from_packages:
+ config_files.append(config.get_config_filename(name, "packages"))
return config_files
-def add_compilers_to_config(compilers, scope=None, init_config=True):
+def add_compilers_to_config(compilers, scope=None):
"""Add compilers to the config for the specified architecture.
Arguments:
compilers: a list of Compiler objects.
scope: configuration scope to modify.
"""
- compiler_config = get_compiler_config(scope, init_config)
+ compiler_config = get_compiler_config(scope, init_config=False)
for compiler in compilers:
if not compiler.cc:
tty.debug(f"{compiler.spec} does not have a C compiler")
@@ -184,6 +278,9 @@ def remove_compiler_from_config(compiler_spec, scope=None):
for current_scope in candidate_scopes:
removal_happened |= _remove_compiler_from_scope(compiler_spec, scope=current_scope)
+ msg = "`spack compiler remove` will not remove compilers defined in packages.yaml"
+ msg += "\nTo remove these compilers, either edit the config or use `spack external remove`"
+ tty.debug(msg)
return removal_happened
@@ -198,7 +295,7 @@ def _remove_compiler_from_scope(compiler_spec, scope):
True if one or more compiler entries were actually removed, False otherwise
"""
assert scope is not None, "a specific scope is needed when calling this function"
- compiler_config = get_compiler_config(scope)
+ compiler_config = get_compiler_config(scope, init_config=False)
filtered_compiler_config = [
compiler_entry
for compiler_entry in compiler_config
@@ -221,7 +318,14 @@ def all_compilers_config(scope=None, init_config=True):
"""Return a set of specs for all the compiler versions currently
available to build with. These are instances of CompilerSpec.
"""
- return get_compiler_config(scope, init_config)
+ from_packages_yaml = get_compiler_config_from_packages(scope)
+ if from_packages_yaml:
+ init_config = False
+ from_compilers_yaml = get_compiler_config(scope, init_config)
+
+ result = from_compilers_yaml + from_packages_yaml
+ key = lambda c: _compiler_from_config_entry(c["compiler"])
+ return list(llnl.util.lang.dedupe(result, key=key))
def all_compiler_specs(scope=None, init_config=True):
@@ -388,7 +492,7 @@ def find_specs_by_arch(compiler_spec, arch_spec, scope=None, init_config=True):
def all_compilers(scope=None, init_config=True):
- config = get_compiler_config(scope, init_config=init_config)
+ config = all_compilers_config(scope, init_config=init_config)
compilers = list()
for items in config:
items = items["compiler"]
@@ -403,10 +507,7 @@ def compilers_for_spec(
"""This gets all compilers that satisfy the supplied CompilerSpec.
Returns an empty list if none are found.
"""
- if use_cache:
- config = all_compilers_config(scope, init_config)
- else:
- config = get_compiler_config(scope, init_config)
+ config = all_compilers_config(scope, init_config)
matches = set(find(compiler_spec, scope, init_config))
compilers = []
@@ -583,9 +684,7 @@ def get_compiler_duplicates(compiler_spec, arch_spec):
scope_to_compilers = {}
for scope in config.scopes:
- compilers = compilers_for_spec(
- compiler_spec, arch_spec=arch_spec, scope=scope, use_cache=False
- )
+ compilers = compilers_for_spec(compiler_spec, arch_spec=arch_spec, scope=scope)
if compilers:
scope_to_compilers[scope] = compilers
diff --git a/lib/spack/spack/cray_manifest.py b/lib/spack/spack/cray_manifest.py
index eb26b3e6b9..22371f68f2 100644
--- a/lib/spack/spack/cray_manifest.py
+++ b/lib/spack/spack/cray_manifest.py
@@ -227,7 +227,7 @@ def read(path, apply_updates):
if apply_updates and compilers:
for compiler in compilers:
try:
- spack.compilers.add_compilers_to_config([compiler], init_config=False)
+ spack.compilers.add_compilers_to_config([compiler])
except Exception:
warnings.warn(
f"Could not add compiler {str(compiler.spec)}: "
diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py
index 727b46d048..ed59b5cdf1 100644
--- a/lib/spack/spack/environment/environment.py
+++ b/lib/spack/spack/environment/environment.py
@@ -1427,7 +1427,7 @@ class Environment:
# Ensure we have compilers in compilers.yaml to avoid that
# processes try to write the config file in parallel
- _ = spack.compilers.get_compiler_config()
+ _ = spack.compilers.get_compiler_config(init_config=True)
# Early return if there is nothing to do
if len(args) == 0:
diff --git a/lib/spack/spack/test/cmd/compiler.py b/lib/spack/spack/test/cmd/compiler.py
index 3a8c662a5e..849b9e7018 100644
--- a/lib/spack/spack/test/cmd/compiler.py
+++ b/lib/spack/spack/test/cmd/compiler.py
@@ -4,6 +4,7 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import shutil
+import sys
import pytest
@@ -247,3 +248,76 @@ def test_compiler_list_empty(no_compilers_yaml, working_env, compilers_dir):
out = compiler("list")
assert not out
assert compiler.returncode == 0
+
+
+@pytest.mark.parametrize(
+ "external,expected",
+ [
+ (
+ {
+ "spec": "gcc@=7.7.7 os=foobar target=x86_64",
+ "prefix": "/path/to/fake",
+ "modules": ["gcc/7.7.7", "foobar"],
+ "extra_attributes": {
+ "paths": {
+ "cc": "/path/to/fake/gcc",
+ "cxx": "/path/to/fake/g++",
+ "fc": "/path/to/fake/gfortran",
+ "f77": "/path/to/fake/gfortran",
+ },
+ "flags": {"fflags": "-ffree-form"},
+ },
+ },
+ """gcc@7.7.7:
+\tpaths:
+\t\tcc = /path/to/fake/gcc
+\t\tcxx = /path/to/fake/g++
+\t\tf77 = /path/to/fake/gfortran
+\t\tfc = /path/to/fake/gfortran
+\tflags:
+\t\tfflags = ['-ffree-form']
+\tmodules = ['gcc/7.7.7', 'foobar']
+\toperating system = foobar
+""",
+ ),
+ (
+ {
+ "spec": "gcc@7.7.7",
+ "prefix": "{prefix}",
+ "modules": ["gcc/7.7.7", "foobar"],
+ "extra_attributes": {"flags": {"fflags": "-ffree-form"}},
+ },
+ """gcc@7.7.7:
+\tpaths:
+\t\tcc = {compilers_dir}{sep}gcc-8{suffix}
+\t\tcxx = {compilers_dir}{sep}g++-8{suffix}
+\t\tf77 = {compilers_dir}{sep}gfortran-8{suffix}
+\t\tfc = {compilers_dir}{sep}gfortran-8{suffix}
+\tflags:
+\t\tfflags = ['-ffree-form']
+\tmodules = ['gcc/7.7.7', 'foobar']
+\toperating system = debian6
+""",
+ ),
+ ],
+)
+def test_compilers_shows_packages_yaml(
+ external, expected, no_compilers_yaml, working_env, compilers_dir
+):
+ """Spack should see a single compiler defined from packages.yaml"""
+ external["prefix"] = external["prefix"].format(prefix=os.path.dirname(compilers_dir))
+ gcc_entry = {"externals": [external]}
+
+ packages = spack.config.get("packages")
+ packages["gcc"] = gcc_entry
+ spack.config.set("packages", packages)
+
+ out = compiler("list")
+ assert out.count("gcc@7.7.7") == 1
+
+ out = compiler("info", "gcc@7.7.7")
+ assert out == expected.format(
+ compilers_dir=str(compilers_dir),
+ sep=os.sep,
+ suffix=".bat" if sys.platform == "win32" else "",
+ )
--
cgit v1.2.3-70-g09d2