Allow compilers to be configured in packages.yaml (#42016)

Co-authored-by: becker33 <becker33@users.noreply.github.com>

8 files changed, 270 insertions, 20 deletions

diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst
index d7f913d646..ab9c274e01 100644
--- a/lib/spack/docs/getting_started.rst
+++ b/lib/spack/docs/getting_started.rst
@@ -250,9 +250,10 @@ Compiler configuration
 
 Spack has the ability to build packages with multiple compilers and
 compiler versions. Compilers can be made available to Spack by
-specifying them manually in ``compilers.yaml``, or automatically by
-running ``spack compiler find``, but for convenience Spack will
-automatically detect compilers the first time it needs them.
+specifying them manually in ``compilers.yaml`` or ``packages.yaml``,
+or automatically by running ``spack compiler find``, but for
+convenience Spack will automatically detect compilers the first time
+it needs them.
 
 .. _cmd-spack-compilers:
 
@@ -457,6 +458,48 @@ specification. The operations available to modify the environment are ``set``, `
          prepend_path: # Similar for append|remove_path
            LD_LIBRARY_PATH: /ld/paths/added/by/setvars/sh
 
+.. note::
+
+   Spack is in the process of moving compilers from a separate
+   attribute to be handled like all other packages. As part of this
+   process, the ``compilers.yaml`` section will eventually be replaced
+   by configuration in the ``packages.yaml`` section. This new
+   configuration is now available, although it is not yet the default
+   behavior.
+
+Compilers can also be configured as external packages in the
+``packages.yaml`` config file. Any external package for a compiler
+(e.g. ``gcc`` or ``llvm``) will be treated as a configured compiler
+assuming the paths to the compiler executables are determinable from
+the prefix.
+
+If the paths to the compiler executable are not determinable from the
+prefix, you can add them to the ``extra_attributes`` field. Similarly,
+all other fields from the compilers config can be added to the
+``extra_attributes`` field for an external representing a compiler.
+
+.. code-block:: yaml
+
+   packages:
+     gcc:
+       external:
+       - spec: gcc@12.2.0 arch=linux-rhel8-skylake
+         prefix: /usr
+         extra_attributes:
+           environment:
+             set:
+               GCC_ROOT: /usr
+       external:
+       - spec: llvm+clang@15.0.0 arch=linux-rhel8-skylake
+         prefix: /usr
+         extra_attributes:
+           paths:
+             cc: /usr/bin/clang-with-suffix
+             cxx: /usr/bin/clang++-with-extra-info
+             fc: /usr/bin/gfortran
+             f77: /usr/bin/gfortran
+           extra_rpaths:
+           - /usr/lib/llvm/
 
 ^^^^^^^^^^^^^^^^^^^^^^^
 Build Your Own Compiler
diff --git a/lib/spack/spack/bootstrap/config.py b/lib/spack/spack/bootstrap/config.py
index 10c5a3db4b..8cba750fc5 100644
--- a/lib/spack/spack/bootstrap/config.py
+++ b/lib/spack/spack/bootstrap/config.py
@@ -147,7 +147,7 @@ def _add_compilers_if_missing() -> None:
             mixed_toolchain=sys.platform == "darwin"
         )
         if new_compilers:
-            spack.compilers.add_compilers_to_config(new_compilers, init_config=False)
+            spack.compilers.add_compilers_to_config(new_compilers)
 
 
 @contextlib.contextmanager
diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py
index 006c6a79a7..860f0a9ee0 100644
--- a/lib/spack/spack/cmd/compiler.py
+++ b/lib/spack/spack/cmd/compiler.py
@@ -89,7 +89,7 @@ def compiler_find(args):
         paths, scope=None, mixed_toolchain=args.mixed_toolchain
     )
     if new_compilers:
-        spack.compilers.add_compilers_to_config(new_compilers, scope=args.scope, init_config=False)
+        spack.compilers.add_compilers_to_config(new_compilers, scope=args.scope)
         n = len(new_compilers)
         s = "s" if n > 1 else ""
 
diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py
index d735845d86..15c11995a7 100644
--- a/lib/spack/spack/compiler.py
+++ b/lib/spack/spack/compiler.py
@@ -334,6 +334,40 @@ class Compiler:
         # used for version checks for API, e.g. C++11 flag
         self._real_version = None
 
+    def __eq__(self, other):
+        return (
+            self.cc == other.cc
+            and self.cxx == other.cxx
+            and self.fc == other.fc
+            and self.f77 == other.f77
+            and self.spec == other.spec
+            and self.operating_system == other.operating_system
+            and self.target == other.target
+            and self.flags == other.flags
+            and self.modules == other.modules
+            and self.environment == other.environment
+            and self.extra_rpaths == other.extra_rpaths
+            and self.enable_implicit_rpaths == other.enable_implicit_rpaths
+        )
+
+    def __hash__(self):
+        return hash(
+            (
+                self.cc,
+                self.cxx,
+                self.fc,
+                self.f77,
+                self.spec,
+                self.operating_system,
+                self.target,
+                str(self.flags),
+                str(self.modules),
+                str(self.environment),
+                str(self.extra_rpaths),
+                self.enable_implicit_rpaths,
+            )
+        )
+
     def verify_executables(self):
         """Raise an error if any of the compiler executables is not valid.
 
diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py
index 1aa0b6a74e..9b73028b12 100644
--- a/lib/spack/spack/compilers/__init__.py
+++ b/lib/spack/spack/compilers/__init__.py
@@ -109,7 +109,7 @@ def _to_dict(compiler):
     return {"compiler": d}
 
 
-def get_compiler_config(scope=None, init_config=True):
+def get_compiler_config(scope=None, init_config=False):
     """Return the compiler configuration for the specified architecture."""
 
     config = spack.config.get("compilers", scope=scope) or []
@@ -118,6 +118,8 @@ def get_compiler_config(scope=None, init_config=True):
 
     merged_config = spack.config.get("compilers")
     if merged_config:
+        # Config is empty for this scope
+        # Do not init config because there is a non-empty scope
         return config
 
     _init_compiler_config(scope=scope)
@@ -125,6 +127,95 @@ def get_compiler_config(scope=None, init_config=True):
     return config
 
 
+def get_compiler_config_from_packages(scope=None):
+    """Return the compiler configuration from packages.yaml"""
+    config = spack.config.get("packages", scope=scope)
+    if not config:
+        return []
+
+    packages = []
+    compiler_package_names = supported_compilers() + list(package_name_to_compiler_name.keys())
+    for name, entry in config.items():
+        if name not in compiler_package_names:
+            continue
+        externals_config = entry.get("externals", None)
+        if not externals_config:
+            continue
+        packages.extend(_compiler_config_from_package_config(externals_config))
+
+    return packages
+
+
+def _compiler_config_from_package_config(config):
+    compilers = []
+    for entry in config:
+        compiler = _compiler_config_from_external(entry)
+        if compiler:
+            compilers.append(compiler)
+
+    return compilers
+
+
+def _compiler_config_from_external(config):
+    spec = spack.spec.parse_with_version_concrete(config["spec"])
+    # use str(spec.versions) to allow `@x.y.z` instead of `@=x.y.z`
+    compiler_spec = spack.spec.CompilerSpec(
+        package_name_to_compiler_name.get(spec.name, spec.name), spec.version
+    )
+
+    extra_attributes = config.get("extra_attributes", {})
+    prefix = config.get("prefix", None)
+
+    compiler_class = class_for_compiler_name(compiler_spec.name)
+    paths = extra_attributes.get("paths", {})
+    compiler_langs = ["cc", "cxx", "fc", "f77"]
+    for lang in compiler_langs:
+        if paths.setdefault(lang, None):
+            continue
+
+        if not prefix:
+            continue
+
+        # Check for files that satisfy the naming scheme for this compiler
+        bindir = os.path.join(prefix, "bin")
+        for f, regex in itertools.product(os.listdir(bindir), compiler_class.search_regexps(lang)):
+            if regex.match(f):
+                paths[lang] = os.path.join(bindir, f)
+
+    if all(v is None for v in paths.values()):
+        return None
+
+    if not spec.architecture:
+        host_platform = spack.platforms.host()
+        operating_system = host_platform.operating_system("default_os")
+        target = host_platform.target("default_target").microarchitecture
+    else:
+        target = spec.target
+        if not target:
+            host_platform = spack.platforms.host()
+            target = host_platform.target("default_target").microarchitecture
+
+        operating_system = spec.os
+        if not operating_system:
+            host_platform = spack.platforms.host()
+            operating_system = host_platform.operating_system("default_os")
+
+    compiler_entry = {
+        "compiler": {
+            "spec": str(compiler_spec),
+            "paths": paths,
+            "flags": extra_attributes.get("flags", {}),
+            "operating_system": str(operating_system),
+            "target": str(target.family),
+            "modules": config.get("modules", []),
+            "environment": extra_attributes.get("environment", {}),
+            "extra_rpaths": extra_attributes.get("extra_rpaths", []),
+            "implicit_rpaths": extra_attributes.get("implicit_rpaths", None),
+        }
+    }
+    return compiler_entry
+
+
 def _init_compiler_config(*, scope):
     """Compiler search used when Spack has no compilers."""
     compilers = find_compilers()
@@ -142,17 +233,20 @@ def compiler_config_files():
         compiler_config = config.get("compilers", scope=name)
         if compiler_config:
             config_files.append(config.get_config_filename(name, "compilers"))
+        compiler_config_from_packages = get_compiler_config_from_packages(scope=name)
+        if compiler_config_from_packages:
+            config_files.append(config.get_config_filename(name, "packages"))
     return config_files
 
 
-def add_compilers_to_config(compilers, scope=None, init_config=True):
+def add_compilers_to_config(compilers, scope=None):
     """Add compilers to the config for the specified architecture.
 
     Arguments:
         compilers: a list of Compiler objects.
         scope: configuration scope to modify.
     """
-    compiler_config = get_compiler_config(scope, init_config)
+    compiler_config = get_compiler_config(scope, init_config=False)
     for compiler in compilers:
         if not compiler.cc:
             tty.debug(f"{compiler.spec} does not have a C compiler")
@@ -184,6 +278,9 @@ def remove_compiler_from_config(compiler_spec, scope=None):
     for current_scope in candidate_scopes:
         removal_happened |= _remove_compiler_from_scope(compiler_spec, scope=current_scope)
 
+    msg = "`spack compiler remove` will not remove compilers defined in packages.yaml"
+    msg += "\nTo remove these compilers, either edit the config or use `spack external remove`"
+    tty.debug(msg)
     return removal_happened
 
 
@@ -198,7 +295,7 @@ def _remove_compiler_from_scope(compiler_spec, scope):
          True if one or more compiler entries were actually removed, False otherwise
     """
     assert scope is not None, "a specific scope is needed when calling this function"
-    compiler_config = get_compiler_config(scope)
+    compiler_config = get_compiler_config(scope, init_config=False)
     filtered_compiler_config = [
         compiler_entry
         for compiler_entry in compiler_config
@@ -221,7 +318,14 @@ def all_compilers_config(scope=None, init_config=True):
     """Return a set of specs for all the compiler versions currently
     available to build with.  These are instances of CompilerSpec.
     """
-    return get_compiler_config(scope, init_config)
+    from_packages_yaml = get_compiler_config_from_packages(scope)
+    if from_packages_yaml:
+        init_config = False
+    from_compilers_yaml = get_compiler_config(scope, init_config)
+
+    result = from_compilers_yaml + from_packages_yaml
+    key = lambda c: _compiler_from_config_entry(c["compiler"])
+    return list(llnl.util.lang.dedupe(result, key=key))
 
 
 def all_compiler_specs(scope=None, init_config=True):
@@ -388,7 +492,7 @@ def find_specs_by_arch(compiler_spec, arch_spec, scope=None, init_config=True):
 
 
 def all_compilers(scope=None, init_config=True):
-    config = get_compiler_config(scope, init_config=init_config)
+    config = all_compilers_config(scope, init_config=init_config)
     compilers = list()
     for items in config:
         items = items["compiler"]
@@ -403,10 +507,7 @@ def compilers_for_spec(
     """This gets all compilers that satisfy the supplied CompilerSpec.
     Returns an empty list if none are found.
     """
-    if use_cache:
-        config = all_compilers_config(scope, init_config)
-    else:
-        config = get_compiler_config(scope, init_config)
+    config = all_compilers_config(scope, init_config)
 
     matches = set(find(compiler_spec, scope, init_config))
     compilers = []
@@ -583,9 +684,7 @@ def get_compiler_duplicates(compiler_spec, arch_spec):
 
     scope_to_compilers = {}
     for scope in config.scopes:
-        compilers = compilers_for_spec(
-            compiler_spec, arch_spec=arch_spec, scope=scope, use_cache=False
-        )
+        compilers = compilers_for_spec(compiler_spec, arch_spec=arch_spec, scope=scope)
         if compilers:
             scope_to_compilers[scope] = compilers
 
diff --git a/lib/spack/spack/cray_manifest.py b/lib/spack/spack/cray_manifest.py
index eb26b3e6b9..22371f68f2 100644
--- a/lib/spack/spack/cray_manifest.py
+++ b/lib/spack/spack/cray_manifest.py
@@ -227,7 +227,7 @@ def read(path, apply_updates):
     if apply_updates and compilers:
         for compiler in compilers:
             try:
-                spack.compilers.add_compilers_to_config([compiler], init_config=False)
+                spack.compilers.add_compilers_to_config([compiler])
             except Exception:
                 warnings.warn(
                     f"Could not add compiler {str(compiler.spec)}: "
diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py
index 727b46d048..ed59b5cdf1 100644
--- a/lib/spack/spack/environment/environment.py
+++ b/lib/spack/spack/environment/environment.py
@@ -1427,7 +1427,7 @@ class Environment:
 
         # Ensure we have compilers in compilers.yaml to avoid that
         # processes try to write the config file in parallel
-        _ = spack.compilers.get_compiler_config()
+        _ = spack.compilers.get_compiler_config(init_config=True)
 
         # Early return if there is nothing to do
         if len(args) == 0:
diff --git a/lib/spack/spack/test/cmd/compiler.py b/lib/spack/spack/test/cmd/compiler.py
index 3a8c662a5e..849b9e7018 100644
--- a/lib/spack/spack/test/cmd/compiler.py
+++ b/lib/spack/spack/test/cmd/compiler.py
@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os
 import shutil
+import sys
 
 import pytest
 
@@ -247,3 +248,76 @@ def test_compiler_list_empty(no_compilers_yaml, working_env, compilers_dir):
     out = compiler("list")
     assert not out
     assert compiler.returncode == 0
+
+
+@pytest.mark.parametrize(
+    "external,expected",
+    [
+        (
+            {
+                "spec": "gcc@=7.7.7 os=foobar target=x86_64",
+                "prefix": "/path/to/fake",
+                "modules": ["gcc/7.7.7", "foobar"],
+                "extra_attributes": {
+                    "paths": {
+                        "cc": "/path/to/fake/gcc",
+                        "cxx": "/path/to/fake/g++",
+                        "fc": "/path/to/fake/gfortran",
+                        "f77": "/path/to/fake/gfortran",
+                    },
+                    "flags": {"fflags": "-ffree-form"},
+                },
+            },
+            """gcc@7.7.7:
+\tpaths:
+\t\tcc = /path/to/fake/gcc
+\t\tcxx = /path/to/fake/g++
+\t\tf77 = /path/to/fake/gfortran
+\t\tfc = /path/to/fake/gfortran
+\tflags:
+\t\tfflags = ['-ffree-form']
+\tmodules  = ['gcc/7.7.7', 'foobar']
+\toperating system  = foobar
+""",
+        ),
+        (
+            {
+                "spec": "gcc@7.7.7",
+                "prefix": "{prefix}",
+                "modules": ["gcc/7.7.7", "foobar"],
+                "extra_attributes": {"flags": {"fflags": "-ffree-form"}},
+            },
+            """gcc@7.7.7:
+\tpaths:
+\t\tcc = {compilers_dir}{sep}gcc-8{suffix}
+\t\tcxx = {compilers_dir}{sep}g++-8{suffix}
+\t\tf77 = {compilers_dir}{sep}gfortran-8{suffix}
+\t\tfc = {compilers_dir}{sep}gfortran-8{suffix}
+\tflags:
+\t\tfflags = ['-ffree-form']
+\tmodules  = ['gcc/7.7.7', 'foobar']
+\toperating system  = debian6
+""",
+        ),
+    ],
+)
+def test_compilers_shows_packages_yaml(
+    external, expected, no_compilers_yaml, working_env, compilers_dir
+):
+    """Spack should see a single compiler defined from packages.yaml"""
+    external["prefix"] = external["prefix"].format(prefix=os.path.dirname(compilers_dir))
+    gcc_entry = {"externals": [external]}
+
+    packages = spack.config.get("packages")
+    packages["gcc"] = gcc_entry
+    spack.config.set("packages", packages)
+
+    out = compiler("list")
+    assert out.count("gcc@7.7.7") == 1
+
+    out = compiler("info", "gcc@7.7.7")
+    assert out == expected.format(
+        compilers_dir=str(compilers_dir),
+        sep=os.sep,
+        suffix=".bat" if sys.platform == "win32" else "",
+    )