spack audit externals: allow selecting platforms and checking extra attributes (#43782)

10 files changed, 124 insertions, 3 deletions

diff --git a/.github/workflows/audit.yaml b/.github/workflows/audit.yaml
index 64ef40611f..de82a31c06 100644
--- a/.github/workflows/audit.yaml
+++ b/.github/workflows/audit.yaml
@@ -34,7 +34,7 @@ jobs:
       run: |
           . share/spack/setup-env.sh
           coverage run $(which spack) audit packages
-          coverage run $(which spack) audit externals        
+          coverage run $(which spack) -d audit externals        
           coverage combine
           coverage xml
     - name: Package audits (without coverage)
diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst
index 7d8efa3987..46ab71b93f 100644
--- a/lib/spack/docs/packaging_guide.rst
+++ b/lib/spack/docs/packaging_guide.rst
@@ -6435,9 +6435,12 @@ the ``paths`` attribute:
          echo "Target: x86_64-pc-linux-gnu"
          echo "Thread model: posix"
          echo "InstalledDir: /usr/bin"
+     platforms: ["linux", "darwin"]
      results:
      - spec: 'llvm@3.9.1 +clang~lld~lldb'
 
+If the ``platforms`` attribute is present, tests are run only if the current host
+matches one of the listed platforms.
 Each test is performed by first creating a temporary directory structure as
 specified in the corresponding ``layout`` and by then running
 package detection and checking that the outcome matches the expected
@@ -6471,6 +6474,10 @@ package detection and checking that the outcome matches the expected
      - A spec that is expected from detection
      - Any valid spec
      - Yes
+   * - ``results:[0]:extra_attributes``
+     - Extra attributes expected on the associated Spec
+     - Nested dictionary with string as keys, and regular expressions as leaf values
+     - No
 
 """""""""""""""""""""""""""""""
 Reuse tests from other packages
diff --git a/lib/spack/spack/audit.py b/lib/spack/spack/audit.py
index bbf2ed19db..96225fd6ac 100644
--- a/lib/spack/spack/audit.py
+++ b/lib/spack/spack/audit.py
@@ -1111,4 +1111,76 @@ def _test_detection_by_executable(pkgs, error_cls):
                 details = [msg.format(s, idx) for s in sorted(not_expected)]
                 errors.append(error_cls(summary=summary, details=details))
 
+            matched_detection = []
+            for candidate in expected_specs:
+                try:
+                    idx = specs.index(candidate)
+                except (AttributeError, ValueError):
+                    pass
+
+                matched_detection.append((candidate, specs[idx]))
+
+            def _compare_extra_attribute(_expected, _detected, *, _spec):
+                result = []
+                # Check items are of the same type
+                if not isinstance(_detected, type(_expected)):
+                    _summary = f'{pkg_name}: error when trying to detect "{_expected}"'
+                    _details = [f"{_detected} was detected instead"]
+                    return [error_cls(summary=_summary, details=_details)]
+
+                # If they are string expected is a regex
+                if isinstance(_expected, str):
+                    try:
+                        _regex = re.compile(_expected)
+                    except re.error:
+                        _summary = f'{pkg_name}: illegal regex in "{_spec}" extra attributes'
+                        _details = [f"{_expected} is not a valid regex"]
+                        return [error_cls(summary=_summary, details=_details)]
+
+                    if not _regex.match(_detected):
+                        _summary = (
+                            f'{pkg_name}: error when trying to match "{_expected}" '
+                            f"in extra attributes"
+                        )
+                        _details = [f"{_detected} does not match the regex"]
+                        return [error_cls(summary=_summary, details=_details)]
+
+                if isinstance(_expected, dict):
+                    _not_detected = set(_expected.keys()) - set(_detected.keys())
+                    if _not_detected:
+                        _summary = f"{pkg_name}: cannot detect some attributes for spec {_spec}"
+                        _details = [
+                            f'"{_expected}" was expected',
+                            f'"{_detected}" was detected',
+                        ] + [f'attribute "{s}" was not detected' for s in sorted(_not_detected)]
+                        result.append(error_cls(summary=_summary, details=_details))
+
+                    _common = set(_expected.keys()) & set(_detected.keys())
+                    for _key in _common:
+                        result.extend(
+                            _compare_extra_attribute(_expected[_key], _detected[_key], _spec=_spec)
+                        )
+
+                return result
+
+            for expected, detected in matched_detection:
+                # We might not want to test all attributes, so avoid not_expected
+                not_detected = set(expected.extra_attributes) - set(detected.extra_attributes)
+                if not_detected:
+                    summary = f"{pkg_name}: cannot detect some attributes for spec {expected}"
+                    details = [
+                        f'"{s}" was not detected [test_id={idx}]' for s in sorted(not_detected)
+                    ]
+                    errors.append(error_cls(summary=summary, details=details))
+
+                common = set(expected.extra_attributes) & set(detected.extra_attributes)
+                for key in common:
+                    errors.extend(
+                        _compare_extra_attribute(
+                            expected.extra_attributes[key],
+                            detected.extra_attributes[key],
+                            _spec=expected,
+                        )
+                    )
+
     return errors
diff --git a/lib/spack/spack/detection/test.py b/lib/spack/spack/detection/test.py
index 657188a38c..353a0a5660 100644
--- a/lib/spack/spack/detection/test.py
+++ b/lib/spack/spack/detection/test.py
@@ -11,6 +11,7 @@ from typing import Any, Deque, Dict, Generator, List, NamedTuple, Tuple
 
 from llnl.util import filesystem
 
+import spack.platforms
 import spack.repo
 import spack.spec
 from spack.util import spack_yaml
@@ -32,6 +33,8 @@ class ExpectedTestResult(NamedTuple):
 
     #: Spec to be detected
     spec: str
+    #: Attributes expected in the external spec
+    extra_attributes: Dict[str, str]
 
 
 class DetectionTest(NamedTuple):
@@ -100,7 +103,10 @@ class Runner:
 
     @property
     def expected_specs(self) -> List[spack.spec.Spec]:
-        return [spack.spec.Spec(r.spec) for r in self.test.results]
+        return [
+            spack.spec.Spec.from_detection(item.spec, extra_attributes=item.extra_attributes)
+            for item in self.test.results
+        ]
 
 
 def detection_tests(pkg_name: str, repository: spack.repo.RepoPath) -> List[Runner]:
@@ -117,9 +123,13 @@ def detection_tests(pkg_name: str, repository: spack.repo.RepoPath) -> List[Runn
     """
     result = []
     detection_tests_content = read_detection_tests(pkg_name, repository)
+    current_platform = str(spack.platforms.host())
 
     tests_by_path = detection_tests_content.get("paths", [])
     for single_test_data in tests_by_path:
+        if current_platform not in single_test_data.get("platforms", [current_platform]):
+            continue
+
         mock_executables = []
         for layout in single_test_data["layout"]:
             mock_executables.append(
@@ -127,7 +137,11 @@ def detection_tests(pkg_name: str, repository: spack.repo.RepoPath) -> List[Runn
             )
         expected_results = []
         for assertion in single_test_data["results"]:
-            expected_results.append(ExpectedTestResult(spec=assertion["spec"]))
+            expected_results.append(
+                ExpectedTestResult(
+                    spec=assertion["spec"], extra_attributes=assertion.get("extra_attributes", {})
+                )
+            )
 
         current_test = DetectionTest(
             pkg_name=pkg_name, layout=mock_executables, results=expected_results
diff --git a/var/spack/repos/builtin/packages/apple-clang/detection_test.yaml b/var/spack/repos/builtin/packages/apple-clang/detection_test.yaml
index dc5b7106ec..dffdf88286 100644
--- a/var/spack/repos/builtin/packages/apple-clang/detection_test.yaml
+++ b/var/spack/repos/builtin/packages/apple-clang/detection_test.yaml
@@ -9,6 +9,7 @@ paths:
           echo "Target: x86_64-apple-darwin19.5.0"
           echo "Thread model: posix"
           echo "InstalledDir: /Library/Developer/CommandLineTools/usr/bin"
+    platforms: ["darwin"]
     results:
       - spec: 'apple-clang@11.0.0'
   # Apple Clang on Apple M1 (Ventura)
@@ -21,6 +22,7 @@ paths:
           echo "Target: arm64-apple-darwin22.6.0"
           echo "Thread model: posix"
           echo "InstalledDir: /Library/Developer/CommandLineTools/usr/bin"
+    platforms: ["darwin"]
     results:
       - spec: 'apple-clang@15.0.0'
   # Test that missing a compiler prevents the package from being detected
@@ -32,4 +34,5 @@ paths:
           echo "Target: x86_64-apple-darwin19.5.0"
           echo "Thread model: posix"
           echo "InstalledDir: /Library/Developer/CommandLineTools/usr/bin"
+    platforms: ["darwin"]
     results: [ ]
diff --git a/var/spack/repos/builtin/packages/gcc/detection_test.yaml b/var/spack/repos/builtin/packages/gcc/detection_test.yaml
index a7a9e55cd1..cdce43196e 100644
--- a/var/spack/repos/builtin/packages/gcc/detection_test.yaml
+++ b/var/spack/repos/builtin/packages/gcc/detection_test.yaml
@@ -20,8 +20,13 @@ paths:
             echo "mock executable got an unexpected flag: $1"
             exit 1
           fi
+    platforms: ["darwin", "linux"]
     results:
       - spec: "gcc@9.4.0 languages=c,c++"
+        extra_attributes:
+          compilers:
+            c: ".*/bin/gcc"
+            cxx: ".*/bin/g++"
   # Mock a version < 7 of GCC that requires -dumpversion and
   # errors with -dumpfullversion
   - layout:
@@ -37,8 +42,14 @@ paths:
             echo "compilation terminated."
             exit 1
           fi
+    platforms: ["darwin", "linux"]
     results:
       - spec: "gcc@5.5.0 languages=c,c++,fortran"
+        extra_attributes:
+          compilers:
+            c: ".*/bin/gcc-5$"
+            cxx: ".*/bin/g[+][+]-5$"
+            fortran: ".*/bin/gfortran-5$"
   # Multiple compilers present at the same time
   - layout:
       - executables:
@@ -50,7 +61,14 @@ paths:
         script: "echo 10.1.0"
     results:
       - spec: "gcc@6.5.0 languages=c"
+        extra_attributes:
+          compilers:
+            c: ".*/bin/x86_64-linux-gnu-gcc-6$"
       - spec: "gcc@10.1.0 languages=c,c++"
+        extra_attributes:
+          compilers:
+            c: ".*/bin/x86_64-linux-gnu-gcc-10$"
+            cxx: ".*/bin/x86_64-linux-gnu-g[+][+]-10$"
   # Apple clang under disguise as gcc should not be detected
   - layout:
       - executables:
@@ -70,4 +88,5 @@ paths:
             echo "mock executable got an unexpected flag: $1"
             exit 1
           fi
+    platforms: ["darwin"]
     results: []
diff --git a/var/spack/repos/builtin/packages/intel/detection_test.yaml b/var/spack/repos/builtin/packages/intel/detection_test.yaml
index 076bfeaaba..1a0e23b82c 100644
--- a/var/spack/repos/builtin/packages/intel/detection_test.yaml
+++ b/var/spack/repos/builtin/packages/intel/detection_test.yaml
@@ -15,5 +15,6 @@ paths:
         script: |
           echo "ifort (IFORT) 18.0.5 20180823"
           echo "Copyright (C) 1985-2018 Intel Corporation.  All rights reserved."
+    platforms: ["darwin", "linux"]
     results:
       - spec: 'intel@18.0.5'
diff --git a/var/spack/repos/builtin/packages/llvm/detection_test.yaml b/var/spack/repos/builtin/packages/llvm/detection_test.yaml
index 48e9d6751a..a5719fb395 100644
--- a/var/spack/repos/builtin/packages/llvm/detection_test.yaml
+++ b/var/spack/repos/builtin/packages/llvm/detection_test.yaml
@@ -14,6 +14,7 @@ paths:
           echo "Target: x86_64-pc-linux-gnu"
           echo "Thread model: posix"
           echo "InstalledDir: /usr/bin"
+    platforms: ["darwin", "linux"]
     results:
       - spec: 'llvm@3.9.1 +clang~lld~lldb'
   # Multiple LLVM packages in the same prefix
@@ -40,6 +41,7 @@ paths:
           echo "Target: x86_64-pc-linux-gnu"
           echo "Thread model: posix"
           echo "InstalledDir: /usr/bin"
+    platforms: ["darwin", "linux"]
     results:
       - spec: 'llvm@8.0.0+clang+lld+lldb'
       - spec: 'llvm@3.9.1+clang~lld~lldb'
@@ -53,4 +55,5 @@ paths:
         echo "Target: x86_64-apple-darwin19.5.0"
         echo "Thread model: posix"
         echo "InstalledDir: /Library/Developer/CommandLineTools/usr/bin"
+    platforms: ["darwin"]
     results: []
diff --git a/var/spack/repos/builtin/packages/xlc/detection_test.yaml b/var/spack/repos/builtin/packages/xlc/detection_test.yaml
index e76c608aba..5bc2c1f13b 100644
--- a/var/spack/repos/builtin/packages/xlc/detection_test.yaml
+++ b/var/spack/repos/builtin/packages/xlc/detection_test.yaml
@@ -13,6 +13,7 @@ paths:
         script: |
           echo "IBM XL Fortran for Linux, V16.1.1 (5725-C73, 5765-J13)"
           echo "Version: 16.01.0001.0006"
+    platforms: ["linux"]
     results:
       - spec: "xlc+r@16.1"
       - spec: "xlc~r@16.1"
diff --git a/var/spack/repos/builtin/packages/xlf/detection_test.yaml b/var/spack/repos/builtin/packages/xlf/detection_test.yaml
index 461ef007d7..1ba6955a94 100644
--- a/var/spack/repos/builtin/packages/xlf/detection_test.yaml
+++ b/var/spack/repos/builtin/packages/xlf/detection_test.yaml
@@ -6,6 +6,7 @@ paths:
         script: |
           echo "IBM XL Fortran for Linux, V16.1.1 (5725-C73, 5765-J13)"
           echo "Version: 16.01.0001.0006"
+    platforms: ["linux"]
     results:
       - spec: "xlf+r@16.1"
       - spec: "xlf~r@16.1"