add `spack test list --all` (#22032)

`spack test list` will show you which *installed* packages can be tested
but it won't show you which packages have tests.

- [x] add `spack test list --all` to show which packages have test methods
- [x] update `has_test_method()` to handle package instances *and*
      package classes.

2 files changed, 38 insertions, 5 deletions

diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py
index e1aedabb81..87dce7a19f 100644
--- a/lib/spack/spack/cmd/test.py
+++ b/lib/spack/spack/cmd/test.py
@@ -10,15 +10,18 @@ import textwrap
 import fnmatch
 import re
 import shutil
+import sys
 
 import llnl.util.tty as tty
+import llnl.util.tty.colify as colify
 
 import spack.install_test
 import spack.environment as ev
 import spack.cmd
 import spack.cmd.common.arguments as arguments
-import spack.report
 import spack.package
+import spack.repo
+import spack.report
 
 description = "run spack's tests for an install"
 section = "admin"
@@ -78,8 +81,11 @@ def setup_parser(subparser):
     arguments.add_common_arguments(run_parser, ['installed_specs'])
 
     # List
-    sp.add_parser('list', description=test_list.__doc__,
-                  help=first_line(test_list.__doc__))
+    list_parser = sp.add_parser('list', description=test_list.__doc__,
+                                help=first_line(test_list.__doc__))
+    list_parser.add_argument(
+        "-a", "--all", action="store_true", dest="list_all",
+        help="list all packages with tests (not just installed)")
 
     # Find
     find_parser = sp.add_parser('find', description=test_find.__doc__,
@@ -188,11 +194,26 @@ environment variables:
 
 
 def has_test_method(pkg):
-    return pkg.test.__func__ != spack.package.PackageBase.test
+    pkg_base = spack.package.PackageBase
+    return (
+        (issubclass(pkg, pkg_base) and pkg.test != pkg_base.test) or
+        (isinstance(pkg, pkg_base) and pkg.test.__func__ != pkg_base.test)
+    )
 
 
 def test_list(args):
-    """List all installed packages with available tests."""
+    """List installed packages with available tests."""
+    if args.list_all:
+        all_packages_with_tests = [
+            pkg_class.name
+            for pkg_class in spack.repo.path.all_package_classes()
+            if has_test_method(pkg_class)
+        ]
+        if sys.stdout.isatty():
+            tty.msg("%d packages with tests." % len(all_packages_with_tests))
+        colify.colify(all_packages_with_tests)
+        return
+
     # TODO: This can be extended to have all of the output formatting options
     # from `spack find`.
     env = ev.get_env(args, 'test')
diff --git a/lib/spack/spack/test/cmd/test.py b/lib/spack/spack/test/cmd/test.py
index 8b1a2d053f..da360f14dd 100644
--- a/lib/spack/spack/test/cmd/test.py
+++ b/lib/spack/spack/test/cmd/test.py
@@ -181,3 +181,15 @@ def test_test_help_cdash(mock_test_stage):
     """Make sure `spack test --help-cdash` describes CDash arguments"""
     out = spack_test('run', '--help-cdash')
     assert 'CDash URL' in out
+
+
+def test_list_all(mock_packages):
+    """make sure `spack test list --all` returns all packages with tests"""
+    pkgs = spack_test("list", "--all").strip().split()
+    assert set(pkgs) == set([
+        "printing-package",
+        "py-extension1",
+        "py-extension2",
+        "test-error",
+        "test-fail",
+    ])