Stand-alone testing: make recipe support and processing spack-/pytest-like (#34236)

This is a refactor of Spack's stand-alone test process to be more spack- and pytest-like. 

It is more spack-like in that test parts are no longer "hidden" in a package's run_test()
method and pytest-like in that any package method whose name starts test_ 
(i.e., a "test" method) is a test part. We also support the ability to embed test parts in a
test method when that makes sense.

Test methods are now implicit test parts. The docstring is the purpose for the test part. 
The name of the method is the name of the test part. The working directory is the active
spec's test stage directory. You can embed test parts using the test_part context manager.

Functionality added by this commit:
* Adds support for multiple test_* stand-alone package test methods, each of which is 
   an implicit test_part for execution and reporting purposes;
* Deprecates package use of run_test();
* Exposes some functionality from run_test() as optional helper methods;
* Adds a SkipTest exception that can be used to flag stand-alone tests as being skipped;
* Updates the packaging guide section on stand-alone tests to provide more examples;
* Restores the ability to run tests "inherited" from provided virtual packages;
* Prints the test log path (like we currently do for build log paths);
* Times and reports the post-install process (since it can include post-install tests);
* Corrects context-related error message to distinguish test recipes from build recipes.

8 files changed, 71 insertions, 15 deletions

diff --git a/var/spack/repos/builtin.mock/packages/fail-test-audit/package.py b/var/spack/repos/builtin.mock/packages/fail-test-audit/package.py
index 33c393bfee..3869d4ac68 100644
--- a/var/spack/repos/builtin.mock/packages/fail-test-audit/package.py
+++ b/var/spack/repos/builtin.mock/packages/fail-test-audit/package.py
@@ -14,8 +14,8 @@ class FailTestAudit(MakefilePackage):
     version("1.0", md5="0123456789abcdef0123456789abcdef")
     version("2.0", md5="abcdef0123456789abcdef0123456789")
 
-    build_time_test_callbacks = ["test"]
+    build_time_test_callbacks = ["test_build_callbacks"]
 
-    def test(self):
-        print("test: test-install-callbacks")
-        print("PASSED")
+    def test_build_callbacks(self):
+        """test build time test callbacks"""
+        print("test-build-callbacks")
diff --git a/var/spack/repos/builtin.mock/packages/mpi/package.py b/var/spack/repos/builtin.mock/packages/mpi/package.py
new file mode 100644
index 0000000000..2568bfef2d
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/mpi/package.py
@@ -0,0 +1,16 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Mpi(Package):
+    """Virtual package for the Message Passing Interface."""
+
+    homepage = "https://www.mpi-forum.org/"
+    virtual = True
+
+    def test_hello(self):
+        print("Hello there!")
diff --git a/var/spack/repos/builtin.mock/packages/mpich/package.py b/var/spack/repos/builtin.mock/packages/mpich/package.py
index 1271419229..3bfe41ab6a 100644
--- a/var/spack/repos/builtin.mock/packages/mpich/package.py
+++ b/var/spack/repos/builtin.mock/packages/mpich/package.py
@@ -28,3 +28,6 @@ class Mpich(Package):
 
     def install(self, spec, prefix):
         touch(prefix.mpich)
+
+    def test_mpich(self):
+        print("Testing mpich")
diff --git a/var/spack/repos/builtin.mock/packages/printing-package/package.py b/var/spack/repos/builtin.mock/packages/printing-package/package.py
index 6357c72bd6..9d41e0a3a6 100644
--- a/var/spack/repos/builtin.mock/packages/printing-package/package.py
+++ b/var/spack/repos/builtin.mock/packages/printing-package/package.py
@@ -26,7 +26,8 @@ class PrintingPackage(Package):
 
         print("AFTER INSTALL")
 
-    def test(self):
-        print("BEFORE TEST")
-        self.run_test("true")  # run /bin/true
-        print("AFTER TEST")
+    def test_print(self):
+        """Test print example."""
+
+        print("Running test_print")
+        print("And a second command")
diff --git a/var/spack/repos/builtin.mock/packages/py-test-callback/package.py b/var/spack/repos/builtin.mock/packages/py-test-callback/package.py
new file mode 100644
index 0000000000..26e0e09997
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/py-test-callback/package.py
@@ -0,0 +1,29 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import spack.pkg.builtin.mock.python as mp
+from spack.package import *
+
+
+class PyTestCallback(mp.Python):
+    """A package for testing stand-alone test methods as a callback."""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/test-callback-1.0.tar.gz"
+
+    # TODO (post-34236): "test" -> "test_callback" once remove "test" support
+    install_time_test_callbacks = ["test"]
+
+    version("1.0", "00000000000000000000000000000110")
+    version("2.0", "00000000000000000000000000000120")
+
+    def install(self, spec, prefix):
+        mkdirp(prefix.bin)
+
+    # TODO (post-34236): "test" -> "test_callback" once remove "test" support
+    def test(self):
+        super(PyTestCallback, self).test()
+
+        print("PyTestCallback test")
diff --git a/var/spack/repos/builtin.mock/packages/simple-standalone-test/package.py b/var/spack/repos/builtin.mock/packages/simple-standalone-test/package.py
index 3cb78cd25e..9c65773aa5 100644
--- a/var/spack/repos/builtin.mock/packages/simple-standalone-test/package.py
+++ b/var/spack/repos/builtin.mock/packages/simple-standalone-test/package.py
@@ -14,6 +14,9 @@ class SimpleStandaloneTest(Package):
 
     version("1.0", md5="0123456789abcdef0123456789abcdef")
 
-    def test(self):
-        msg = "simple stand-alone test"
-        self.run_test("echo", [msg], expected=[msg], purpose="test: running {0}".format(msg))
+    provides("standalone-test")
+
+    def test_echo(self):
+        """simple stand-alone test"""
+        echo = which("echo")
+        echo("testing echo", output=str.split, error=str.split)
diff --git a/var/spack/repos/builtin.mock/packages/test-error/package.py b/var/spack/repos/builtin.mock/packages/test-error/package.py
index 5128a265a4..b8f37b4719 100644
--- a/var/spack/repos/builtin.mock/packages/test-error/package.py
+++ b/var/spack/repos/builtin.mock/packages/test-error/package.py
@@ -17,5 +17,7 @@ class TestError(Package):
     def install(self, spec, prefix):
         mkdirp(prefix.bin)
 
-    def test(self):
-        self.run_test("false")
+    def test_false(self):
+        """TestError test"""
+        false = which("false")
+        false()
diff --git a/var/spack/repos/builtin.mock/packages/test-fail/package.py b/var/spack/repos/builtin.mock/packages/test-fail/package.py
index bcaa038e73..6f0416498b 100644
--- a/var/spack/repos/builtin.mock/packages/test-fail/package.py
+++ b/var/spack/repos/builtin.mock/packages/test-fail/package.py
@@ -17,5 +17,7 @@ class TestFail(Package):
     def install(self, spec, prefix):
         mkdirp(prefix.bin)
 
-    def test(self):
-        self.run_test("true", expected=["not in the output"])
+    def test_fails(self):
+        """trigger test failure"""
+        unknown = which("unknown-program")
+        unknown()