Stand-alone/Smoke tests: copy cached test sources to test stage (#23713)

8 files changed, 104 insertions, 26 deletions

diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst
index 4d6b96a85d..b9cd85ed82 100644
--- a/lib/spack/docs/packaging_guide.rst
+++ b/lib/spack/docs/packaging_guide.rst
@@ -4293,19 +4293,37 @@ can be implemented as shown below.
 
 In this case, the method copies the associated files from the build
 stage **after** the software is installed to the package's metadata
-directory. The result is the following directory and files will be
-available for use in stand-alone tests:
+directory. The result is the directory and files will be cached in
+paths under ``self.install_test_root`` as follows:
 
-* ``join_path(self.install_test_root, 'tests')`` along with its files and subdirectories
+* ``join_path(self.install_test_root, 'tests')`` along with its files
+  and subdirectories
 * ``join_path(self.install_test_root, 'examples', 'foo.c')``
 * ``join_path(self.install_test_root, 'examples', 'bar.c')``
 
+These paths are **automatically copied** to the test stage directory
+where they are available to the package's ``test`` method through the
+``self.test_suite.current_test_cache_dir`` property. In our example,
+the method can access the directory and files using the following
+paths:
+
+* ``join_path(self.test_suite.current_test_cache_dir, 'tests')``
+* ``join_path(self.test_suite.current_test_cache_dir, 'examples', 'foo.c')``
+* ``join_path(self.test_suite.current_test_cache_dir, 'examples', 'bar.c')``
+
+.. note::
+
+    Library developers will want to build the associated tests under
+    the ``self.test_suite.current_test_cache_dir`` and against their
+    **installed** libraries before running them.
+
 .. note::
 
     While source and input files are generally recommended, binaries
     **may** also be cached by the build process for install testing.
     Only you, as the package writer or maintainer, know whether these
-    would be appropriate stand-alone tests.
+    would be appropriate for ensuring the installed software continues
+    to work as the underlying system evolves.
 
 .. note::
 
@@ -4327,11 +4345,12 @@ Examples include:
 - expected test output
 
 These extra files should be added to the ``test`` subdirectory of the
-package in the Spack repository. Spack will automatically copy any files
-in that directory to the test staging directory during stand-alone testing.
+package in the Spack repository. 
 
-The ``test`` method can access those files from the
-``self.test_suite.current_test_data_dir`` directory.
+Spack will **automatically copy** the contents of that directory to the
+test staging directory for stand-alone testing. The ``test`` method can
+access those files using the ``self.test_suite.current_test_data_dir``
+property.
 
 .. _expected_test_output_from_file:
 
@@ -4530,13 +4549,17 @@ where each argument has the following meaning:
   
   The default of ``None`` corresponds to the current directory (``'.'``).
 
+"""""""""""""""""""""""""""""""""""""""""
+Accessing package- and test-related files
+"""""""""""""""""""""""""""""""""""""""""
+
 You may need to access files from one or more locations when writing
-the tests. This can happen if the software's repository does not
+stand-alone tests. This can happen if the software's repository does not
 include test source files or includes files but no way to build the
 executables using the installed headers and libraries. In these
 cases, you may need to reference the files relative to one or more
-root directory and associated package property. These are given in
-the table below.
+root directory. The properties containing package- and test-related
+directory paths are provided in the table below.
 
 .. list-table:: Directory-to-property mapping
    :header-rows: 1
@@ -4550,10 +4573,16 @@ the table below.
    * - Package Dependency's Files
      - ``self.spec['<dependency-package>'].prefix``
      - ``self.spec['trilinos'].prefix.include``
-   * - Copied Build-time Files
+   * - Test Suite Stage Files
+     - ``self.test_suite.stage``
+     - ``join_path(self.test_suite.stage, 'results.txt')``
+   * - Cached Build-time Files
      - ``self.install_test_root``
      - ``join_path(self.install_test_root, 'examples', 'foo.c')``
-   * - Custom Package Files
+   * - Staged Cached Build-time Files
+     - ``self.test_suite.current_test_cache_dir``
+     - ``join_path(self.test_suite.current_test_cache_dir, 'examples', 'foo.c')``
+   * - Staged Custom Package Files
      - ``self.test_suite.current_test_data_dir``
      - ``join_path(self.test_suite.current_test_data_dir, 'hello.f90')``
 
diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py
index ae50a49215..cf832e9f4d 100644
--- a/lib/spack/spack/install_test.py
+++ b/lib/spack/spack/install_test.py
@@ -188,6 +188,13 @@ class TestSuite(object):
         return self.stage.join(self.test_pkg_id(spec))
 
     @property
+    def current_test_cache_dir(self):
+        assert self.current_test_spec and self.current_base_spec
+        test_spec = self.current_test_spec
+        base_spec = self.current_base_spec
+        return self.test_dir_for_spec(base_spec).cache.join(test_spec.name)
+
+    @property
     def current_test_data_dir(self):
         assert self.current_test_spec and self.current_base_spec
         test_spec = self.current_test_spec
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index b24f6bf113..169958f704 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -2603,6 +2603,14 @@ def test_process(pkg, kwargs):
                     except spack.repo.UnknownPackageError:
                         continue
 
+                    # copy installed test sources cache into test cache dir
+                    if spec.concrete:
+                        cache_source = spec_pkg.install_test_root
+                        cache_dir = pkg.test_suite.current_test_cache_dir
+                        if (os.path.isdir(cache_source) and
+                                not os.path.exists(cache_dir)):
+                            fsys.install_tree(cache_source, cache_dir)
+
                     # copy test data into test data dir
                     data_source = Prefix(spec_pkg.package_dir).test
                     data_dir = pkg.test_suite.current_test_data_dir
diff --git a/lib/spack/spack/test/cmd/mark.py b/lib/spack/spack/test/cmd/mark.py
index 450bc1f30e..f51937071b 100644
--- a/lib/spack/spack/test/cmd/mark.py
+++ b/lib/spack/spack/test/cmd/mark.py
@@ -30,7 +30,7 @@ def test_mark_all_explicit(mutable_database):
     mark('-e', '-a')
     gc('-y')
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 14
+    assert len(all_specs) == 15
 
 
 @pytest.mark.db
@@ -47,7 +47,7 @@ def test_mark_one_explicit(mutable_database):
     uninstall('-y', '-a', 'mpileaks')
     gc('-y')
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 2
+    assert len(all_specs) == 3
 
 
 @pytest.mark.db
@@ -55,7 +55,7 @@ def test_mark_one_implicit(mutable_database):
     mark('-i', 'externaltest')
     gc('-y')
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 13
+    assert len(all_specs) == 14
 
 
 @pytest.mark.db
@@ -64,4 +64,4 @@ def test_mark_all_implicit_then_explicit(mutable_database):
     mark('-e', '-a')
     gc('-y')
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 14
+    assert len(all_specs) == 15
diff --git a/lib/spack/spack/test/cmd/uninstall.py b/lib/spack/spack/test/cmd/uninstall.py
index 9a5d1587f0..916b8adfab 100644
--- a/lib/spack/spack/test/cmd/uninstall.py
+++ b/lib/spack/spack/test/cmd/uninstall.py
@@ -42,7 +42,7 @@ def test_recursive_uninstall(mutable_database):
     uninstall('-y', '-a', '--dependents', 'callpath')
 
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 8
+    assert len(all_specs) == 9
     # query specs with multiple configurations
     mpileaks_specs = [s for s in all_specs if s.satisfies('mpileaks')]
     callpath_specs = [s for s in all_specs if s.satisfies('callpath')]
@@ -56,7 +56,7 @@ def test_recursive_uninstall(mutable_database):
 @pytest.mark.db
 @pytest.mark.regression('3690')
 @pytest.mark.parametrize('constraint,expected_number_of_specs', [
-    ('dyninst', 7), ('libelf', 5)
+    ('dyninst', 8), ('libelf', 6)
 ])
 def test_uninstall_spec_with_multiple_roots(
         constraint, expected_number_of_specs, mutable_database
@@ -69,7 +69,7 @@ def test_uninstall_spec_with_multiple_roots(
 
 @pytest.mark.db
 @pytest.mark.parametrize('constraint,expected_number_of_specs', [
-    ('dyninst', 13), ('libelf', 13)
+    ('dyninst', 14), ('libelf', 14)
 ])
 def test_force_uninstall_spec_with_ref_count_not_zero(
         constraint, expected_number_of_specs, mutable_database
@@ -141,7 +141,8 @@ def test_force_uninstall_and_reinstall_by_hash(mutable_database):
             [s for s in all_specs if s.satisfies('mpi')]
         )
     all_specs, mpileaks_specs, callpath_specs, mpi_specs = db_specs()
-    assert len(all_specs) == 13
+    total_specs = len(all_specs)
+    assert total_specs == 14
     assert len(mpileaks_specs) == 3
     assert len(callpath_specs) == 2
     assert len(mpi_specs) == 3
@@ -152,7 +153,7 @@ def test_force_uninstall_and_reinstall_by_hash(mutable_database):
     validate_callpath_spec(True)
 
     all_specs, mpileaks_specs, callpath_specs, mpi_specs = db_specs()
-    assert len(all_specs) == 14      # back to 14
+    assert len(all_specs) == total_specs + 1   # back to total_specs+1
     assert len(mpileaks_specs) == 3
     assert len(callpath_specs) == 3  # back to 3
     assert len(mpi_specs) == 3
diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py
index 6cc3ad16db..38a6b3fd5b 100644
--- a/lib/spack/spack/test/conftest.py
+++ b/lib/spack/spack/test/conftest.py
@@ -567,6 +567,7 @@ def _populate(mock_db):
     _install('mpileaks ^mpich2')
     _install('mpileaks ^zmpi')
     _install('externaltest')
+    _install('trivial-smoke-test')
 
 
 @pytest.fixture(scope='session')
diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py
index cb9fbada2a..c8c12eac39 100644
--- a/lib/spack/spack/test/database.py
+++ b/lib/spack/spack/test/database.py
@@ -424,7 +424,7 @@ def test_005_db_exists(database):
 def test_010_all_install_sanity(database):
     """Ensure that the install layout reflects what we think it does."""
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 14
+    assert len(all_specs) == 15
 
     # Query specs with multiple configurations
     mpileaks_specs = [s for s in all_specs if s.satisfies('mpileaks')]
@@ -545,7 +545,8 @@ def test_041_ref_counts_deprecate(mutable_database):
 def test_050_basic_query(database):
     """Ensure querying database is consistent with what is installed."""
     # query everything
-    assert len(spack.store.db.query()) == 16
+    total_specs = len(spack.store.db.query())
+    assert total_specs == 17
 
     # query specs with multiple configurations
     mpileaks_specs = database.query('mpileaks')
@@ -571,10 +572,10 @@ def test_050_basic_query(database):
     assert len(database.query('mpileaks ^zmpi')) == 1
 
     # Query by date
-    assert len(database.query(start_date=datetime.datetime.min)) == 16
+    assert len(database.query(start_date=datetime.datetime.min)) == total_specs
     assert len(database.query(start_date=datetime.datetime.max)) == 0
     assert len(database.query(end_date=datetime.datetime.min)) == 0
-    assert len(database.query(end_date=datetime.datetime.max)) == 16
+    assert len(database.query(end_date=datetime.datetime.max)) == total_specs
 
 
 def test_060_remove_and_add_root_package(mutable_database):
diff --git a/lib/spack/spack/test/test_suite.py b/lib/spack/spack/test/test_suite.py
index 50c5989a3d..4c16b2d2f6 100644
--- a/lib/spack/spack/test/test_suite.py
+++ b/lib/spack/spack/test/test_suite.py
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os
+
+import llnl.util.filesystem as fs
 import spack.install_test
 import spack.spec
 
@@ -51,3 +53,32 @@ def test_write_test_result(mock_packages, mock_test_stage):
         msg = lines[0]
         assert result in msg
         assert spec.name in msg
+
+
+def test_do_test(mock_packages, mock_test_stage, install_mockery):
+    """Perform a stand-alone test with files to copy."""
+    spec = spack.spec.Spec('trivial-smoke-test').concretized()
+    test_name = 'test_do_test'
+    test_filename = 'test_file.in'
+
+    pkg = spec.package
+    pkg.create_extra_test_source()
+
+    test_suite = spack.install_test.TestSuite([spec], test_name)
+    test_suite.current_test_spec = spec
+    test_suite.current_base_spec = spec
+    test_suite.ensure_stage()
+
+    # Save off target paths for current spec since test suite processing
+    # assumes testing multiple specs.
+    cached_filename = fs.join_path(test_suite.current_test_cache_dir,
+                                   pkg.test_source_filename)
+    data_filename = fs.join_path(test_suite.current_test_data_dir,
+                                 test_filename)
+
+    # Run the test, making sure to retain the test stage directory
+    # so we can ensure the files were copied.
+    test_suite(remove_directory=False)
+
+    assert os.path.exists(cached_filename)
+    assert os.path.exists(data_filename)