Stand-alone/Smoke tests: copy cached test sources to test stage (#23713)

12 files changed, 176 insertions, 73 deletions

diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst
index 4d6b96a85d..b9cd85ed82 100644
--- a/lib/spack/docs/packaging_guide.rst
+++ b/lib/spack/docs/packaging_guide.rst
@@ -4293,19 +4293,37 @@ can be implemented as shown below.
 
 In this case, the method copies the associated files from the build
 stage **after** the software is installed to the package's metadata
-directory. The result is the following directory and files will be
-available for use in stand-alone tests:
+directory. The result is the directory and files will be cached in
+paths under ``self.install_test_root`` as follows:
 
-* ``join_path(self.install_test_root, 'tests')`` along with its files and subdirectories
+* ``join_path(self.install_test_root, 'tests')`` along with its files
+  and subdirectories
 * ``join_path(self.install_test_root, 'examples', 'foo.c')``
 * ``join_path(self.install_test_root, 'examples', 'bar.c')``
 
+These paths are **automatically copied** to the test stage directory
+where they are available to the package's ``test`` method through the
+``self.test_suite.current_test_cache_dir`` property. In our example,
+the method can access the directory and files using the following
+paths:
+
+* ``join_path(self.test_suite.current_test_cache_dir, 'tests')``
+* ``join_path(self.test_suite.current_test_cache_dir, 'examples', 'foo.c')``
+* ``join_path(self.test_suite.current_test_cache_dir, 'examples', 'bar.c')``
+
+.. note::
+
+    Library developers will want to build the associated tests under
+    the ``self.test_suite.current_test_cache_dir`` and against their
+    **installed** libraries before running them.
+
 .. note::
 
     While source and input files are generally recommended, binaries
     **may** also be cached by the build process for install testing.
     Only you, as the package writer or maintainer, know whether these
-    would be appropriate stand-alone tests.
+    would be appropriate for ensuring the installed software continues
+    to work as the underlying system evolves.
 
 .. note::
 
@@ -4327,11 +4345,12 @@ Examples include:
 - expected test output
 
 These extra files should be added to the ``test`` subdirectory of the
-package in the Spack repository. Spack will automatically copy any files
-in that directory to the test staging directory during stand-alone testing.
+package in the Spack repository. 
 
-The ``test`` method can access those files from the
-``self.test_suite.current_test_data_dir`` directory.
+Spack will **automatically copy** the contents of that directory to the
+test staging directory for stand-alone testing. The ``test`` method can
+access those files using the ``self.test_suite.current_test_data_dir``
+property.
 
 .. _expected_test_output_from_file:
 
@@ -4530,13 +4549,17 @@ where each argument has the following meaning:
   
   The default of ``None`` corresponds to the current directory (``'.'``).
 
+"""""""""""""""""""""""""""""""""""""""""
+Accessing package- and test-related files
+"""""""""""""""""""""""""""""""""""""""""
+
 You may need to access files from one or more locations when writing
-the tests. This can happen if the software's repository does not
+stand-alone tests. This can happen if the software's repository does not
 include test source files or includes files but no way to build the
 executables using the installed headers and libraries. In these
 cases, you may need to reference the files relative to one or more
-root directory and associated package property. These are given in
-the table below.
+root directory. The properties containing package- and test-related
+directory paths are provided in the table below.
 
 .. list-table:: Directory-to-property mapping
    :header-rows: 1
@@ -4550,10 +4573,16 @@ the table below.
    * - Package Dependency's Files
      - ``self.spec['<dependency-package>'].prefix``
      - ``self.spec['trilinos'].prefix.include``
-   * - Copied Build-time Files
+   * - Test Suite Stage Files
+     - ``self.test_suite.stage``
+     - ``join_path(self.test_suite.stage, 'results.txt')``
+   * - Cached Build-time Files
      - ``self.install_test_root``
      - ``join_path(self.install_test_root, 'examples', 'foo.c')``
-   * - Custom Package Files
+   * - Staged Cached Build-time Files
+     - ``self.test_suite.current_test_cache_dir``
+     - ``join_path(self.test_suite.current_test_cache_dir, 'examples', 'foo.c')``
+   * - Staged Custom Package Files
      - ``self.test_suite.current_test_data_dir``
      - ``join_path(self.test_suite.current_test_data_dir, 'hello.f90')``
 
diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py
index ae50a49215..cf832e9f4d 100644
--- a/lib/spack/spack/install_test.py
+++ b/lib/spack/spack/install_test.py
@@ -188,6 +188,13 @@ class TestSuite(object):
         return self.stage.join(self.test_pkg_id(spec))
 
     @property
+    def current_test_cache_dir(self):
+        assert self.current_test_spec and self.current_base_spec
+        test_spec = self.current_test_spec
+        base_spec = self.current_base_spec
+        return self.test_dir_for_spec(base_spec).cache.join(test_spec.name)
+
+    @property
     def current_test_data_dir(self):
         assert self.current_test_spec and self.current_base_spec
         test_spec = self.current_test_spec
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index b24f6bf113..169958f704 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -2603,6 +2603,14 @@ def test_process(pkg, kwargs):
                     except spack.repo.UnknownPackageError:
                         continue
 
+                    # copy installed test sources cache into test cache dir
+                    if spec.concrete:
+                        cache_source = spec_pkg.install_test_root
+                        cache_dir = pkg.test_suite.current_test_cache_dir
+                        if (os.path.isdir(cache_source) and
+                                not os.path.exists(cache_dir)):
+                            fsys.install_tree(cache_source, cache_dir)
+
                     # copy test data into test data dir
                     data_source = Prefix(spec_pkg.package_dir).test
                     data_dir = pkg.test_suite.current_test_data_dir
diff --git a/lib/spack/spack/test/cmd/mark.py b/lib/spack/spack/test/cmd/mark.py
index 450bc1f30e..f51937071b 100644
--- a/lib/spack/spack/test/cmd/mark.py
+++ b/lib/spack/spack/test/cmd/mark.py
@@ -30,7 +30,7 @@ def test_mark_all_explicit(mutable_database):
     mark('-e', '-a')
     gc('-y')
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 14
+    assert len(all_specs) == 15
 
 
 @pytest.mark.db
@@ -47,7 +47,7 @@ def test_mark_one_explicit(mutable_database):
     uninstall('-y', '-a', 'mpileaks')
     gc('-y')
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 2
+    assert len(all_specs) == 3
 
 
 @pytest.mark.db
@@ -55,7 +55,7 @@ def test_mark_one_implicit(mutable_database):
     mark('-i', 'externaltest')
     gc('-y')
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 13
+    assert len(all_specs) == 14
 
 
 @pytest.mark.db
@@ -64,4 +64,4 @@ def test_mark_all_implicit_then_explicit(mutable_database):
     mark('-e', '-a')
     gc('-y')
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 14
+    assert len(all_specs) == 15
diff --git a/lib/spack/spack/test/cmd/uninstall.py b/lib/spack/spack/test/cmd/uninstall.py
index 9a5d1587f0..916b8adfab 100644
--- a/lib/spack/spack/test/cmd/uninstall.py
+++ b/lib/spack/spack/test/cmd/uninstall.py
@@ -42,7 +42,7 @@ def test_recursive_uninstall(mutable_database):
     uninstall('-y', '-a', '--dependents', 'callpath')
 
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 8
+    assert len(all_specs) == 9
     # query specs with multiple configurations
     mpileaks_specs = [s for s in all_specs if s.satisfies('mpileaks')]
     callpath_specs = [s for s in all_specs if s.satisfies('callpath')]
@@ -56,7 +56,7 @@ def test_recursive_uninstall(mutable_database):
 @pytest.mark.db
 @pytest.mark.regression('3690')
 @pytest.mark.parametrize('constraint,expected_number_of_specs', [
-    ('dyninst', 7), ('libelf', 5)
+    ('dyninst', 8), ('libelf', 6)
 ])
 def test_uninstall_spec_with_multiple_roots(
         constraint, expected_number_of_specs, mutable_database
@@ -69,7 +69,7 @@ def test_uninstall_spec_with_multiple_roots(
 
 @pytest.mark.db
 @pytest.mark.parametrize('constraint,expected_number_of_specs', [
-    ('dyninst', 13), ('libelf', 13)
+    ('dyninst', 14), ('libelf', 14)
 ])
 def test_force_uninstall_spec_with_ref_count_not_zero(
         constraint, expected_number_of_specs, mutable_database
@@ -141,7 +141,8 @@ def test_force_uninstall_and_reinstall_by_hash(mutable_database):
             [s for s in all_specs if s.satisfies('mpi')]
         )
     all_specs, mpileaks_specs, callpath_specs, mpi_specs = db_specs()
-    assert len(all_specs) == 13
+    total_specs = len(all_specs)
+    assert total_specs == 14
     assert len(mpileaks_specs) == 3
     assert len(callpath_specs) == 2
     assert len(mpi_specs) == 3
@@ -152,7 +153,7 @@ def test_force_uninstall_and_reinstall_by_hash(mutable_database):
     validate_callpath_spec(True)
 
     all_specs, mpileaks_specs, callpath_specs, mpi_specs = db_specs()
-    assert len(all_specs) == 14      # back to 14
+    assert len(all_specs) == total_specs + 1   # back to total_specs+1
     assert len(mpileaks_specs) == 3
     assert len(callpath_specs) == 3  # back to 3
     assert len(mpi_specs) == 3
diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py
index 6cc3ad16db..38a6b3fd5b 100644
--- a/lib/spack/spack/test/conftest.py
+++ b/lib/spack/spack/test/conftest.py
@@ -567,6 +567,7 @@ def _populate(mock_db):
     _install('mpileaks ^mpich2')
     _install('mpileaks ^zmpi')
     _install('externaltest')
+    _install('trivial-smoke-test')
 
 
 @pytest.fixture(scope='session')
diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py
index cb9fbada2a..c8c12eac39 100644
--- a/lib/spack/spack/test/database.py
+++ b/lib/spack/spack/test/database.py
@@ -424,7 +424,7 @@ def test_005_db_exists(database):
 def test_010_all_install_sanity(database):
     """Ensure that the install layout reflects what we think it does."""
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 14
+    assert len(all_specs) == 15
 
     # Query specs with multiple configurations
     mpileaks_specs = [s for s in all_specs if s.satisfies('mpileaks')]
@@ -545,7 +545,8 @@ def test_041_ref_counts_deprecate(mutable_database):
 def test_050_basic_query(database):
     """Ensure querying database is consistent with what is installed."""
     # query everything
-    assert len(spack.store.db.query()) == 16
+    total_specs = len(spack.store.db.query())
+    assert total_specs == 17
 
     # query specs with multiple configurations
     mpileaks_specs = database.query('mpileaks')
@@ -571,10 +572,10 @@ def test_050_basic_query(database):
     assert len(database.query('mpileaks ^zmpi')) == 1
 
     # Query by date
-    assert len(database.query(start_date=datetime.datetime.min)) == 16
+    assert len(database.query(start_date=datetime.datetime.min)) == total_specs
     assert len(database.query(start_date=datetime.datetime.max)) == 0
     assert len(database.query(end_date=datetime.datetime.min)) == 0
-    assert len(database.query(end_date=datetime.datetime.max)) == 16
+    assert len(database.query(end_date=datetime.datetime.max)) == total_specs
 
 
 def test_060_remove_and_add_root_package(mutable_database):
diff --git a/lib/spack/spack/test/test_suite.py b/lib/spack/spack/test/test_suite.py
index 50c5989a3d..4c16b2d2f6 100644
--- a/lib/spack/spack/test/test_suite.py
+++ b/lib/spack/spack/test/test_suite.py
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os
+
+import llnl.util.filesystem as fs
 import spack.install_test
 import spack.spec
 
@@ -51,3 +53,32 @@ def test_write_test_result(mock_packages, mock_test_stage):
         msg = lines[0]
         assert result in msg
         assert spec.name in msg
+
+
+def test_do_test(mock_packages, mock_test_stage, install_mockery):
+    """Perform a stand-alone test with files to copy."""
+    spec = spack.spec.Spec('trivial-smoke-test').concretized()
+    test_name = 'test_do_test'
+    test_filename = 'test_file.in'
+
+    pkg = spec.package
+    pkg.create_extra_test_source()
+
+    test_suite = spack.install_test.TestSuite([spec], test_name)
+    test_suite.current_test_spec = spec
+    test_suite.current_base_spec = spec
+    test_suite.ensure_stage()
+
+    # Save off target paths for current spec since test suite processing
+    # assumes testing multiple specs.
+    cached_filename = fs.join_path(test_suite.current_test_cache_dir,
+                                   pkg.test_source_filename)
+    data_filename = fs.join_path(test_suite.current_test_data_dir,
+                                 test_filename)
+
+    # Run the test, making sure to retain the test stage directory
+    # so we can ensure the files were copied.
+    test_suite(remove_directory=False)
+
+    assert os.path.exists(cached_filename)
+    assert os.path.exists(data_filename)
diff --git a/var/spack/repos/builtin.mock/packages/trivial-smoke-test/package.py b/var/spack/repos/builtin.mock/packages/trivial-smoke-test/package.py
new file mode 100644
index 0000000000..6dbdd07ea8
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/trivial-smoke-test/package.py
@@ -0,0 +1,25 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class TrivialSmokeTest(Package):
+    """This package is a stub with trivial smoke test features."""
+    homepage = "http://www.example.com/trivial_test"
+    url      = "http://www.unit-test-should-replace-this-url/trivial_test-1.0.tar.gz"
+
+    version('1.0', 'foobarbaz')
+
+    test_source_filename = 'cached_file.in'
+
+    @run_before('install')
+    def create_extra_test_source(self):
+        mkdirp(self.install_test_root)
+        touch(join_path(self.install_test_root, self.test_source_filename))
+
+    @run_after('install')
+    def copy_test_sources(self):
+        self.cache_extra_test_sources([self.test_source_filename])
diff --git a/var/spack/repos/builtin.mock/packages/trivial-smoke-test/test/test_file.in b/var/spack/repos/builtin.mock/packages/trivial-smoke-test/test/test_file.in
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/trivial-smoke-test/test/test_file.in
diff --git a/var/spack/repos/builtin/packages/hypre/package.py b/var/spack/repos/builtin/packages/hypre/package.py
index 43213507c4..3b8c6cd3c1 100644
--- a/var/spack/repos/builtin/packages/hypre/package.py
+++ b/var/spack/repos/builtin/packages/hypre/package.py
@@ -220,26 +220,36 @@ class Hypre(Package, CudaPackage):
                         '-rhsone')
             make("install")
 
+    extra_install_tests = join_path('src', 'examples')
+
     @run_after('install')
     def cache_test_sources(self):
-        srcs = ['src/examples']
-        self.cache_extra_test_sources(srcs)
+        self.cache_extra_test_sources(self.extra_install_tests)
+
+    @property
+    def _cached_tests_work_dir(self):
+        """The working directory for cached test sources."""
+        return join_path(self.test_suite.current_test_cache_dir,
+                         self.extra_install_tests)
 
     def test(self):
         """Perform smoke test on installed HYPRE package."""
-
-        if '+mpi' in self.spec:
-            examples_dir = join_path(self.install_test_root, 'src/examples')
-            with working_dir(examples_dir, create=False):
-                make("HYPRE_DIR=" + self.prefix, "bigint")
-
-                reason = "test: ensuring HYPRE examples run"
-                self.run_test('./ex5big', [], [], installed=True,
-                              purpose=reason, skip_missing=True, work_dir='.')
-                self.run_test('./ex15big', [], [], installed=True,
-                              purpose=reason, skip_missing=True, work_dir='.')
-
-                make("distclean")
+        if '+mpi' not in self.spec:
+            print('Skipping: HYPRE must be installed with +mpi to run tests')
+            return
+
+        # Build copied and cached test examples
+        self.run_test('make',
+                      ['HYPRE_DIR={0}'.format(self.prefix), 'bigint'],
+                      purpose='test: building selected examples',
+                      work_dir=self._cached_tests_work_dir)
+
+        # Run the examples built above
+        for exe in ['./ex5big', './ex15big']:
+            self.run_test(exe, [], [], installed=False,
+                          purpose='test: ensuring {0} runs'.format(exe),
+                          skip_missing=True,
+                          work_dir=self._cached_tests_work_dir)
 
     @property
     def headers(self):
diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py
index 697171a02a..67809c7ce9 100644
--- a/var/spack/repos/builtin/packages/openmpi/package.py
+++ b/var/spack/repos/builtin/packages/openmpi/package.py
@@ -891,9 +891,10 @@ class Openmpi(AutotoolsPackage):
             'shmemrun': ls,
         }
 
-        for exe in checks:
-            options, expected, status = checks[exe]
-            reason = 'test: checking {0} output'.format(exe)
+        for binary in checks:
+            options, expected, status = checks[binary]
+            exe = join_path(self.prefix.bin, binary)
+            reason = 'test: checking {0} output'.format(binary)
             self.run_test(exe, options, expected, status, installed=True,
                           purpose=reason, skip_missing=True)
 
@@ -942,36 +943,28 @@ class Openmpi(AutotoolsPackage):
             'shmemcxx': comp_vers,
         }
 
-        for exe in checks:
-            expected = checks[exe]
+        for binary in checks:
+            expected = checks[binary]
             purpose = 'test: ensuring version of {0} is {1}' \
-                .format(exe, expected)
+                .format(binary, expected)
+            exe = join_path(self.prefix.bin, binary)
             self.run_test(exe, '--version', expected, installed=True,
                           purpose=purpose, skip_missing=True)
 
-    def _test_build_examples(self):
-        # Build the examples copied during installation and return "status"
-        reason = 'test: ensuring ability to build the examples'
-        return self.run_test('make', ['all'], [],
-                             purpose=reason,
-                             work_dir=join_path(self.install_test_root,
-                                                self.extra_install_tests))
-
-    def _test_clean_examples(self):
-        # Clean up any example build files
-        reason = 'test: ensuring copied examples cleaned up'
-        return self.run_test('make', ['clean'], [],
-                             purpose=reason,
-                             work_dir=join_path(self.install_test_root,
-                                                self.extra_install_tests))
+    @property
+    def _cached_tests_work_dir(self):
+        """The working directory for cached test sources."""
+        return join_path(self.test_suite.current_test_cache_dir,
+                         self.extra_install_tests)
 
     def _test_examples(self):
-        # First ensure can build copied examples
-        if not self._test_build_examples():
-            self._test_clean_examples()
-            return
+        """Run test examples copied from source at build-time."""
+        # Build the copied, cached test examples
+        self.run_test('make', ['all'], [],
+                      purpose='test: building cached test examples',
+                      work_dir=self._cached_tests_work_dir)
 
-        # Now run examples with known, simple-to-verify results
+        # Run examples with known, simple-to-verify results
         have_spml = self.spec.satisfies('@2.0.0:2.1.6')
 
         hello_world = (['Hello, world', 'I am', '0 of', '1'], 0)
@@ -1015,14 +1008,11 @@ class Openmpi(AutotoolsPackage):
                 .format(exe, status)
             self.run_test(exe, [], expected, status, installed=False,
                           purpose=reason, skip_missing=True,
-                          work_dir=join_path(self.install_test_root,
-                                             self.extra_install_tests))
-
-        self._test_clean_examples()
+                          work_dir=self._cached_tests_work_dir)
 
     def test(self):
-        """Perform smoke tests on the installed package."""
-        # Simple version check tests on known packages
+        """Perform stand-alone/smoke tests on the installed package."""
+        # Simple version check tests on selected installed binaries
         self._test_check_versions()
 
         # Test the operation of selected executables