Testing: optionally run tests on externally installed packages (#28701)

Since Spack does not install external packages, this commit skips them by
default when running stand-alone tests. The assumption is that such packages
have likely undergone an acceptance test process. 

However, the tests can be run against installed externals using 
```
% spack test run --externals ...
```

5 files changed, 64 insertions, 8 deletions

diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py
index 01dac8d39e..ddb4990e4c 100644
--- a/lib/spack/spack/cmd/test.py
+++ b/lib/spack/spack/cmd/test.py
@@ -54,6 +54,10 @@ def setup_parser(subparser):
         help="Stop after the first failed package."
     )
     run_parser.add_argument(
+        '--externals', action='store_true',
+        help="Test packages that are externally installed."
+    )
+    run_parser.add_argument(
         '--keep-stage',
         action='store_true',
         help='Keep testing directory for debugging'
@@ -203,7 +207,8 @@ environment variables:
     with reporter('test', test_suite.stage):
         test_suite(remove_directory=not args.keep_stage,
                    dirty=args.dirty,
-                   fail_first=args.fail_first)
+                   fail_first=args.fail_first,
+                   externals=args.externals)
 
 
 def test_list(args):
diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py
index 660135422d..1857f26e6d 100644
--- a/lib/spack/spack/install_test.py
+++ b/lib/spack/spack/install_test.py
@@ -128,6 +128,7 @@ class TestSuite(object):
         remove_directory = kwargs.get('remove_directory', True)
         dirty = kwargs.get('dirty', False)
         fail_first = kwargs.get('fail_first', False)
+        externals = kwargs.get('externals', False)
 
         for spec in self.specs:
             try:
@@ -149,9 +150,7 @@ class TestSuite(object):
                 fs.mkdirp(test_dir)
 
                 # run the package tests
-                spec.package.do_test(
-                    dirty=dirty
-                )
+                spec.package.do_test(dirty=dirty, externals=externals)
 
                 # Clean up on success
                 if remove_directory:
@@ -160,7 +159,18 @@ class TestSuite(object):
                 # Log test status based on whether any non-pass-only test
                 # functions were called
                 tested = os.path.exists(self.tested_file_for_spec(spec))
-                status = 'PASSED' if tested else 'NO-TESTS'
+                if tested:
+                    status = 'PASSED'
+                else:
+                    self.ensure_stage()
+                    if spec.external and not externals:
+                        status = 'SKIPPED'
+                        msg = 'Skipped external package'
+                    else:
+                        status = 'NO-TESTS'
+                        msg = 'No tests to run'
+                    _add_msg_to_file(self.log_file_for_spec(spec), msg)
+
                 self.write_test_result(spec, status)
             except BaseException as exc:
                 self.fails += 1
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 447e7a9d46..c10caf89fc 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -1796,7 +1796,7 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)):
                 fsys.mkdirp(os.path.dirname(dest_path))
                 fsys.copy(src_path, dest_path)
 
-    def do_test(self, dirty=False):
+    def do_test(self, dirty=False, externals=False):
         if self.test_requires_compiler:
             compilers = spack.compilers.compilers_for_spec(
                 self.spec.compiler, arch_spec=self.spec.architecture)
@@ -1813,6 +1813,12 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)):
         self.tested_file = self.test_suite.tested_file_for_spec(self.spec)
         fsys.touch(self.test_log_file)  # Otherwise log_parse complains
 
+        if self.spec.external and not externals:
+            with open(self.test_log_file, 'w') as ofd:
+                ofd.write('Testing package {0}\n'
+                          .format(self.test_suite.test_pkg_id(self.spec)))
+            return
+
         kwargs = {'dirty': dirty, 'fake': False, 'context': 'test'}
         spack.build_environment.start_build_process(self, test_process, kwargs)
 
diff --git a/lib/spack/spack/report.py b/lib/spack/spack/report.py
index 084a9d3883..8697d98f5e 100644
--- a/lib/spack/spack/report.py
+++ b/lib/spack/spack/report.py
@@ -171,8 +171,15 @@ class InfoCollector(object):
                 value = None
                 try:
                     value = do_fn(instance, *args, **kwargs)
-                    package['result'] = 'success'
-                    package['stdout'] = fetch_log(pkg, do_fn, self.dir)
+
+                    externals = kwargs.get('externals', False)
+                    skip_externals = pkg.spec.external and not externals
+                    if do_fn.__name__ == 'do_test' and skip_externals:
+                        package['result'] = 'skipped'
+                        package['stdout'] = 'Skipped external package'
+                    else:
+                        package['result'] = 'success'
+                        package['stdout'] = fetch_log(pkg, do_fn, self.dir)
                     package['installed_from_binary_cache'] = \
                         pkg.installed_from_binary_cache
                     if do_fn.__name__ == '_install_task' and installed_already:
diff --git a/lib/spack/spack/test/test_suite.py b/lib/spack/spack/test/test_suite.py
index e1242cb70d..7641dfcb0b 100644
--- a/lib/spack/spack/test/test_suite.py
+++ b/lib/spack/spack/test/test_suite.py
@@ -87,6 +87,34 @@ def test_do_test(mock_packages, install_mockery, mock_test_stage):
     assert os.path.exists(data_filename)
 
 
+@pytest.mark.parametrize('arguments,status,msg', [
+    ({}, 'SKIPPED', 'Skipped'),
+    ({'externals': True}, 'NO-TESTS', 'No tests'),
+])
+def test_test_external(mock_packages, install_mockery, mock_test_stage,
+                       arguments, status, msg):
+    def ensure_results(filename, expected):
+        assert os.path.exists(filename)
+        with open(filename, 'r') as fd:
+            lines = fd.readlines()
+            have = False
+            for line in lines:
+                if expected in line:
+                    have = True
+                    break
+            assert have
+
+    name = 'trivial-smoke-test'
+    spec = spack.spec.Spec(name).concretized()
+    spec.external_path = '/path/to/external/{0}'.format(name)
+
+    test_suite = spack.install_test.TestSuite([spec])
+    test_suite(**arguments)
+
+    ensure_results(test_suite.results_file, status)
+    ensure_results(test_suite.log_file_for_spec(spec), msg)
+
+
 def test_test_stage_caches(mock_packages, install_mockery, mock_test_stage):
     def ensure_current_cache_fail(test_suite):
         with pytest.raises(spack.install_test.TestSuiteSpecError):