Add test deptype (#5132)

* Add '--test=all' and '--test=root' options to test either the root or the root and all dependencies.
* add a test dependency type that is only used when --test is enabled.
* test dependencies are not added to the spec, but they are provided in the test environment.

9 files changed, 181 insertions, 11 deletions

diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index ed3b9e22d7..3a613e1fd0 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -158,6 +158,9 @@ dirty = _config.get('dirty', False)
 build_jobs = _config.get('build_jobs', multiprocessing.cpu_count())
 
 
+package_testing = spack.package_prefs.PackageTesting()
+
+
 #-----------------------------------------------------------------------------
 # When packages call 'from spack import *', this extra stuff is brought in.
 #
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index a456efb8ca..20ee31ce71 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -191,9 +191,9 @@ def set_build_environment_variables(pkg, env, dirty):
         dirty (bool): Skip unsetting the user's environment settings
     """
     # Gather information about various types of dependencies
-    build_deps      = pkg.spec.dependencies(deptype='build')
+    build_deps      = pkg.spec.dependencies(deptype=('build', 'test'))
     link_deps       = pkg.spec.traverse(root=False, deptype=('link'))
-    build_link_deps = pkg.spec.traverse(root=False, deptype=('build', 'link'))
+    build_link_deps = list(build_deps) + list(link_deps)
     rpath_deps      = get_rpath_deps(pkg)
 
     build_prefixes      = [dep.prefix for dep in build_deps]
diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py
index d42d06d06b..d4a59c9e8f 100644
--- a/lib/spack/spack/cmd/install.py
+++ b/lib/spack/spack/cmd/install.py
@@ -97,9 +97,18 @@ the dependencies"""
         nargs=argparse.REMAINDER,
         help="spec of the package to install"
     )
-    subparser.add_argument(
+    testing = subparser.add_mutually_exclusive_group()
+    testing.add_argument(
+        '--test', default=None,
+        choices=['root', 'all'],
+        help="""If 'root' is chosen, run package tests during
+installation for top-level packages (but skip tests for dependencies).
+if 'all' is chosen, run package tests during installation for all
+packages. If neither are chosen, don't run tests for any packages."""
+    )
+    testing.add_argument(
         '--run-tests', action='store_true',
-        help="run package level tests during installation"
+        help='run package tests during installation (same as --test=all)'
     )
     subparser.add_argument(
         '--log-format',
@@ -325,12 +334,21 @@ def install(parser, args, **kwargs):
         'install_source': args.install_source,
         'install_deps': 'dependencies' in args.things_to_install,
         'make_jobs': args.jobs,
-        'run_tests': args.run_tests,
         'verbose': args.verbose,
         'fake': args.fake,
         'dirty': args.dirty
     })
 
+    if args.run_tests:
+        tty.warn("Deprecated option: --run-tests: use --test=all instead")
+
+    specs = spack.cmd.parse_specs(args.package)
+    if args.test == 'all' or args.run_tests:
+        spack.package_testing.test_all()
+    elif args.test == 'root':
+        for spec in specs:
+            spack.package_testing.test(spec.name)
+
     # Spec from cli
     specs = []
     if args.file:
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 897b5ea9bc..b70113e9f5 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -1210,7 +1210,6 @@ class PackageBase(with_metaclass(PackageMeta, object)):
                    skip_patch=False,
                    verbose=False,
                    make_jobs=None,
-                   run_tests=False,
                    fake=False,
                    explicit=False,
                    dirty=None,
@@ -1235,7 +1234,6 @@ class PackageBase(with_metaclass(PackageMeta, object)):
                 suppresses it)
             make_jobs (int): Number of make jobs to use for install. Default
                 is ncpus
-            run_tests (bool): Run tests within the package's install()
             fake (bool): Don't really build; install fake stub files instead.
             explicit (bool): True if package was explicitly installed, False
                 if package was implicitly installed (as a dependency).
@@ -1281,7 +1279,6 @@ class PackageBase(with_metaclass(PackageMeta, object)):
                     skip_patch=skip_patch,
                     verbose=verbose,
                     make_jobs=make_jobs,
-                    run_tests=run_tests,
                     dirty=dirty,
                     **kwargs
                 )
@@ -1289,7 +1286,7 @@ class PackageBase(with_metaclass(PackageMeta, object)):
         tty.msg('Installing %s' % self.name)
 
         # Set run_tests flag before starting build.
-        self.run_tests = run_tests
+        self.run_tests = spack.package_testing.check(self.name)
 
         # Set parallelism before starting build.
         self.make_jobs = make_jobs
diff --git a/lib/spack/spack/package_prefs.py b/lib/spack/spack/package_prefs.py
index a79408df43..9a3059e748 100644
--- a/lib/spack/spack/package_prefs.py
+++ b/lib/spack/spack/package_prefs.py
@@ -204,6 +204,25 @@ class PackagePrefs(object):
                     if name in pkg.variants)
 
 
+class PackageTesting(object):
+    def __init__(self):
+        self.packages_to_test = set()
+        self._test_all = False
+
+    def test(self, package_name):
+        self.packages_to_test.add(package_name)
+
+    def test_all(self):
+        self._test_all = True
+
+    def clear(self):
+        self._test_all = False
+        self.packages_to_test.clear()
+
+    def check(self, package_name):
+        return self._test_all or (package_name in self.packages_to_test)
+
+
 def spec_externals(spec):
     """Return a list of external specs (w/external directory path filled in),
        one for each known external installation."""
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 5c082d90c8..584396e82d 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -197,7 +197,7 @@ _separators = '[\\%s]' % '\\'.join(color_formats.keys())
 _any_version = VersionList([':'])
 
 #: Types of dependencies that Spack understands.
-alldeps = ('build', 'link', 'run')
+alldeps = ('build', 'link', 'run', 'test')
 
 #: Max integer helps avoid passing too large a value to cyaml.
 maxint = 2 ** (ctypes.sizeof(ctypes.c_int) * 8 - 1) - 1
@@ -2089,7 +2089,8 @@ class Spec(object):
                 pkg_dep = self._evaluate_dependency_conditions(dep_name)
                 deptypes = pkg.dependency_types[dep_name]
                 # If pkg_dep is a dependency, merge it.
-                if pkg_dep:
+                if pkg_dep and (spack.package_testing.check(self.name) or
+                                set(deptypes) - set(['test'])):
                     changed |= self._merge_dependency(
                         pkg_dep, deptypes, visited, spec_deps, provider_index)
             any_change |= changed
diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py
index 66b79642a3..cb39e62cf8 100644
--- a/lib/spack/spack/test/cmd/install.py
+++ b/lib/spack/spack/test/cmd/install.py
@@ -44,6 +44,15 @@ def parser():
     return parser
 
 
+@pytest.fixture()
+def noop_install(monkeypatch):
+
+    def noop(*args, **kwargs):
+        return
+
+    monkeypatch.setattr(spack.package.PackageBase, 'do_install', noop)
+
+
 def test_install_package_and_dependency(
         tmpdir, builtin_mock, mock_archive, mock_fetch, config,
         install_mockery):
@@ -64,6 +73,33 @@ def test_install_package_and_dependency(
     assert not spack.repo.get(s).stage.created
 
 
+@pytest.mark.usefixtures('noop_install', 'builtin_mock', 'config')
+def test_install_runtests():
+    assert not spack.package_testing._test_all
+    assert not spack.package_testing.packages_to_test
+
+    install('--test=root', 'dttop')
+    assert not spack.package_testing._test_all
+    assert spack.package_testing.packages_to_test == set(['dttop'])
+
+    spack.package_testing.clear()
+
+    install('--test=all', 'a')
+    assert spack.package_testing._test_all
+    assert not spack.package_testing.packages_to_test
+
+    spack.package_testing.clear()
+
+    install('--run-tests', 'a')
+    assert spack.package_testing._test_all
+    assert not spack.package_testing.packages_to_test
+
+    spack.package_testing.clear()
+
+    assert not spack.package_testing._test_all
+    assert not spack.package_testing.packages_to_test
+
+
 def test_install_package_already_installed(
         tmpdir, builtin_mock, mock_archive, mock_fetch, config,
         install_mockery):
diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py
index f918478b09..e19729e717 100644
--- a/lib/spack/spack/test/conftest.py
+++ b/lib/spack/spack/test/conftest.py
@@ -44,6 +44,8 @@ import spack.util.executable
 import spack.util.pattern
 from spack.package import PackageBase
 from spack.fetch_strategy import *
+from spack.spec import Spec
+from spack.version import Version
 
 
 ##########
@@ -552,3 +554,58 @@ def mock_svn_repository():
     t = Bunch(checks=checks, url=url, hash=get_rev, path=str(repodir))
     yield t
     current.chdir()
+
+
+##########
+# Mock packages
+##########
+
+
+class MockPackage(object):
+
+    def __init__(self, name, dependencies, dependency_types, conditions=None,
+                 versions=None):
+        self.name = name
+        self.spec = None
+        dep_to_conditions = ordereddict_backport.OrderedDict()
+        for dep in dependencies:
+            if not conditions or dep.name not in conditions:
+                dep_to_conditions[dep.name] = {name: dep.name}
+            else:
+                dep_to_conditions[dep.name] = conditions[dep.name]
+        self.dependencies = dep_to_conditions
+        self.dependency_types = dict(
+            (x.name, y) for x, y in zip(dependencies, dependency_types))
+        if versions:
+            self.versions = versions
+        else:
+            versions = list(Version(x) for x in [1, 2, 3])
+            self.versions = dict((x, {'preferred': False}) for x in versions)
+        self.variants = {}
+        self.provided = {}
+        self.conflicts = {}
+
+
+class MockPackageMultiRepo(object):
+
+    def __init__(self, packages):
+        self.specToPkg = dict((x.name, x) for x in packages)
+
+    def get(self, spec):
+        if not isinstance(spec, spack.spec.Spec):
+            spec = Spec(spec)
+        return self.specToPkg[spec.name]
+
+    def get_pkg_class(self, name):
+        return self.specToPkg[name]
+
+    def exists(self, name):
+        return name in self.specToPkg
+
+    def is_virtual(self, name):
+        return False
+
+    def repo_for_pkg(self, name):
+        import collections
+        Repo = collections.namedtuple('Repo', ['namespace'])
+        return Repo('mockrepo')
diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py
index 90c7b89c5f..648a49de9a 100644
--- a/lib/spack/spack/test/spec_dag.py
+++ b/lib/spack/spack/test/spec_dag.py
@@ -30,6 +30,7 @@ import spack
 import spack.architecture
 import spack.package
 
+from spack.test.conftest import MockPackage, MockPackageMultiRepo
 from spack.spec import Spec, canonical_deptype, alldeps
 
 
@@ -69,6 +70,44 @@ def set_dependency(saved_deps):
     return _mock
 
 
+@pytest.mark.usefixtures('config')
+def test_test_deptype():
+    """Ensure that test-only dependencies are only included for specified
+packages in the following spec DAG::
+
+        w
+       /|
+      x y
+        |
+        z
+
+w->y deptypes are (link, build), w->x and y->z deptypes are (test)
+
+"""
+    saved_repo = spack.repo
+
+    default = ('build', 'link')
+    test_only = ('test',)
+
+    x = MockPackage('x', [], [])
+    z = MockPackage('z', [], [])
+    y = MockPackage('y', [z], [test_only])
+    w = MockPackage('w', [x, y], [test_only, default])
+
+    mock_repo = MockPackageMultiRepo([w, x, y, z])
+    try:
+        spack.package_testing.test(w.name)
+        spack.repo = mock_repo
+        spec = Spec('w')
+        spec.concretize()
+
+        assert ('x' in spec)
+        assert ('z' not in spec)
+    finally:
+        spack.repo = saved_repo
+        spack.package_testing.clear()
+
+
 @pytest.mark.usefixtures('refresh_builtin_mock')
 class TestSpecDag(object):