From 099fa1df34a157a323aa023b69c52399a441b81b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 12 Nov 2015 13:23:19 -0800 Subject: Use nose to run unit tests. 1. Adding a plugin to keep track of the total number of tests run as well as the number of tests with failures/errors. 2. Some nose plugins (including xunit which will be added in a future commit) assign stdout to a stream object that does not have a .fileno attribute. spack.util.executable.Executable now avoids passing stdout to subprocess (and always uses subprocess.PIPE) TODO: 1. Still need to figure out how to activate the plugin (as of now it is being ignored by nose). Newer versions of nose appear to make this simpler (e.g. the "addplugins" argument to nose.run) 2. Need to include new version of nose in order to use xunit --- lib/spack/spack/test/__init__.py | 27 ++++++------- lib/spack/spack/test/tally_plugin.py | 73 ++++++++++++++++++++++++++++++++++++ lib/spack/spack/test/unit_install.py | 6 +-- lib/spack/spack/util/executable.py | 7 +++- 4 files changed, 93 insertions(+), 20 deletions(-) create mode 100644 lib/spack/spack/test/tally_plugin.py (limited to 'lib') diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 0f776bfea4..9d577d6539 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -24,7 +24,9 @@ ############################################################################## import sys import unittest +import nose +from spack.test.tally_plugin import Tally import llnl.util.tty as tty from llnl.util.tty.colify import colify @@ -81,28 +83,23 @@ def run(names, verbose=False): "Valid names are:") colify(sorted(test_names), indent=4) sys.exit(1) - - runner = unittest.TextTestRunner(verbosity=verbosity) - - testsRun = errors = failures = 0 + + tally = Tally() + tally.enabled = True for test in names: module = 'spack.test.' + test print module - suite = unittest.defaultTestLoader.loadTestsFromName(module) - + tty.msg("Running test: %s" % test) - result = runner.run(suite) - testsRun += result.testsRun - errors += len(result.errors) - failures += len(result.failures) + result = nose.run(argv=["", module], plugins=[tally]) - succeeded = not errors and not failures + succeeded = not tally.failCount and not tally.errorCount tty.msg("Tests Complete.", - "%5d tests run" % testsRun, - "%5d failures" % failures, - "%5d errors" % errors) + "%5d tests run" % tally.numberOfTests, + "%5d failures" % tally.failCount, + "%5d errors" % tally.errorCount) - if not errors and not failures: + if succeeded: tty.info("OK", format='g') else: tty.info("FAIL", format='r') diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py new file mode 100644 index 0000000000..d8638bf7e3 --- /dev/null +++ b/lib/spack/spack/test/tally_plugin.py @@ -0,0 +1,73 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from nose.plugins import Plugin + +import os + +class Tally(Plugin): + name = 'tally' + + def __init__(self): + super(Tally, self).__init__() + self.successes = set() + self.failures = set() + self.errors = set() + + @property + def successCount(self): + return len(self.successes) + + @property + def failCount(self): + return len(self.failures) + + @property + def errorCount(self): + return len(self.errors) + + @property + def numberOfTests(self): + return self.errorCount + self.failCount + self.successCount + + def options(self, parser, env=os.environ): + super(Tally, self).options(parser, env=env) + + def configure(self, options, conf): + super(Tally, self).configure(options, conf) + + def begin(self): + print ">>> TALLY PLUGIN BEGIN" + + def addSuccess(self, test): + self.successes.add(test) + + def addError(self, test, err): + self.errors.add(test) + + def addFailure(self, test, err): + test.failures.add(test) + + def finalize(self, result): + pass diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index c4b9092f05..0bcb3f3767 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -90,7 +90,7 @@ class UnitInstallTest(unittest.TestCase): pkgX.installed = True pkgY.installed = True - test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=test_fetch_log) + test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=mock_fetch_log) self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED, @@ -101,7 +101,7 @@ class UnitInstallTest(unittest.TestCase): pkgX.installed = True pkgY.installed = True - test_install.create_test_output(specX, [specX], mo, getLogFunc=test_fetch_log) + test_install.create_test_output(specX, [specX], mo, getLogFunc=mock_fetch_log) self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED}) @@ -116,6 +116,6 @@ class MockPackageDb(object): def get(self, spec): return self.specToPkg[spec] -def test_fetch_log(path): +def mock_fetch_log(path): return [] diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index d1dfb62ffb..6848a7e2a5 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -95,11 +95,14 @@ class Executable(object): proc = subprocess.Popen( cmd, stdin=input, - stderr=error, - stdout=subprocess.PIPE if return_output else output) + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) out, err = proc.communicate() self.returncode = proc.returncode + output.write(out) + error.write(err) + rc = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:" -- cgit v1.2.3-70-g09d2