summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTodd Gamblin <tgamblin@llnl.gov>2016-04-27 17:32:21 -0700
committerTodd Gamblin <tgamblin@llnl.gov>2016-04-27 17:32:21 -0700
commit8773a0b747be1c21170d8e4d573b6025768d9c27 (patch)
tree622eb3b55d09c33e937c2d40e2b4e2c0be2bcafa
parent8617ceddf3ad4245d07bde32ac65fc45bffcb9ba (diff)
parentb1ba869b37803ba9cda28ab64b7f0052e8eda11d (diff)
downloadspack-8773a0b747be1c21170d8e4d573b6025768d9c27.tar.gz
spack-8773a0b747be1c21170d8e4d573b6025768d9c27.tar.bz2
spack-8773a0b747be1c21170d8e4d573b6025768d9c27.tar.xz
spack-8773a0b747be1c21170d8e4d573b6025768d9c27.zip
Merge pull request #847 from epfl-scitas/features/test_install_with_time
test-install command : added elapsed time + xml is prettyprinted
-rw-r--r--lib/spack/spack/cmd/test-install.py300
-rw-r--r--lib/spack/spack/test/__init__.py4
-rw-r--r--lib/spack/spack/test/cmd/test_install.py190
-rw-r--r--lib/spack/spack/test/unit_install.py126
4 files changed, 349 insertions, 271 deletions
diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py
index 656873a2f0..3277e15548 100644
--- a/lib/spack/spack/cmd/test-install.py
+++ b/lib/spack/spack/cmd/test-install.py
@@ -23,87 +23,106 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import argparse
-import xml.etree.ElementTree as ET
-import itertools
-import re
-import os
import codecs
+import os
+import time
+import xml.dom.minidom
+import xml.etree.ElementTree as ET
import llnl.util.tty as tty
-from llnl.util.filesystem import *
-
import spack
+import spack.cmd
+from llnl.util.filesystem import *
from spack.build_environment import InstallError
from spack.fetch_strategy import FetchError
-import spack.cmd
description = "Run package installation as a unit test, output formatted results."
-def setup_parser(subparser):
- subparser.add_argument(
- '-j', '--jobs', action='store', type=int,
- help="Explicitly set number of make jobs. Default is #cpus.")
-
- subparser.add_argument(
- '-n', '--no-checksum', action='store_true', dest='no_checksum',
- help="Do not check packages against checksum")
- subparser.add_argument(
- '-o', '--output', action='store', help="test output goes in this file")
+def setup_parser(subparser):
+ subparser.add_argument('-j',
+ '--jobs',
+ action='store',
+ type=int,
+ help="Explicitly set number of make jobs. Default is #cpus.")
- subparser.add_argument(
- 'package', nargs=argparse.REMAINDER, help="spec of package to install")
+ subparser.add_argument('-n',
+ '--no-checksum',
+ action='store_true',
+ dest='no_checksum',
+ help="Do not check packages against checksum")
+ subparser.add_argument('-o', '--output', action='store', help="test output goes in this file")
-class JunitResultFormat(object):
- def __init__(self):
- self.root = ET.Element('testsuite')
- self.tests = []
-
- def add_test(self, buildId, testResult, buildInfo=None):
- self.tests.append((buildId, testResult, buildInfo))
-
- def write_to(self, stream):
- self.root.set('tests', '{0}'.format(len(self.tests)))
- for buildId, testResult, buildInfo in self.tests:
- testcase = ET.SubElement(self.root, 'testcase')
- testcase.set('classname', buildId.name)
- testcase.set('name', buildId.stringId())
- if testResult == TestResult.FAILED:
- failure = ET.SubElement(testcase, 'failure')
- failure.set('type', "Build Error")
- failure.text = buildInfo
- elif testResult == TestResult.SKIPPED:
- skipped = ET.SubElement(testcase, 'skipped')
- skipped.set('type', "Skipped Build")
- skipped.text = buildInfo
- ET.ElementTree(self.root).write(stream)
+ subparser.add_argument('package', nargs=argparse.REMAINDER, help="spec of package to install")
class TestResult(object):
PASSED = 0
FAILED = 1
SKIPPED = 2
+ ERRORED = 3
-class BuildId(object):
- def __init__(self, spec):
- self.name = spec.name
- self.version = spec.version
- self.hashId = spec.dag_hash()
-
- def stringId(self):
- return "-".join(str(x) for x in (self.name, self.version, self.hashId))
-
- def __hash__(self):
- return hash((self.name, self.version, self.hashId))
-
- def __eq__(self, other):
- if not isinstance(other, BuildId):
- return False
+class TestSuite(object):
+ def __init__(self, filename):
+ self.filename = filename
+ self.root = ET.Element('testsuite')
+ self.tests = []
- return ((self.name, self.version, self.hashId) ==
- (other.name, other.version, other.hashId))
+ def __enter__(self):
+ return self
+
+ def append(self, item):
+ if not isinstance(item, TestCase):
+ raise TypeError('only TestCase instances may be appended to a TestSuite instance')
+ self.tests.append(item) # Append the item to the list of tests
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ # Prepare the header for the entire test suite
+ number_of_errors = sum(x.result_type == TestResult.ERRORED for x in self.tests)
+ self.root.set('errors', str(number_of_errors))
+ number_of_failures = sum(x.result_type == TestResult.FAILED for x in self.tests)
+ self.root.set('failures', str(number_of_failures))
+ self.root.set('tests', str(len(self.tests)))
+
+ for item in self.tests:
+ self.root.append(item.element)
+
+ with open(self.filename, 'wb') as file:
+ xml_string = ET.tostring(self.root)
+ xml_string = xml.dom.minidom.parseString(xml_string).toprettyxml()
+ file.write(xml_string)
+
+
+class TestCase(object):
+
+ results = {
+ TestResult.PASSED: None,
+ TestResult.SKIPPED: 'skipped',
+ TestResult.FAILED: 'failure',
+ TestResult.ERRORED: 'error',
+ }
+
+ def __init__(self, classname, name, time=None):
+ self.element = ET.Element('testcase')
+ self.element.set('classname', str(classname))
+ self.element.set('name', str(name))
+ if time is not None:
+ self.element.set('time', str(time))
+ self.result_type = None
+
+ def set_result(self, result_type, message=None, error_type=None, text=None):
+ self.result_type = result_type
+ result = TestCase.results[self.result_type]
+ if result is not None and result is not TestResult.PASSED:
+ subelement = ET.SubElement(self.element, result)
+ if error_type is not None:
+ subelement.set('type', error_type)
+ if message is not None:
+ subelement.set('message', str(message))
+ if text is not None:
+ subelement.text = text
def fetch_log(path):
@@ -114,46 +133,76 @@ def fetch_log(path):
def failed_dependencies(spec):
- return set(childSpec for childSpec in spec.dependencies.itervalues() if not
- spack.repo.get(childSpec).installed)
-
-
-def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log):
- # Post-order traversal is not strictly required but it makes sense to output
- # tests for dependencies first.
- for spec in topSpec.traverse(order='post'):
- if spec not in newInstalls:
- continue
-
- failedDeps = failed_dependencies(spec)
- package = spack.repo.get(spec)
- if failedDeps:
- result = TestResult.SKIPPED
- dep = iter(failedDeps).next()
- depBID = BuildId(dep)
- errOutput = "Skipped due to failed dependency: {0}".format(
- depBID.stringId())
- elif (not package.installed) and (not package.stage.source_path):
- result = TestResult.FAILED
- errOutput = "Failure to fetch package resources."
- elif not package.installed:
- result = TestResult.FAILED
- lines = getLogFunc(package.build_log_path)
- errMessages = list(line for line in lines if
- re.search('error:', line, re.IGNORECASE))
- errOutput = errMessages if errMessages else lines[-10:]
- errOutput = '\n'.join(itertools.chain(
- [spec.to_yaml(), "Errors:"], errOutput,
- ["Build Log:", package.build_log_path]))
- else:
- result = TestResult.PASSED
- errOutput = None
-
- bId = BuildId(spec)
- output.add_test(bId, result, errOutput)
+ return set(item for item in spec.dependencies.itervalues() if not spack.repo.get(item).installed)
+
+
+def get_top_spec_or_die(args):
+ specs = spack.cmd.parse_specs(args.package, concretize=True)
+ if len(specs) > 1:
+ tty.die("Only 1 top-level package can be specified")
+ top_spec = iter(specs).next()
+ return top_spec
+
+
+def install_single_spec(spec, number_of_jobs):
+ package = spack.repo.get(spec)
+
+ # If it is already installed, skip the test
+ if spack.repo.get(spec).installed:
+ testcase = TestCase(package.name, package.spec.short_spec, time=0.0)
+ testcase.set_result(TestResult.SKIPPED, message='Skipped [already installed]', error_type='already_installed')
+ return testcase
+
+ # If it relies on dependencies that did not install, skip
+ if failed_dependencies(spec):
+ testcase = TestCase(package.name, package.spec.short_spec, time=0.0)
+ testcase.set_result(TestResult.SKIPPED, message='Skipped [failed dependencies]', error_type='dep_failed')
+ return testcase
+
+ # Otherwise try to install the spec
+ try:
+ start_time = time.time()
+ package.do_install(keep_prefix=False,
+ keep_stage=True,
+ ignore_deps=False,
+ make_jobs=number_of_jobs,
+ verbose=True,
+ fake=False)
+ duration = time.time() - start_time
+ testcase = TestCase(package.name, package.spec.short_spec, duration)
+ testcase.set_result(TestResult.PASSED)
+ except InstallError:
+ # An InstallError is considered a failure (the recipe didn't work correctly)
+ duration = time.time() - start_time
+ # Try to get the log
+ lines = fetch_log(package.build_log_path)
+ text = '\n'.join(lines)
+ testcase = TestCase(package.name, package.spec.short_spec, duration)
+ testcase.set_result(TestResult.FAILED, message='Installation failure', text=text)
+
+ except FetchError:
+ # A FetchError is considered an error (we didn't even start building)
+ duration = time.time() - start_time
+ testcase = TestCase(package.name, package.spec.short_spec, duration)
+ testcase.set_result(TestResult.ERRORED, message='Unable to fetch package')
+
+ return testcase
+
+
+def get_filename(args, top_spec):
+ if not args.output:
+ fname = 'test-{x.name}-{x.version}-{hash}.xml'.format(x=top_spec, hash=top_spec.dag_hash())
+ output_directory = join_path(os.getcwd(), 'test-output')
+ if not os.path.exists(output_directory):
+ os.mkdir(output_directory)
+ output_filename = join_path(output_directory, fname)
+ else:
+ output_filename = args.output
+ return output_filename
def test_install(parser, args):
+ # Check the input
if not args.package:
tty.die("install requires a package argument")
@@ -162,50 +211,15 @@ def test_install(parser, args):
tty.die("The -j option must be a positive integer!")
if args.no_checksum:
- spack.do_checksum = False # TODO: remove this global.
-
- specs = spack.cmd.parse_specs(args.package, concretize=True)
- if len(specs) > 1:
- tty.die("Only 1 top-level package can be specified")
- topSpec = iter(specs).next()
-
- newInstalls = set()
- for spec in topSpec.traverse():
- package = spack.repo.get(spec)
- if not package.installed:
- newInstalls.add(spec)
-
- if not args.output:
- bId = BuildId(topSpec)
- outputDir = join_path(os.getcwd(), "test-output")
- if not os.path.exists(outputDir):
- os.mkdir(outputDir)
- outputFpath = join_path(outputDir, "test-{0}.xml".format(bId.stringId()))
- else:
- outputFpath = args.output
-
- for spec in topSpec.traverse(order='post'):
- # Calling do_install for the top-level package would be sufficient but
- # this attempts to keep going if any package fails (other packages which
- # are not dependents may succeed)
- package = spack.repo.get(spec)
- if (not failed_dependencies(spec)) and (not package.installed):
- try:
- package.do_install(
- keep_prefix=False,
- keep_stage=True,
- ignore_deps=False,
- make_jobs=args.jobs,
- verbose=True,
- fake=False)
- except InstallError:
- pass
- except FetchError:
- pass
-
- jrf = JunitResultFormat()
- handled = {}
- create_test_output(topSpec, newInstalls, jrf)
-
- with open(outputFpath, 'wb') as F:
- jrf.write_to(F)
+ spack.do_checksum = False # TODO: remove this global.
+
+ # Get the one and only top spec
+ top_spec = get_top_spec_or_die(args)
+ # Get the filename of the test
+ output_filename = get_filename(args, top_spec)
+ # TEST SUITE
+ with TestSuite(output_filename) as test_suite:
+ # Traverse in post order : each spec is a test case
+ for spec in top_spec.traverse(order='post'):
+ test_case = install_single_spec(spec, args.jobs)
+ test_suite.append(test_case)
diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py
index 175a49428c..3c5edde66b 100644
--- a/lib/spack/spack/test/__init__.py
+++ b/lib/spack/spack/test/__init__.py
@@ -61,14 +61,14 @@ test_names = ['versions',
'optional_deps',
'make_executable',
'configure_guess',
- 'unit_install',
'lock',
'database',
'namespace_trie',
'yaml',
'sbang',
'environment',
- 'cmd.uninstall']
+ 'cmd.uninstall',
+ 'cmd.test_install']
def list_tests():
diff --git a/lib/spack/spack/test/cmd/test_install.py b/lib/spack/spack/test/cmd/test_install.py
new file mode 100644
index 0000000000..2206c7bea1
--- /dev/null
+++ b/lib/spack/spack/test/cmd/test_install.py
@@ -0,0 +1,190 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import collections
+from contextlib import contextmanager
+
+import StringIO
+
+FILE_REGISTRY = collections.defaultdict(StringIO.StringIO)
+
+# Monkey-patch open to write module files to a StringIO instance
+@contextmanager
+def mock_open(filename, mode):
+ if not mode == 'wb':
+ raise RuntimeError('test.test_install : unexpected opening mode for monkey-patched open')
+
+ FILE_REGISTRY[filename] = StringIO.StringIO()
+
+ try:
+ yield FILE_REGISTRY[filename]
+ finally:
+ handle = FILE_REGISTRY[filename]
+ FILE_REGISTRY[filename] = handle.getvalue()
+ handle.close()
+
+import os
+import itertools
+import unittest
+
+import spack
+import spack.cmd
+
+
+# The use of __import__ is necessary to maintain a name with hyphen (which cannot be an identifier in python)
+test_install = __import__("spack.cmd.test-install", fromlist=['test_install'])
+
+
+class MockSpec(object):
+ def __init__(self, name, version, hashStr=None):
+ self.dependencies = {}
+ self.name = name
+ self.version = version
+ self.hash = hashStr if hashStr else hash((name, version))
+
+ def traverse(self, order=None):
+ for _, spec in self.dependencies.items():
+ yield spec
+ yield self
+ #allDeps = itertools.chain.from_iterable(i.traverse() for i in self.dependencies.itervalues())
+ #return set(itertools.chain([self], allDeps))
+
+ def dag_hash(self):
+ return self.hash
+
+ @property
+ def short_spec(self):
+ return '-'.join([self.name, str(self.version), str(self.hash)])
+
+
+class MockPackage(object):
+ def __init__(self, spec, buildLogPath):
+ self.name = spec.name
+ self.spec = spec
+ self.installed = False
+ self.build_log_path = buildLogPath
+
+ def do_install(self, *args, **kwargs):
+ self.installed = True
+
+
+class MockPackageDb(object):
+ def __init__(self, init=None):
+ self.specToPkg = {}
+ if init:
+ self.specToPkg.update(init)
+
+ def get(self, spec):
+ return self.specToPkg[spec]
+
+
+def mock_fetch_log(path):
+ return []
+
+specX = MockSpec('X', "1.2.0")
+specY = MockSpec('Y', "2.3.8")
+specX.dependencies['Y'] = specY
+pkgX = MockPackage(specX, 'logX')
+pkgY = MockPackage(specY, 'logY')
+
+
+class MockArgs(object):
+ def __init__(self, package):
+ self.package = package
+ self.jobs = None
+ self.no_checksum = False
+ self.output = None
+
+
+# TODO: add test(s) where Y fails to install
+class TestInstallTest(unittest.TestCase):
+ """
+ Tests test-install where X->Y
+ """
+
+ def setUp(self):
+ super(TestInstallTest, self).setUp()
+
+ # Monkey patch parse specs
+ def monkey_parse_specs(x, concretize):
+ if x == 'X':
+ return [specX]
+ elif x == 'Y':
+ return [specY]
+ return []
+
+ self.parse_specs = spack.cmd.parse_specs
+ spack.cmd.parse_specs = monkey_parse_specs
+
+ # Monkey patch os.mkdirp
+ self.os_mkdir = os.mkdir
+ os.mkdir = lambda x: True
+
+ # Monkey patch open
+ test_install.open = mock_open
+
+ # Clean FILE_REGISTRY
+ FILE_REGISTRY = collections.defaultdict(StringIO.StringIO)
+
+ pkgX.installed = False
+ pkgY.installed = False
+
+ # Monkey patch pkgDb
+ self.saved_db = spack.repo
+ pkgDb = MockPackageDb({specX: pkgX, specY: pkgY})
+ spack.repo = pkgDb
+
+ def tearDown(self):
+ # Remove the monkey patched test_install.open
+ test_install.open = open
+
+ # Remove the monkey patched os.mkdir
+ os.mkdir = self.os_mkdir
+ del self.os_mkdir
+
+ # Remove the monkey patched parse_specs
+ spack.cmd.parse_specs = self.parse_specs
+ del self.parse_specs
+ super(TestInstallTest, self).tearDown()
+
+ spack.repo = self.saved_db
+
+ def test_installing_both(self):
+ test_install.test_install(None, MockArgs('X') )
+ self.assertEqual(len(FILE_REGISTRY), 1)
+ for _, content in FILE_REGISTRY.items():
+ self.assertTrue('tests="2"' in content)
+ self.assertTrue('failures="0"' in content)
+ self.assertTrue('errors="0"' in content)
+
+ def test_dependency_already_installed(self):
+ pkgX.installed = True
+ pkgY.installed = True
+ test_install.test_install(None, MockArgs('X'))
+ self.assertEqual(len(FILE_REGISTRY), 1)
+ for _, content in FILE_REGISTRY.items():
+ self.assertTrue('tests="2"' in content)
+ self.assertTrue('failures="0"' in content)
+ self.assertTrue('errors="0"' in content)
+ self.assertEqual(sum('skipped' in line for line in content.split('\n')), 2)
diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py
deleted file mode 100644
index 18615b7efe..0000000000
--- a/lib/spack/spack/test/unit_install.py
+++ /dev/null
@@ -1,126 +0,0 @@
-##############################################################################
-# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
-# Produced at the Lawrence Livermore National Laboratory.
-#
-# This file is part of Spack.
-# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
-# LLNL-CODE-647188
-#
-# For details, see https://github.com/llnl/spack
-# Please also see the LICENSE file for our notice and the LGPL.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License (as published by
-# the Free Software Foundation) version 2.1 dated February 1999.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
-# conditions of the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this program; if not, write to the Free Software Foundation,
-# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-##############################################################################
-import itertools
-import unittest
-
-import spack
-
-test_install = __import__("spack.cmd.test-install",
- fromlist=["BuildId", "create_test_output", "TestResult"])
-
-class MockOutput(object):
- def __init__(self):
- self.results = {}
-
- def add_test(self, buildId, passed=True, buildInfo=None):
- self.results[buildId] = passed
-
- def write_to(self, stream):
- pass
-
-class MockSpec(object):
- def __init__(self, name, version, hashStr=None):
- self.dependencies = {}
- self.name = name
- self.version = version
- self.hash = hashStr if hashStr else hash((name, version))
-
- def traverse(self, order=None):
- allDeps = itertools.chain.from_iterable(i.traverse() for i in
- self.dependencies.itervalues())
- return set(itertools.chain([self], allDeps))
-
- def dag_hash(self):
- return self.hash
-
- def to_yaml(self):
- return "<<<MOCK YAML {0}>>>".format(test_install.BuildId(self).stringId())
-
-class MockPackage(object):
- def __init__(self, buildLogPath):
- self.installed = False
- self.build_log_path = buildLogPath
-
-specX = MockSpec("X", "1.2.0")
-specY = MockSpec("Y", "2.3.8")
-specX.dependencies['Y'] = specY
-pkgX = MockPackage('logX')
-pkgY = MockPackage('logY')
-bIdX = test_install.BuildId(specX)
-bIdY = test_install.BuildId(specY)
-
-class UnitInstallTest(unittest.TestCase):
- """Tests test-install where X->Y"""
-
- def setUp(self):
- super(UnitInstallTest, self).setUp()
-
- pkgX.installed = False
- pkgY.installed = False
-
- self.saved_db = spack.repo
- pkgDb = MockPackageDb({specX:pkgX, specY:pkgY})
- spack.repo = pkgDb
-
-
- def tearDown(self):
- super(UnitInstallTest, self).tearDown()
-
- spack.repo = self.saved_db
-
- def test_installing_both(self):
- mo = MockOutput()
-
- pkgX.installed = True
- pkgY.installed = True
- test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=mock_fetch_log)
-
- self.assertEqual(mo.results,
- {bIdX:test_install.TestResult.PASSED,
- bIdY:test_install.TestResult.PASSED})
-
-
- def test_dependency_already_installed(self):
- mo = MockOutput()
-
- pkgX.installed = True
- pkgY.installed = True
- test_install.create_test_output(specX, [specX], mo, getLogFunc=mock_fetch_log)
- self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED})
-
- #TODO: add test(s) where Y fails to install
-
-
-class MockPackageDb(object):
- def __init__(self, init=None):
- self.specToPkg = {}
- if init:
- self.specToPkg.update(init)
-
- def get(self, spec):
- return self.specToPkg[spec]
-
-def mock_fetch_log(path):
- return []