1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
|
# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import argparse
import os
import pytest
import spack.cmd.install
import spack.config
import spack.package
from spack.cmd.test import has_test_method
from spack.main import SpackCommand
install = SpackCommand('install')
spack_test = SpackCommand('test')
def test_test_package_not_installed(
tmpdir, mock_packages, mock_archive, mock_fetch, config,
install_mockery_mutable_config, mock_test_stage):
output = spack_test('run', 'libdwarf')
assert "No installed packages match spec libdwarf" in output
@pytest.mark.parametrize('arguments,expected', [
(['run'], spack.config.get('config:dirty')), # default from config file
(['run', '--clean'], False),
(['run', '--dirty'], True),
])
def test_test_dirty_flag(arguments, expected):
parser = argparse.ArgumentParser()
spack.cmd.test.setup_parser(parser)
args = parser.parse_args(arguments)
assert args.dirty == expected
def test_test_dup_alias(
mock_test_stage, mock_packages, mock_archive, mock_fetch,
install_mockery_mutable_config, capfd):
"""Ensure re-using an alias fails with suggestion to change."""
install('libdwarf')
# Run the tests with the alias once
out = spack_test('run', '--alias', 'libdwarf', 'libdwarf')
assert "Spack test libdwarf" in out
# Try again with the alias but don't let it fail on the error
with capfd.disabled():
out = spack_test(
'run', '--alias', 'libdwarf', 'libdwarf', fail_on_error=False)
assert "already exists" in out
def test_test_output(mock_test_stage, mock_packages, mock_archive, mock_fetch,
install_mockery_mutable_config):
"""Ensure output printed from pkgs is captured by output redirection."""
install('printing-package')
spack_test('run', '--alias', 'printpkg', 'printing-package')
stage_files = os.listdir(mock_test_stage)
assert len(stage_files) == 1
# Grab test stage directory contents
testdir = os.path.join(mock_test_stage, stage_files[0])
testdir_files = os.listdir(testdir)
# Grab the output from the test log
testlog = list(filter(lambda x: x.endswith('out.txt') and
x != 'results.txt', testdir_files))
outfile = os.path.join(testdir, testlog[0])
with open(outfile, 'r') as f:
output = f.read()
assert "BEFORE TEST" in output
assert "true: expect command status in [" in output
assert "AFTER TEST" in output
assert "FAILED" not in output
def test_test_output_on_error(
mock_packages, mock_archive, mock_fetch, install_mockery_mutable_config,
capfd, mock_test_stage
):
install('test-error')
# capfd interferes with Spack's capturing
with capfd.disabled():
out = spack_test('run', 'test-error', fail_on_error=False)
assert "TestFailure" in out
assert "Command exited with status 1" in out
def test_test_output_on_failure(
mock_packages, mock_archive, mock_fetch, install_mockery_mutable_config,
capfd, mock_test_stage
):
install('test-fail')
with capfd.disabled():
out = spack_test('run', 'test-fail', fail_on_error=False)
assert "Expected 'not in the output' to match output of `true`" in out
assert "TestFailure" in out
def test_show_log_on_error(
mock_packages, mock_archive, mock_fetch,
install_mockery_mutable_config, capfd, mock_test_stage
):
"""Make sure spack prints location of test log on failure."""
install('test-error')
with capfd.disabled():
out = spack_test('run', 'test-error', fail_on_error=False)
assert 'See test log' in out
assert mock_test_stage in out
@pytest.mark.usefixtures(
'mock_packages', 'mock_archive', 'mock_fetch',
'install_mockery_mutable_config'
)
@pytest.mark.parametrize('pkg_name,msgs', [
('test-error', ['FAILED: Command exited', 'TestFailure']),
('test-fail', ['FAILED: Expected', 'TestFailure'])
])
def test_junit_output_with_failures(tmpdir, mock_test_stage, pkg_name, msgs):
install(pkg_name)
with tmpdir.as_cwd():
spack_test('run',
'--log-format=junit', '--log-file=test.xml',
pkg_name,
fail_on_error=False)
files = tmpdir.listdir()
filename = tmpdir.join('test.xml')
assert filename in files
content = filename.open().read()
# Count failures and errors correctly
assert 'tests="1"' in content
assert 'failures="1"' in content
assert 'errors="0"' in content
# We want to have both stdout and stderr
assert '<system-out>' in content
for msg in msgs:
assert msg in content
def test_cdash_output_test_error(
tmpdir, mock_fetch, install_mockery_mutable_config, mock_packages,
mock_archive, mock_test_stage, capfd):
install('test-error')
with tmpdir.as_cwd():
spack_test('run',
'--log-format=cdash',
'--log-file=cdash_reports',
'test-error',
fail_on_error=False)
report_dir = tmpdir.join('cdash_reports')
print(tmpdir.listdir())
assert report_dir in tmpdir.listdir()
report_file = report_dir.join('test-error_Test.xml')
assert report_file in report_dir.listdir()
content = report_file.open().read()
assert 'FAILED: Command exited with status 1' in content
def test_cdash_upload_clean_test(
tmpdir, mock_fetch, install_mockery_mutable_config, mock_packages,
mock_archive, mock_test_stage):
install('printing-package')
with tmpdir.as_cwd():
spack_test('run',
'--log-file=cdash_reports',
'--log-format=cdash',
'printing-package')
report_dir = tmpdir.join('cdash_reports')
assert report_dir in tmpdir.listdir()
report_file = report_dir.join('printing-package_Test.xml')
assert report_file in report_dir.listdir()
content = report_file.open().read()
assert '</Test>' in content
assert '<Text>' not in content
def test_test_help_does_not_show_cdash_options(mock_test_stage, capsys):
"""Make sure `spack test --help` does not describe CDash arguments"""
with pytest.raises(SystemExit):
spack_test('run', '--help')
captured = capsys.readouterr()
assert 'CDash URL' not in captured.out
def test_test_help_cdash(mock_test_stage):
"""Make sure `spack test --help-cdash` describes CDash arguments"""
out = spack_test('run', '--help-cdash')
assert 'CDash URL' in out
def test_test_list_all(mock_packages):
"""make sure `spack test list --all` returns all packages with tests"""
pkgs = spack_test("list", "--all").strip().split()
assert set(pkgs) == set([
"printing-package",
"py-extension1",
"py-extension2",
"test-error",
"test-fail",
])
def test_test_list(
mock_packages, mock_archive, mock_fetch, install_mockery_mutable_config
):
pkg_with_tests = 'printing-package'
install(pkg_with_tests)
output = spack_test("list")
assert pkg_with_tests in output
def test_has_test_method_fails(capsys):
with pytest.raises(SystemExit):
has_test_method('printing-package')
captured = capsys.readouterr()[1]
assert 'is not a class' in captured
|
