1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
|
##############################################################################
# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""
These tests check the database is functioning properly,
both in memory and in its file
"""
import multiprocessing
import shutil
import tempfile
import spack
from llnl.util.filesystem import join_path
from llnl.util.lock import *
from llnl.util.tty.colify import colify
from spack.database import Database
from spack.directory_layout import YamlDirectoryLayout
from spack.test.mock_packages_test import *
def _print_ref_counts():
"""Print out all ref counts for the graph used here, for debugging"""
recs = []
def add_rec(spec):
cspecs = spack.installed_db.query(spec, installed=any)
if not cspecs:
recs.append("[ %-7s ] %-20s-" % ('', spec))
else:
key = cspecs[0].dag_hash()
rec = spack.installed_db.get_record(cspecs[0])
recs.append("[ %-7s ] %-20s%d" % (key[:7], spec, rec.ref_count))
with spack.installed_db.read_transaction():
add_rec('mpileaks ^mpich')
add_rec('callpath ^mpich')
add_rec('mpich')
add_rec('mpileaks ^mpich2')
add_rec('callpath ^mpich2')
add_rec('mpich2')
add_rec('mpileaks ^zmpi')
add_rec('callpath ^zmpi')
add_rec('zmpi')
add_rec('fake')
add_rec('dyninst')
add_rec('libdwarf')
add_rec('libelf')
colify(recs, cols=3)
class DatabaseTest(MockPackagesTest):
def _mock_install(self, spec):
s = Spec(spec)
s.concretize()
pkg = spack.repo.get(s)
pkg.do_install(fake=True)
def _mock_remove(self, spec):
specs = spack.installed_db.query(spec)
assert(len(specs) == 1)
spec = specs[0]
spec.package.do_uninstall(spec)
def setUp(self):
super(DatabaseTest, self).setUp()
#
# TODO: make the mockup below easier.
#
# Make a fake install directory
self.install_path = tempfile.mkdtemp()
self.spack_install_path = spack.install_path
spack.install_path = self.install_path
self.install_layout = YamlDirectoryLayout(self.install_path)
self.spack_install_layout = spack.install_layout
spack.install_layout = self.install_layout
# Make fake database and fake install directory.
self.installed_db = Database(self.install_path)
self.spack_installed_db = spack.installed_db
spack.installed_db = self.installed_db
# make a mock database with some packages installed note that
# the ref count for dyninst here will be 3, as it's recycled
# across each install.
#
# Here is what the mock DB looks like:
#
# o mpileaks o mpileaks' o mpileaks''
# |\ |\ |\
# | o callpath | o callpath' | o callpath''
# |/| |/| |/|
# o | mpich o | mpich2 o | zmpi
# | | o | fake
# | | |
# | |______________/
# | .____________/
# |/
# o dyninst
# |\
# | o libdwarf
# |/
# o libelf
#
# Transaction used to avoid repeated writes.
with spack.installed_db.write_transaction():
self._mock_install('mpileaks ^mpich')
self._mock_install('mpileaks ^mpich2')
self._mock_install('mpileaks ^zmpi')
def tearDown(self):
super(DatabaseTest, self).tearDown()
shutil.rmtree(self.install_path)
spack.install_path = self.spack_install_path
spack.install_layout = self.spack_install_layout
spack.installed_db = self.spack_installed_db
def test_005_db_exists(self):
"""Make sure db cache file exists after creating."""
index_file = join_path(self.install_path, '.spack-db', 'index.yaml')
lock_file = join_path(self.install_path, '.spack-db', 'lock')
self.assertTrue(os.path.exists(index_file))
self.assertTrue(os.path.exists(lock_file))
def test_010_all_install_sanity(self):
"""Ensure that the install layout reflects what we think it does."""
all_specs = spack.install_layout.all_specs()
self.assertEqual(len(all_specs), 13)
# query specs with multiple configurations
mpileaks_specs = [s for s in all_specs if s.satisfies('mpileaks')]
callpath_specs = [s for s in all_specs if s.satisfies('callpath')]
mpi_specs = [s for s in all_specs if s.satisfies('mpi')]
self.assertEqual(len(mpileaks_specs), 3)
self.assertEqual(len(callpath_specs), 3)
self.assertEqual(len(mpi_specs), 3)
# query specs with single configurations
dyninst_specs = [s for s in all_specs if s.satisfies('dyninst')]
libdwarf_specs = [s for s in all_specs if s.satisfies('libdwarf')]
libelf_specs = [s for s in all_specs if s.satisfies('libelf')]
self.assertEqual(len(dyninst_specs), 1)
self.assertEqual(len(libdwarf_specs), 1)
self.assertEqual(len(libelf_specs), 1)
# Query by dependency
self.assertEqual(len([s for s in all_specs if s.satisfies('mpileaks ^mpich')]), 1)
self.assertEqual(len([s for s in all_specs if s.satisfies('mpileaks ^mpich2')]), 1)
self.assertEqual(len([s for s in all_specs if s.satisfies('mpileaks ^zmpi')]), 1)
def test_015_write_and_read(self):
# write and read DB
with spack.installed_db.write_transaction():
specs = spack.installed_db.query()
recs = [spack.installed_db.get_record(s) for s in specs]
for spec, rec in zip(specs, recs):
new_rec = spack.installed_db.get_record(spec)
self.assertEqual(new_rec.ref_count, rec.ref_count)
self.assertEqual(new_rec.spec, rec.spec)
self.assertEqual(new_rec.path, rec.path)
self.assertEqual(new_rec.installed, rec.installed)
def _check_db_sanity(self):
"""Utiilty function to check db against install layout."""
expected = sorted(spack.install_layout.all_specs())
actual = sorted(self.installed_db.query())
self.assertEqual(len(expected), len(actual))
for e, a in zip(expected, actual):
self.assertEqual(e, a)
def test_020_db_sanity(self):
"""Make sure query() returns what's actually in the db."""
self._check_db_sanity()
def test_030_db_sanity_from_another_process(self):
def read_and_modify():
self._check_db_sanity() # check that other process can read DB
with self.installed_db.write_transaction():
self._mock_remove('mpileaks ^zmpi')
p = multiprocessing.Process(target=read_and_modify, args=())
p.start()
p.join()
# ensure child process change is visible in parent process
with self.installed_db.read_transaction():
self.assertEqual(len(self.installed_db.query('mpileaks ^zmpi')), 0)
def test_040_ref_counts(self):
"""Ensure that we got ref counts right when we read the DB."""
self.installed_db._check_ref_counts()
def test_050_basic_query(self):
"""Ensure that querying the database is consistent with what is installed."""
# query everything
self.assertEqual(len(spack.installed_db.query()), 13)
# query specs with multiple configurations
mpileaks_specs = self.installed_db.query('mpileaks')
callpath_specs = self.installed_db.query('callpath')
mpi_specs = self.installed_db.query('mpi')
self.assertEqual(len(mpileaks_specs), 3)
self.assertEqual(len(callpath_specs), 3)
self.assertEqual(len(mpi_specs), 3)
# query specs with single configurations
dyninst_specs = self.installed_db.query('dyninst')
libdwarf_specs = self.installed_db.query('libdwarf')
libelf_specs = self.installed_db.query('libelf')
self.assertEqual(len(dyninst_specs), 1)
self.assertEqual(len(libdwarf_specs), 1)
self.assertEqual(len(libelf_specs), 1)
# Query by dependency
self.assertEqual(len(self.installed_db.query('mpileaks ^mpich')), 1)
self.assertEqual(len(self.installed_db.query('mpileaks ^mpich2')), 1)
self.assertEqual(len(self.installed_db.query('mpileaks ^zmpi')), 1)
def _check_remove_and_add_package(self, spec):
"""Remove a spec from the DB, then add it and make sure everything's
still ok once it is added. This checks that it was
removed, that it's back when added again, and that ref
counts are consistent.
"""
original = self.installed_db.query()
self.installed_db._check_ref_counts()
# Remove spec
concrete_spec = self.installed_db.remove(spec)
self.installed_db._check_ref_counts()
remaining = self.installed_db.query()
# ensure spec we removed is gone
self.assertEqual(len(original) - 1, len(remaining))
self.assertTrue(all(s in original for s in remaining))
self.assertTrue(concrete_spec not in remaining)
# add it back and make sure everything is ok.
self.installed_db.add(concrete_spec, "")
installed = self.installed_db.query()
self.assertEqual(len(installed), len(original))
# sanity check against direcory layout and check ref counts.
self._check_db_sanity()
self.installed_db._check_ref_counts()
def test_060_remove_and_add_root_package(self):
self._check_remove_and_add_package('mpileaks ^mpich')
def test_070_remove_and_add_dependency_package(self):
self._check_remove_and_add_package('dyninst')
def test_080_root_ref_counts(self):
rec = self.installed_db.get_record('mpileaks ^mpich')
# Remove a top-level spec from the DB
self.installed_db.remove('mpileaks ^mpich')
# record no longer in DB
self.assertEqual(self.installed_db.query('mpileaks ^mpich', installed=any), [])
# record's deps have updated ref_counts
self.assertEqual(self.installed_db.get_record('callpath ^mpich').ref_count, 0)
self.assertEqual(self.installed_db.get_record('mpich').ref_count, 1)
# put the spec back
self.installed_db.add(rec.spec, rec.path)
# record is present again
self.assertEqual(len(self.installed_db.query('mpileaks ^mpich', installed=any)), 1)
# dependencies have ref counts updated
self.assertEqual(self.installed_db.get_record('callpath ^mpich').ref_count, 1)
self.assertEqual(self.installed_db.get_record('mpich').ref_count, 2)
def test_090_non_root_ref_counts(self):
mpileaks_mpich_rec = self.installed_db.get_record('mpileaks ^mpich')
callpath_mpich_rec = self.installed_db.get_record('callpath ^mpich')
# "force remove" a non-root spec from the DB
self.installed_db.remove('callpath ^mpich')
# record still in DB but marked uninstalled
self.assertEqual(self.installed_db.query('callpath ^mpich', installed=True), [])
self.assertEqual(len(self.installed_db.query('callpath ^mpich', installed=any)), 1)
# record and its deps have same ref_counts
self.assertEqual(self.installed_db.get_record('callpath ^mpich', installed=any).ref_count, 1)
self.assertEqual(self.installed_db.get_record('mpich').ref_count, 2)
# remove only dependent of uninstalled callpath record
self.installed_db.remove('mpileaks ^mpich')
# record and parent are completely gone.
self.assertEqual(self.installed_db.query('mpileaks ^mpich', installed=any), [])
self.assertEqual(self.installed_db.query('callpath ^mpich', installed=any), [])
# mpich ref count updated properly.
mpich_rec = self.installed_db.get_record('mpich')
self.assertEqual(mpich_rec.ref_count, 0)
|