1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
|
# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import shutil
import sys
import pytest
import llnl.util.filesystem as fs
import spack.error
import spack.patch
import spack.repo
import spack.store
import spack.util.spack_json as sjson
from spack.package_base import (
InstallError,
PackageBase,
PackageStillNeededError,
_spack_build_envfile,
_spack_build_logfile,
_spack_configure_argsfile,
spack_times_log,
)
from spack.spec import Spec
def find_nothing(*args):
raise spack.repo.UnknownPackageError("Repo package access is disabled for test")
def test_install_and_uninstall(install_mockery, mock_fetch, monkeypatch):
spec = Spec("trivial-install-test-package").concretized()
spec.package.do_install()
assert spec.installed
spec.package.do_uninstall()
assert not spec.installed
@pytest.mark.regression("11870")
def test_uninstall_non_existing_package(install_mockery, mock_fetch, monkeypatch):
"""Ensure that we can uninstall a package that has been deleted from the repo"""
spec = Spec("trivial-install-test-package").concretized()
spec.package.do_install()
assert spec.installed
# Mock deletion of the package
spec._package = None
monkeypatch.setattr(spack.repo.PATH, "get", find_nothing)
with pytest.raises(spack.repo.UnknownPackageError):
spec.package
# Ensure we can uninstall it
PackageBase.uninstall_by_spec(spec)
assert not spec.installed
def test_pkg_attributes(install_mockery, mock_fetch, monkeypatch):
# Get a basic concrete spec for the dummy package.
spec = Spec("attributes-foo-app ^attributes-foo")
spec.concretize()
assert spec.concrete
pkg = spec.package
pkg.do_install()
foo = "attributes-foo"
assert spec["bar"].prefix == spec[foo].prefix
assert spec["baz"].prefix == spec[foo].prefix
assert spec[foo].home == spec[foo].prefix
assert spec["bar"].home == spec[foo].home
assert spec["baz"].home == spec[foo].prefix.baz
foo_headers = spec[foo].headers
# assert foo_headers.basenames == ['foo.h']
assert foo_headers.directories == [spec[foo].home.include]
bar_headers = spec["bar"].headers
# assert bar_headers.basenames == ['bar.h']
assert bar_headers.directories == [spec["bar"].home.include]
baz_headers = spec["baz"].headers
# assert baz_headers.basenames == ['baz.h']
assert baz_headers.directories == [spec["baz"].home.include]
lib_suffix = ".so"
if sys.platform == "win32":
lib_suffix = ".dll"
elif sys.platform == "darwin":
lib_suffix = ".dylib"
foo_libs = spec[foo].libs
assert foo_libs.basenames == ["libFoo" + lib_suffix]
assert foo_libs.directories == [spec[foo].home.lib64]
bar_libs = spec["bar"].libs
assert bar_libs.basenames == ["libFooBar" + lib_suffix]
assert bar_libs.directories == [spec["bar"].home.lib64]
baz_libs = spec["baz"].libs
assert baz_libs.basenames == ["libFooBaz" + lib_suffix]
assert baz_libs.directories == [spec["baz"].home.lib]
def mock_remove_prefix(*args):
raise MockInstallError("Intentional error", "Mock remove_prefix method intentionally fails")
class RemovePrefixChecker:
def __init__(self, wrapped_rm_prefix):
self.removed = False
self.wrapped_rm_prefix = wrapped_rm_prefix
def remove_prefix(self):
self.removed = True
self.wrapped_rm_prefix()
class MockStage:
def __init__(self, wrapped_stage):
self.wrapped_stage = wrapped_stage
self.test_destroyed = False
def __enter__(self):
self.create()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is None:
self.destroy()
def destroy(self):
self.test_destroyed = True
self.wrapped_stage.destroy()
def create(self):
self.wrapped_stage.create()
def __getattr__(self, attr):
if attr == "wrapped_stage":
# This attribute may not be defined at some point during unpickling
raise AttributeError()
return getattr(self.wrapped_stage, attr)
def test_partial_install_delete_prefix_and_stage(install_mockery, mock_fetch, working_env):
s = Spec("canfail").concretized()
instance_rm_prefix = s.package.remove_prefix
try:
s.package.remove_prefix = mock_remove_prefix
with pytest.raises(MockInstallError):
s.package.do_install()
assert os.path.isdir(s.package.prefix)
rm_prefix_checker = RemovePrefixChecker(instance_rm_prefix)
s.package.remove_prefix = rm_prefix_checker.remove_prefix
# must clear failure markings for the package before re-installing it
spack.store.STORE.failure_tracker.clear(s, True)
s.package.set_install_succeed()
s.package.stage = MockStage(s.package.stage)
s.package.do_install(restage=True)
assert rm_prefix_checker.removed
assert s.package.stage.test_destroyed
assert s.package.spec.installed
finally:
s.package.remove_prefix = instance_rm_prefix
@pytest.mark.disable_clean_stage_check
def test_failing_overwrite_install_should_keep_previous_installation(
mock_fetch, install_mockery, working_env
):
"""
Make sure that whenever `spack install --overwrite` fails, spack restores
the original install prefix instead of cleaning it.
"""
# Do a successful install
s = Spec("canfail").concretized()
s.package.set_install_succeed()
# Do a failing overwrite install
s.package.do_install()
s.package.set_install_fail()
kwargs = {"overwrite": [s.dag_hash()]}
with pytest.raises(Exception):
s.package.do_install(**kwargs)
assert s.package.spec.installed
assert os.path.exists(s.prefix)
def test_dont_add_patches_to_installed_package(install_mockery, mock_fetch, monkeypatch):
dependency = Spec("dependency-install")
dependency.concretize()
dependency.package.do_install()
dependency_hash = dependency.dag_hash()
dependent = Spec("dependent-install ^/" + dependency_hash)
dependent.concretize()
monkeypatch.setitem(
dependency.package.patches,
"dependency-install",
[spack.patch.UrlPatch(dependent.package, "file://fake.patch", sha256="unused-hash")],
)
assert dependent["dependency-install"] == dependency
def test_installed_dependency_request_conflicts(install_mockery, mock_fetch, mutable_mock_repo):
dependency = Spec("dependency-install")
dependency.concretize()
dependency.package.do_install()
dependency_hash = dependency.dag_hash()
dependent = Spec("conflicting-dependent ^/" + dependency_hash)
with pytest.raises(spack.error.UnsatisfiableSpecError):
dependent.concretize()
def test_install_dependency_symlinks_pkg(install_mockery, mock_fetch, mutable_mock_repo):
"""Test dependency flattening/symlinks mock package."""
spec = Spec("flatten-deps")
spec.concretize()
pkg = spec.package
pkg.do_install()
# Ensure dependency directory exists after the installation.
dependency_dir = os.path.join(pkg.prefix, "dependency-install")
assert os.path.isdir(dependency_dir)
def test_install_times(install_mockery, mock_fetch, mutable_mock_repo):
"""Test install times added."""
spec = Spec("dev-build-test-install-phases").concretized()
spec.package.do_install()
# Ensure dependency directory exists after the installation.
install_times = os.path.join(spec.package.prefix, ".spack", spack_times_log)
assert os.path.isfile(install_times)
# Ensure the phases are included
with open(install_times, "r") as timefile:
times = sjson.load(timefile.read())
# The order should be maintained
phases = [x["name"] for x in times["phases"]]
assert phases == ["stage", "one", "two", "three", "install", "post-install"]
assert all(isinstance(x["seconds"], float) for x in times["phases"])
def test_flatten_deps(install_mockery, mock_fetch, mutable_mock_repo):
"""Explicitly test the flattening code for coverage purposes."""
# Unfortunately, executing the 'flatten-deps' spec's installation does
# not affect code coverage results, so be explicit here.
spec = Spec("dependent-install")
spec.concretize()
pkg = spec.package
pkg.do_install()
# Demonstrate that the directory does not appear under the spec
# prior to the flatten operation.
dependency_name = "dependency-install"
assert dependency_name not in os.listdir(pkg.prefix)
# Flatten the dependencies and ensure the dependency directory is there.
spack.package_base.flatten_dependencies(spec, pkg.prefix)
dependency_dir = os.path.join(pkg.prefix, dependency_name)
assert os.path.isdir(dependency_dir)
@pytest.fixture()
def install_upstream(tmpdir_factory, gen_mock_layout, install_mockery):
"""Provides a function that installs a specified set of specs to an
upstream database. The function returns a store which points to the
upstream, as well as the upstream layout (for verifying that dependent
installs are using the upstream installs).
"""
mock_db_root = str(tmpdir_factory.mktemp("mock_db_root"))
prepared_db = spack.database.Database(mock_db_root)
upstream_layout = gen_mock_layout("/a/")
spack.config.CONFIG.push_scope(
spack.config.InternalConfigScope(
name="install-upstream-fixture",
data={"upstreams": {"mock1": {"install_tree": prepared_db.root}}},
)
)
def _install_upstream(*specs):
for spec_str in specs:
s = spack.spec.Spec(spec_str).concretized()
prepared_db.add(s, upstream_layout)
downstream_root = str(tmpdir_factory.mktemp("mock_downstream_db_root"))
return downstream_root, upstream_layout
return _install_upstream
def test_installed_upstream_external(install_upstream, mock_fetch):
"""Check that when a dependency package is recorded as installed in
an upstream database that it is not reinstalled.
"""
store_root, _ = install_upstream("externaltool")
with spack.store.use_store(store_root):
dependent = spack.spec.Spec("externaltest")
dependent.concretize()
new_dependency = dependent["externaltool"]
assert new_dependency.external
assert new_dependency.prefix == os.path.sep + os.path.join("path", "to", "external_tool")
dependent.package.do_install()
assert not os.path.exists(new_dependency.prefix)
assert os.path.exists(dependent.prefix)
def test_installed_upstream(install_upstream, mock_fetch):
"""Check that when a dependency package is recorded as installed in
an upstream database that it is not reinstalled.
"""
store_root, upstream_layout = install_upstream("dependency-install")
with spack.store.use_store(store_root):
dependency = spack.spec.Spec("dependency-install").concretized()
dependent = spack.spec.Spec("dependent-install").concretized()
new_dependency = dependent["dependency-install"]
assert new_dependency.installed_upstream
assert new_dependency.prefix == upstream_layout.path_for_spec(dependency)
dependent.package.do_install()
assert not os.path.exists(new_dependency.prefix)
assert os.path.exists(dependent.prefix)
@pytest.mark.disable_clean_stage_check
def test_partial_install_keep_prefix(install_mockery, mock_fetch, monkeypatch, working_env):
s = Spec("canfail").concretized()
# If remove_prefix is called at any point in this test, that is an error
monkeypatch.setattr(spack.package_base.PackageBase, "remove_prefix", mock_remove_prefix)
with pytest.raises(spack.build_environment.ChildError):
s.package.do_install(keep_prefix=True)
assert os.path.exists(s.package.prefix)
# must clear failure markings for the package before re-installing it
spack.store.STORE.failure_tracker.clear(s, True)
s.package.set_install_succeed()
s.package.stage = MockStage(s.package.stage)
s.package.do_install(keep_prefix=True)
assert s.package.spec.installed
assert not s.package.stage.test_destroyed
def test_second_install_no_overwrite_first(install_mockery, mock_fetch, monkeypatch):
s = Spec("canfail").concretized()
monkeypatch.setattr(spack.package_base.PackageBase, "remove_prefix", mock_remove_prefix)
s.package.set_install_succeed()
s.package.do_install()
assert s.package.spec.installed
# If Package.install is called after this point, it will fail
s.package.set_install_fail()
s.package.do_install()
def test_install_prefix_collision_fails(config, mock_fetch, mock_packages, tmpdir):
"""
Test that different specs with coinciding install prefixes will fail
to install.
"""
projections = {"projections": {"all": "all-specs-project-to-this-prefix"}}
with spack.store.use_store(str(tmpdir), extra_data=projections):
with spack.config.override("config:checksum", False):
pkg_a = Spec("libelf@0.8.13").concretized().package
pkg_b = Spec("libelf@0.8.12").concretized().package
pkg_a.do_install()
with pytest.raises(InstallError, match="Install prefix collision"):
pkg_b.do_install()
def test_store(install_mockery, mock_fetch):
spec = Spec("cmake-client").concretized()
pkg = spec.package
pkg.do_install()
@pytest.mark.disable_clean_stage_check
def test_failing_build(install_mockery, mock_fetch, capfd):
spec = Spec("failing-build").concretized()
pkg = spec.package
with pytest.raises(spack.build_environment.ChildError, match="Expected failure"):
pkg.do_install()
class MockInstallError(spack.error.SpackError):
pass
def test_uninstall_by_spec_errors(mutable_database):
"""Test exceptional cases with the uninstall command."""
# Try to uninstall a spec that has not been installed
spec = Spec("dependent-install")
spec.concretize()
with pytest.raises(InstallError, match="is not installed"):
PackageBase.uninstall_by_spec(spec)
# Try an unforced uninstall of a spec with dependencies
rec = mutable_database.get_record("mpich")
with pytest.raises(PackageStillNeededError, match="Cannot uninstall"):
PackageBase.uninstall_by_spec(rec.spec)
@pytest.mark.disable_clean_stage_check
def test_nosource_pkg_install(install_mockery, mock_fetch, mock_packages, capfd, ensure_debug):
"""Test install phases with the nosource package."""
spec = Spec("nosource").concretized()
pkg = spec.package
# Make sure install works even though there is no associated code.
pkg.do_install()
out = capfd.readouterr()
assert "Installing dependency-install" in out[0]
# Make sure a warning for missing code is issued
assert "Missing a source id for nosource" in out[1]
@pytest.mark.disable_clean_stage_check
def test_nosource_bundle_pkg_install(
install_mockery, mock_fetch, mock_packages, capfd, ensure_debug
):
"""Test install phases with the nosource-bundle package."""
spec = Spec("nosource-bundle").concretized()
pkg = spec.package
# Make sure install works even though there is no associated code.
pkg.do_install()
out = capfd.readouterr()
assert "Installing dependency-install" in out[0]
# Make sure a warning for missing code is *not* issued
assert "Missing a source id for nosource" not in out[1]
def test_nosource_pkg_install_post_install(install_mockery, mock_fetch, mock_packages):
"""Test install phases with the nosource package with post-install."""
spec = Spec("nosource-install").concretized()
pkg = spec.package
# Make sure both the install and post-install package methods work.
pkg.do_install()
# Ensure the file created in the package's `install` method exists.
install_txt = os.path.join(spec.prefix, "install.txt")
assert os.path.isfile(install_txt)
# Ensure the file created in the package's `post-install` method exists.
post_install_txt = os.path.join(spec.prefix, "post-install.txt")
assert os.path.isfile(post_install_txt)
def test_pkg_build_paths(install_mockery):
# Get a basic concrete spec for the trivial install package.
spec = Spec("trivial-install-test-package").concretized()
log_path = spec.package.log_path
assert log_path.endswith(_spack_build_logfile)
env_path = spec.package.env_path
assert env_path.endswith(_spack_build_envfile)
# Backward compatibility checks
log_dir = os.path.dirname(log_path)
fs.mkdirp(log_dir)
with fs.working_dir(log_dir):
# Start with the older of the previous log filenames
older_log = "spack-build.out"
fs.touch(older_log)
assert spec.package.log_path.endswith(older_log)
# Now check the newer log filename
last_log = "spack-build.txt"
fs.rename(older_log, last_log)
assert spec.package.log_path.endswith(last_log)
# Check the old environment file
last_env = "spack-build.env"
fs.rename(last_log, last_env)
assert spec.package.env_path.endswith(last_env)
# Cleanup
shutil.rmtree(log_dir)
def test_pkg_install_paths(install_mockery):
# Get a basic concrete spec for the trivial install package.
spec = Spec("trivial-install-test-package").concretized()
log_path = os.path.join(spec.prefix, ".spack", _spack_build_logfile)
assert spec.package.install_log_path == log_path
env_path = os.path.join(spec.prefix, ".spack", _spack_build_envfile)
assert spec.package.install_env_path == env_path
args_path = os.path.join(spec.prefix, ".spack", _spack_configure_argsfile)
assert spec.package.install_configure_args_path == args_path
# Backward compatibility checks
log_dir = os.path.dirname(log_path)
fs.mkdirp(log_dir)
with fs.working_dir(log_dir):
# Start with the older of the previous install log filenames
older_log = "build.out"
fs.touch(older_log)
assert spec.package.install_log_path.endswith(older_log)
# Now check the newer install log filename
last_log = "build.txt"
fs.rename(older_log, last_log)
assert spec.package.install_log_path.endswith(last_log)
# Check the old install environment file
last_env = "build.env"
fs.rename(last_log, last_env)
assert spec.package.install_env_path.endswith(last_env)
# Cleanup
shutil.rmtree(log_dir)
def test_log_install_without_build_files(install_mockery):
"""Test the installer log function when no build files are present."""
# Get a basic concrete spec for the trivial install package.
spec = Spec("trivial-install-test-package").concretized()
# Attempt installing log without the build log file
with pytest.raises(IOError, match="No such file or directory"):
spack.installer.log(spec.package)
def test_log_install_with_build_files(install_mockery, monkeypatch):
"""Test the installer's log function when have build files."""
config_log = "config.log"
# Retain the original function for use in the monkey patch that is used
# to raise an exception under the desired condition for test coverage.
orig_install_fn = fs.install
def _install(src, dest):
orig_install_fn(src, dest)
if src.endswith(config_log):
raise Exception("Mock log install error")
monkeypatch.setattr(fs, "install", _install)
spec = Spec("trivial-install-test-package").concretized()
# Set up mock build files and try again to include archive failure
log_path = spec.package.log_path
log_dir = os.path.dirname(log_path)
fs.mkdirp(log_dir)
with fs.working_dir(log_dir):
fs.touch(log_path)
fs.touch(spec.package.env_path)
fs.touch(spec.package.env_mods_path)
fs.touch(spec.package.configure_args_path)
install_path = os.path.dirname(spec.package.install_log_path)
fs.mkdirp(install_path)
source = spec.package.stage.source_path
config = os.path.join(source, "config.log")
fs.touchp(config)
monkeypatch.setattr(
type(spec.package), "archive_files", ["missing", "..", config], raising=False
)
spack.installer.log(spec.package)
assert os.path.exists(spec.package.install_log_path)
assert os.path.exists(spec.package.install_env_path)
assert os.path.exists(spec.package.install_configure_args_path)
archive_dir = os.path.join(install_path, "archived-files")
source_dir = os.path.dirname(source)
rel_config = os.path.relpath(config, source_dir)
assert os.path.exists(os.path.join(archive_dir, rel_config))
assert not os.path.exists(os.path.join(archive_dir, "missing"))
expected_errs = ["OUTSIDE SOURCE PATH", "FAILED TO ARCHIVE"] # for '..' # for rel_config
with open(os.path.join(archive_dir, "errors.txt"), "r") as fd:
for ln, expected in zip(fd, expected_errs):
assert expected in ln
# Cleanup
shutil.rmtree(log_dir)
def test_unconcretized_install(install_mockery, mock_fetch, mock_packages):
"""Test attempts to perform install phases with unconcretized spec."""
spec = Spec("trivial-install-test-package")
pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)
with pytest.raises(ValueError, match="must have a concrete spec"):
pkg_cls(spec).do_install()
with pytest.raises(ValueError, match="only patch concrete packages"):
pkg_cls(spec).do_patch()
def test_install_error():
try:
msg = "test install error"
long_msg = "this is the long version of test install error"
raise InstallError(msg, long_msg=long_msg)
except Exception as exc:
assert exc.__class__.__name__ == "InstallError"
assert exc.message == msg
assert exc.long_message == long_msg
@pytest.mark.disable_clean_stage_check
def test_empty_install_sanity_check_prefix(
monkeypatch, install_mockery, mock_fetch, mock_packages
):
"""Test empty install triggers sanity_check_prefix."""
spec = Spec("failing-empty-install").concretized()
with pytest.raises(spack.build_environment.ChildError, match="Nothing was installed"):
spec.package.do_install()
|