# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ This test checks the binary packaging infrastructure """ import argparse import os import pathlib import platform import shutil from collections import OrderedDict import pytest from llnl.util import filesystem as fs from llnl.util.symlink import symlink import spack.binary_distribution as bindist import spack.cmd.buildcache as buildcache import spack.error import spack.package_base import spack.repo import spack.store import spack.util.gpg import spack.util.url as url_util from spack.fetch_strategy import URLFetchStrategy from spack.paths import mock_gpg_keys_path from spack.relocate import ( macho_find_paths, macho_make_paths_normal, macho_make_paths_relative, needs_binary_relocation, needs_text_relocation, relocate_links, relocate_text, ) from spack.spec import Spec pytestmark = pytest.mark.not_on_windows("does not run on windows") @pytest.mark.usefixtures("install_mockery", "mock_gnupghome") def test_buildcache(mock_archive, tmp_path, monkeypatch, mutable_config): # Install a test package spec = Spec("trivial-install-test-package").concretized() monkeypatch.setattr(spec.package, "fetcher", URLFetchStrategy(mock_archive.url)) spec.package.do_install() pkghash = "/" + str(spec.dag_hash(7)) # Put some non-relocatable file in there dummy_txt = pathlib.Path(spec.prefix) / "dummy.txt" dummy_txt.write_text(spec.prefix) # Create an absolute symlink linkname = os.path.join(spec.prefix, "link_to_dummy.txt") symlink(dummy_txt, linkname) # Create the build cache and put it directly into the mirror mirror_path = str(tmp_path / "test-mirror") spack.mirror.create(mirror_path, specs=[]) # register mirror with spack config mirrors = {"spack-mirror-test": url_util.path_to_file_url(mirror_path)} spack.config.set("mirrors", mirrors) with spack.stage.Stage(mirrors["spack-mirror-test"], name="build_cache", keep=True): parser = argparse.ArgumentParser() buildcache.setup_parser(parser) create_args = ["create", "-f", "--rebuild-index", mirror_path, pkghash] # Create a private key to sign package with if gpg2 available spack.util.gpg.create( name="test key 1", expires="0", email="spack@googlegroups.com", comment="Spack test key", ) args = parser.parse_args(create_args) buildcache.buildcache(parser, args) # trigger overwrite warning buildcache.buildcache(parser, args) # Uninstall the package spec.package.do_uninstall(force=True) install_args = ["install", "-f", pkghash] args = parser.parse_args(install_args) # Test install buildcache.buildcache(parser, args) files = os.listdir(spec.prefix) assert "link_to_dummy.txt" in files assert "dummy.txt" in files # Validate the relocation information buildinfo = bindist.read_buildinfo_file(spec.prefix) assert buildinfo["relocate_textfiles"] == ["dummy.txt"] assert buildinfo["relocate_links"] == ["link_to_dummy.txt"] args = parser.parse_args(["keys"]) buildcache.buildcache(parser, args) args = parser.parse_args(["list"]) buildcache.buildcache(parser, args) args = parser.parse_args(["list"]) buildcache.buildcache(parser, args) args = parser.parse_args(["list", "trivial"]) buildcache.buildcache(parser, args) # Copy a key to the mirror to have something to download shutil.copyfile(mock_gpg_keys_path + "/external.key", mirror_path + "/external.key") args = parser.parse_args(["keys"]) buildcache.buildcache(parser, args) args = parser.parse_args(["keys", "-f"]) buildcache.buildcache(parser, args) args = parser.parse_args(["keys", "-i", "-t"]) buildcache.buildcache(parser, args) def test_relocate_text(tmp_path): """Tests that a text file containing the original directory of an installation, can be relocated to a target directory. """ original_dir = "/home/spack/opt/spack" relocation_dir = "/opt/rh/devtoolset/" dummy_txt = tmp_path / "dummy.txt" dummy_txt.write_text(original_dir) relocate_text([str(dummy_txt)], {original_dir: relocation_dir}) text = dummy_txt.read_text() assert relocation_dir in text assert original_dir not in text def test_relocate_links(tmpdir): tmpdir.ensure("new_prefix_a", dir=True) own_prefix_path = str(tmpdir.join("prefix_a", "file")) dep_prefix_path = str(tmpdir.join("prefix_b", "file")) new_own_prefix_path = str(tmpdir.join("new_prefix_a", "file")) new_dep_prefix_path = str(tmpdir.join("new_prefix_b", "file")) system_path = os.path.join(os.path.sep, "system", "path") fs.touchp(own_prefix_path) fs.touchp(new_own_prefix_path) fs.touchp(dep_prefix_path) fs.touchp(new_dep_prefix_path) # Old prefixes to new prefixes prefix_to_prefix = OrderedDict( [ # map /prefix_a -> /new_prefix_a (str(tmpdir.join("prefix_a")), str(tmpdir.join("new_prefix_a"))), # map /prefix_b -> /new_prefix_b (str(tmpdir.join("prefix_b")), str(tmpdir.join("new_prefix_b"))), # map -> /fallback/path -- this is just to see we respect order. (str(tmpdir), os.path.join(os.path.sep, "fallback", "path")), ] ) with tmpdir.join("new_prefix_a").as_cwd(): # To be relocated os.symlink(own_prefix_path, "to_self") os.symlink(dep_prefix_path, "to_dependency") # To be ignored os.symlink(system_path, "to_system") os.symlink("relative", "to_self_but_relative") relocate_links(["to_self", "to_dependency", "to_system"], prefix_to_prefix) # These two are relocated assert os.readlink("to_self") == str(tmpdir.join("new_prefix_a", "file")) assert os.readlink("to_dependency") == str(tmpdir.join("new_prefix_b", "file")) # These two are not. assert os.readlink("to_system") == system_path assert os.readlink("to_self_but_relative") == "relative" def test_needs_relocation(): assert needs_binary_relocation("application", "x-sharedlib") assert needs_binary_relocation("application", "x-executable") assert not needs_binary_relocation("application", "x-octet-stream") assert not needs_binary_relocation("text", "x-") assert needs_text_relocation("text", "x-") assert not needs_text_relocation("symbolic link to", "x-") assert needs_binary_relocation("application", "x-mach-binary") def test_replace_paths(tmpdir): with tmpdir.as_cwd(): suffix = "dylib" if platform.system().lower() == "darwin" else "so" hash_a = "53moz6jwnw3xpiztxwhc4us26klribws" hash_b = "tk62dzu62kd4oh3h3heelyw23hw2sfee" hash_c = "hdkhduizmaddpog6ewdradpobnbjwsjl" hash_d = "hukkosc7ahff7o65h6cdhvcoxm57d4bw" hash_loco = "zy4oigsc4eovn5yhr2lk4aukwzoespob" prefix2hash = {} old_spack_dir = os.path.join(f"{tmpdir}", "Users", "developer", "spack") fs.mkdirp(old_spack_dir) oldprefix_a = os.path.join(f"{old_spack_dir}", f"pkgA-{hash_a}") oldlibdir_a = os.path.join(f"{oldprefix_a}", "lib") fs.mkdirp(oldlibdir_a) prefix2hash[str(oldprefix_a)] = hash_a oldprefix_b = os.path.join(f"{old_spack_dir}", f"pkgB-{hash_b}") oldlibdir_b = os.path.join(f"{oldprefix_b}", "lib") fs.mkdirp(oldlibdir_b) prefix2hash[str(oldprefix_b)] = hash_b oldprefix_c = os.path.join(f"{old_spack_dir}", f"pkgC-{hash_c}") oldlibdir_c = os.path.join(f"{oldprefix_c}", "lib") oldlibdir_cc = os.path.join(f"{oldlibdir_c}", "C") fs.mkdirp(oldlibdir_c) prefix2hash[str(oldprefix_c)] = hash_c oldprefix_d = os.path.join(f"{old_spack_dir}", f"pkgD-{hash_d}") oldlibdir_d = os.path.join(f"{oldprefix_d}", "lib") fs.mkdirp(oldlibdir_d) prefix2hash[str(oldprefix_d)] = hash_d oldprefix_local = os.path.join(f"{tmpdir}", "usr", "local") oldlibdir_local = os.path.join(f"{oldprefix_local}", "lib") fs.mkdirp(oldlibdir_local) prefix2hash[str(oldprefix_local)] = hash_loco libfile_a = f"libA.{suffix}" libfile_b = f"libB.{suffix}" libfile_c = f"libC.{suffix}" libfile_d = f"libD.{suffix}" libfile_loco = f"libloco.{suffix}" old_libnames = [ os.path.join(oldlibdir_a, libfile_a), os.path.join(oldlibdir_b, libfile_b), os.path.join(oldlibdir_c, libfile_c), os.path.join(oldlibdir_d, libfile_d), os.path.join(oldlibdir_local, libfile_loco), ] for old_libname in old_libnames: with open(old_libname, "a"): os.utime(old_libname, None) hash2prefix = dict() new_spack_dir = os.path.join(f"{tmpdir}", "Users", "Shared", "spack") fs.mkdirp(new_spack_dir) prefix_a = os.path.join(new_spack_dir, f"pkgA-{hash_a}") libdir_a = os.path.join(prefix_a, "lib") fs.mkdirp(libdir_a) hash2prefix[hash_a] = str(prefix_a) prefix_b = os.path.join(new_spack_dir, f"pkgB-{hash_b}") libdir_b = os.path.join(prefix_b, "lib") fs.mkdirp(libdir_b) hash2prefix[hash_b] = str(prefix_b) prefix_c = os.path.join(new_spack_dir, f"pkgC-{hash_c}") libdir_c = os.path.join(prefix_c, "lib") libdir_cc = os.path.join(libdir_c, "C") fs.mkdirp(libdir_cc) hash2prefix[hash_c] = str(prefix_c) prefix_d = os.path.join(new_spack_dir, f"pkgD-{hash_d}") libdir_d = os.path.join(prefix_d, "lib") fs.mkdirp(libdir_d) hash2prefix[hash_d] = str(prefix_d) prefix_local = os.path.join(f"{tmpdir}", "usr", "local") libdir_local = os.path.join(prefix_local, "lib") fs.mkdirp(libdir_local) hash2prefix[hash_loco] = str(prefix_local) new_libnames = [ os.path.join(libdir_a, libfile_a), os.path.join(libdir_b, libfile_b), os.path.join(libdir_cc, libfile_c), os.path.join(libdir_d, libfile_d), os.path.join(libdir_local, libfile_loco), ] for new_libname in new_libnames: with open(new_libname, "a"): os.utime(new_libname, None) prefix2prefix = dict() for prefix, hash in prefix2hash.items(): prefix2prefix[prefix] = hash2prefix[hash] out_dict = macho_find_paths( [oldlibdir_a, oldlibdir_b, oldlibdir_c, oldlibdir_cc, oldlibdir_local], [ os.path.join(oldlibdir_a, libfile_a), os.path.join(oldlibdir_b, libfile_b), os.path.join(oldlibdir_local, libfile_loco), ], os.path.join(oldlibdir_cc, libfile_c), old_spack_dir, prefix2prefix, ) assert out_dict == { oldlibdir_a: libdir_a, oldlibdir_b: libdir_b, oldlibdir_c: libdir_c, oldlibdir_cc: libdir_cc, libdir_local: libdir_local, os.path.join(oldlibdir_a, libfile_a): os.path.join(libdir_a, libfile_a), os.path.join(oldlibdir_b, libfile_b): os.path.join(libdir_b, libfile_b), os.path.join(oldlibdir_local, libfile_loco): os.path.join(libdir_local, libfile_loco), os.path.join(oldlibdir_cc, libfile_c): os.path.join(libdir_cc, libfile_c), } out_dict = macho_find_paths( [oldlibdir_a, oldlibdir_b, oldlibdir_c, oldlibdir_cc, oldlibdir_local], [ os.path.join(oldlibdir_a, libfile_a), os.path.join(oldlibdir_b, libfile_b), os.path.join(oldlibdir_cc, libfile_c), os.path.join(oldlibdir_local, libfile_loco), ], None, old_spack_dir, prefix2prefix, ) assert out_dict == { oldlibdir_a: libdir_a, oldlibdir_b: libdir_b, oldlibdir_c: libdir_c, oldlibdir_cc: libdir_cc, libdir_local: libdir_local, os.path.join(oldlibdir_a, libfile_a): os.path.join(libdir_a, libfile_a), os.path.join(oldlibdir_b, libfile_b): os.path.join(libdir_b, libfile_b), os.path.join(oldlibdir_local, libfile_loco): os.path.join(libdir_local, libfile_loco), os.path.join(oldlibdir_cc, libfile_c): os.path.join(libdir_cc, libfile_c), } out_dict = macho_find_paths( [oldlibdir_a, oldlibdir_b, oldlibdir_c, oldlibdir_cc, oldlibdir_local], [ f"@rpath/{libfile_a}", f"@rpath/{libfile_b}", f"@rpath/{libfile_c}", f"@rpath/{libfile_loco}", ], None, old_spack_dir, prefix2prefix, ) assert out_dict == { f"@rpath/{libfile_a}": f"@rpath/{libfile_a}", f"@rpath/{libfile_b}": f"@rpath/{libfile_b}", f"@rpath/{libfile_c}": f"@rpath/{libfile_c}", f"@rpath/{libfile_loco}": f"@rpath/{libfile_loco}", oldlibdir_a: libdir_a, oldlibdir_b: libdir_b, oldlibdir_c: libdir_c, oldlibdir_cc: libdir_cc, libdir_local: libdir_local, } out_dict = macho_find_paths( [oldlibdir_a, oldlibdir_b, oldlibdir_d, oldlibdir_local], [f"@rpath/{libfile_a}", f"@rpath/{libfile_b}", f"@rpath/{libfile_loco}"], None, old_spack_dir, prefix2prefix, ) assert out_dict == { f"@rpath/{libfile_a}": f"@rpath/{libfile_a}", f"@rpath/{libfile_b}": f"@rpath/{libfile_b}", f"@rpath/{libfile_loco}": f"@rpath/{libfile_loco}", oldlibdir_a: libdir_a, oldlibdir_b: libdir_b, oldlibdir_d: libdir_d, libdir_local: libdir_local, } def test_macho_make_paths(): out = macho_make_paths_relative( "/Users/Shared/spack/pkgC/lib/libC.dylib", "/Users/Shared/spack", ("/Users/Shared/spack/pkgA/lib", "/Users/Shared/spack/pkgB/lib", "/usr/local/lib"), ( "/Users/Shared/spack/pkgA/libA.dylib", "/Users/Shared/spack/pkgB/libB.dylib", "/usr/local/lib/libloco.dylib", ), "/Users/Shared/spack/pkgC/lib/libC.dylib", ) assert out == { "/Users/Shared/spack/pkgA/lib": "@loader_path/../../pkgA/lib", "/Users/Shared/spack/pkgB/lib": "@loader_path/../../pkgB/lib", "/usr/local/lib": "/usr/local/lib", "/Users/Shared/spack/pkgA/libA.dylib": "@loader_path/../../pkgA/libA.dylib", "/Users/Shared/spack/pkgB/libB.dylib": "@loader_path/../../pkgB/libB.dylib", "/usr/local/lib/libloco.dylib": "/usr/local/lib/libloco.dylib", "/Users/Shared/spack/pkgC/lib/libC.dylib": "@rpath/libC.dylib", } out = macho_make_paths_normal( "/Users/Shared/spack/pkgC/lib/libC.dylib", ("@loader_path/../../pkgA/lib", "@loader_path/../../pkgB/lib", "/usr/local/lib"), ( "@loader_path/../../pkgA/libA.dylib", "@loader_path/../../pkgB/libB.dylib", "/usr/local/lib/libloco.dylib", ), "@rpath/libC.dylib", ) assert out == { "@rpath/libC.dylib": "/Users/Shared/spack/pkgC/lib/libC.dylib", "@loader_path/../../pkgA/lib": "/Users/Shared/spack/pkgA/lib", "@loader_path/../../pkgB/lib": "/Users/Shared/spack/pkgB/lib", "/usr/local/lib": "/usr/local/lib", "@loader_path/../../pkgA/libA.dylib": "/Users/Shared/spack/pkgA/libA.dylib", "@loader_path/../../pkgB/libB.dylib": "/Users/Shared/spack/pkgB/libB.dylib", "/usr/local/lib/libloco.dylib": "/usr/local/lib/libloco.dylib", } out = macho_make_paths_relative( "/Users/Shared/spack/pkgC/bin/exeC", "/Users/Shared/spack", ("/Users/Shared/spack/pkgA/lib", "/Users/Shared/spack/pkgB/lib", "/usr/local/lib"), ( "/Users/Shared/spack/pkgA/libA.dylib", "/Users/Shared/spack/pkgB/libB.dylib", "/usr/local/lib/libloco.dylib", ), None, ) assert out == { "/Users/Shared/spack/pkgA/lib": "@loader_path/../../pkgA/lib", "/Users/Shared/spack/pkgB/lib": "@loader_path/../../pkgB/lib", "/usr/local/lib": "/usr/local/lib", "/Users/Shared/spack/pkgA/libA.dylib": "@loader_path/../../pkgA/libA.dylib", "/Users/Shared/spack/pkgB/libB.dylib": "@loader_path/../../pkgB/libB.dylib", "/usr/local/lib/libloco.dylib": "/usr/local/lib/libloco.dylib", } out = macho_make_paths_normal( "/Users/Shared/spack/pkgC/bin/exeC", ("@loader_path/../../pkgA/lib", "@loader_path/../../pkgB/lib", "/usr/local/lib"), ( "@loader_path/../../pkgA/libA.dylib", "@loader_path/../../pkgB/libB.dylib", "/usr/local/lib/libloco.dylib", ), None, ) assert out == { "@loader_path/../../pkgA/lib": "/Users/Shared/spack/pkgA/lib", "@loader_path/../../pkgB/lib": "/Users/Shared/spack/pkgB/lib", "/usr/local/lib": "/usr/local/lib", "@loader_path/../../pkgA/libA.dylib": "/Users/Shared/spack/pkgA/libA.dylib", "@loader_path/../../pkgB/libB.dylib": "/Users/Shared/spack/pkgB/libB.dylib", "/usr/local/lib/libloco.dylib": "/usr/local/lib/libloco.dylib", } @pytest.fixture() def mock_download(): """Mock a failing download strategy.""" class FailedDownloadStrategy(spack.fetch_strategy.FetchStrategy): def mirror_id(self): return None def fetch(self): raise spack.fetch_strategy.FailedDownloadError( "", "This FetchStrategy always fails" ) fetcher = FailedDownloadStrategy() @property def fake_fn(self): return fetcher orig_fn = spack.package_base.PackageBase.fetcher spack.package_base.PackageBase.fetcher = fake_fn yield spack.package_base.PackageBase.fetcher = orig_fn @pytest.mark.parametrize( "manual,instr", [(False, False), (False, True), (True, False), (True, True)] ) @pytest.mark.disable_clean_stage_check def test_manual_download( install_mockery, mock_download, default_mock_concretization, monkeypatch, manual, instr ): """ Ensure expected fetcher fail message based on manual download and instr. """ @property def _instr(pkg): return f"Download instructions for {pkg.spec.name}" spec = default_mock_concretization("a") spec.package.manual_download = manual if instr: monkeypatch.setattr(spack.package_base.PackageBase, "download_instr", _instr) expected = spec.package.download_instr if manual else "All fetchers failed" with pytest.raises(spack.error.FetchError, match=expected): spec.package.do_fetch() @pytest.fixture() def fetching_not_allowed(monkeypatch): class FetchingNotAllowed(spack.fetch_strategy.FetchStrategy): def mirror_id(self): return None def fetch(self): raise Exception("Sources are fetched but shouldn't have been") monkeypatch.setattr(spack.package_base.PackageBase, "fetcher", FetchingNotAllowed()) def test_fetch_without_code_is_noop( default_mock_concretization, install_mockery, fetching_not_allowed ): """do_fetch for packages without code should be a no-op""" pkg = default_mock_concretization("a").package pkg.has_code = False pkg.do_fetch() def test_fetch_external_package_is_noop( default_mock_concretization, install_mockery, fetching_not_allowed ): """do_fetch for packages without code should be a no-op""" spec = default_mock_concretization("a") spec.external_path = "/some/where" assert spec.external spec.package.do_fetch()