# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import errno
import os
import shutil
import pytest
import spack.binary_distribution
import spack.cmd.buildcache
import spack.environment as ev
import spack.main
import spack.spec
import spack.util.url
from spack.spec import Spec
buildcache = spack.main.SpackCommand("buildcache")
install = spack.main.SpackCommand("install")
env = spack.main.SpackCommand("env")
add = spack.main.SpackCommand("add")
gpg = spack.main.SpackCommand("gpg")
mirror = spack.main.SpackCommand("mirror")
uninstall = spack.main.SpackCommand("uninstall")
pytestmark = pytest.mark.not_on_windows("does not run on windows")
@pytest.fixture()
def mock_get_specs(database, monkeypatch):
specs = database.query_local()
monkeypatch.setattr(spack.binary_distribution, "update_cache_and_get_specs", lambda: specs)
@pytest.fixture()
def mock_get_specs_multiarch(database, monkeypatch):
specs = [spec.copy() for spec in database.query_local()]
# make one spec that is NOT the test architecture
for spec in specs:
if spec.name == "mpileaks":
spec.architecture = spack.spec.ArchSpec("linux-rhel7-x86_64")
break
monkeypatch.setattr(spack.binary_distribution, "update_cache_and_get_specs", lambda: specs)
def test_buildcache_preview_just_runs():
# TODO: remove in Spack 0.21
buildcache("preview", "mpileaks")
@pytest.mark.db
@pytest.mark.regression("13757")
def test_buildcache_list_duplicates(mock_get_specs, capsys):
with capsys.disabled():
output = buildcache("list", "mpileaks", "@2.3")
assert output.count("mpileaks") == 3
@pytest.mark.db
@pytest.mark.regression("17827")
def test_buildcache_list_allarch(database, mock_get_specs_multiarch, capsys):
with capsys.disabled():
output = buildcache("list", "--allarch")
assert output.count("mpileaks") == 3
with capsys.disabled():
output = buildcache("list")
assert output.count("mpileaks") == 2
def tests_buildcache_create(install_mockery, mock_fetch, monkeypatch, tmpdir):
""" "Ensure that buildcache create creates output files"""
pkg = "trivial-install-test-package"
install(pkg)
buildcache("push", "--unsigned", str(tmpdir), pkg)
spec = Spec(pkg).concretized()
tarball_path = spack.binary_distribution.tarball_path_name(spec, ".spack")
tarball = spack.binary_distribution.tarball_name(spec, ".spec.json")
assert os.path.exists(os.path.join(str(tmpdir), "build_cache", tarball_path))
assert os.path.exists(os.path.join(str(tmpdir), "build_cache", tarball))
def tests_buildcache_create_env(
install_mockery, mock_fetch, monkeypatch, tmpdir, mutable_mock_env_path
):
""" "Ensure that buildcache create creates output files from env"""
pkg = "trivial-install-test-package"
env("create", "test")
with ev.read("test"):
add(pkg)
install()
buildcache("push", "--unsigned", str(tmpdir))
spec = Spec(pkg).concretized()
tarball_path = spack.binary_distribution.tarball_path_name(spec, ".spack")
tarball = spack.binary_distribution.tarball_name(spec, ".spec.json")
assert os.path.exists(os.path.join(str(tmpdir), "build_cache", tarball_path))
assert os.path.exists(os.path.join(str(tmpdir), "build_cache", tarball))
def test_buildcache_create_fails_on_noargs(tmpdir):
"""Ensure that buildcache create fails when given no args or
environment."""
with pytest.raises(spack.main.SpackCommandError):
buildcache("push", "--unsigned", str(tmpdir))
def test_buildcache_create_fail_on_perm_denied(install_mockery, mock_fetch, monkeypatch, tmpdir):
"""Ensure that buildcache create fails on permission denied error."""
install("trivial-install-test-package")
tmpdir.chmod(0)
with pytest.raises(OSError) as error:
buildcache("push", "--unsigned", str(tmpdir), "trivial-install-test-package")
assert error.value.errno == errno.EACCES
tmpdir.chmod(0o700)
def test_update_key_index(
tmpdir,
mutable_mock_env_path,
install_mockery,
mock_packages,
mock_fetch,
mock_stage,
mock_gnupghome,
):
"""Test the update-index command with the --keys option"""
working_dir = tmpdir.join("working_dir")
mirror_dir = working_dir.join("mirror")
mirror_url = "file://{0}".format(mirror_dir.strpath)
mirror("add", "test-mirror", mirror_url)
gpg("create", "Test Signing Key", "nobody@nowhere.com")
s = Spec("libdwarf").concretized()
# Install a package
install(s.name)
# Put installed package in the buildcache, which, because we're signing
# it, should result in the public key getting pushed to the buildcache
# as well.
buildcache("push", mirror_dir.strpath, s.name)
# Now make sure that when we pass the "--keys" argument to update-index
# it causes the index to get update.
buildcache("update-index", "--keys", mirror_dir.strpath)
key_dir_list = os.listdir(os.path.join(mirror_dir.strpath, "build_cache", "_pgp"))
uninstall("-y", s.name)
mirror("rm", "test-mirror")
assert "index.json" in key_dir_list
def test_buildcache_sync(
mutable_mock_env_path,
install_mockery_mutable_config,
mock_packages,
mock_fetch,
mock_stage,
tmpdir,
):
"""
Make sure buildcache sync works in an environment-aware manner, ignoring
any specs that may be in the mirror but not in the environment.
"""
working_dir = tmpdir.join("working_dir")
src_mirror_dir = working_dir.join("src_mirror").strpath
src_mirror_url = "file://{0}".format(src_mirror_dir)
dest_mirror_dir = working_dir.join("dest_mirror").strpath
dest_mirror_url = "file://{0}".format(dest_mirror_dir)
in_env_pkg = "trivial-install-test-package"
out_env_pkg = "libdwarf"
def verify_mirror_contents():
dest_list = os.listdir(os.path.join(dest_mirror_dir, "build_cache"))
found_pkg = False
for p in dest_list:
assert out_env_pkg not in p
if in_env_pkg in p:
found_pkg = True
if not found_pkg:
print("Expected to find {0} in {1}".format(in_env_pkg, dest_mirror_dir))
assert False
# Install a package and put it in the buildcache
s = Spec(out_env_pkg).concretized()
install(s.name)
buildcache("push", "-u", "-f", src_mirror_url, s.name)
env("create", "test")
with ev.read("test"):
add(in_env_pkg)
install()
buildcache("push", "-u", "-f", src_mirror_url, in_env_pkg)
# Now run the spack buildcache sync command with all the various options
# for specifying mirrors
# Use urls to specify mirrors
buildcache("sync", src_mirror_url, dest_mirror_url)
verify_mirror_contents()
shutil.rmtree(dest_mirror_dir)
# Use local directory paths to specify fs locations
buildcache("sync", src_mirror_dir, dest_mirror_dir)
verify_mirror_contents()
shutil.rmtree(dest_mirror_dir)
# Use mirror names to specify mirrors
mirror("add", "src", src_mirror_url)
mirror("add", "dest", dest_mirror_url)
buildcache("sync", "src", "dest")
verify_mirror_contents()
def test_buildcache_create_install(
mutable_mock_env_path,
install_mockery_mutable_config,
mock_packages,
mock_fetch,
mock_stage,
monkeypatch,
tmpdir,
):
""" "Ensure that buildcache create creates output files"""
pkg = "trivial-install-test-package"
install(pkg)
buildcache("push", "--unsigned", str(tmpdir), pkg)
spec = Spec(pkg).concretized()
tarball_path = spack.binary_distribution.tarball_path_name(spec, ".spack")
tarball = spack.binary_distribution.tarball_name(spec, ".spec.json")
assert os.path.exists(os.path.join(str(tmpdir), "build_cache", tarball_path))
assert os.path.exists(os.path.join(str(tmpdir), "build_cache", tarball))
@pytest.mark.parametrize(
"things_to_install,expected",
[
(
"",
[
"dttop",
"dtbuild1",
"dtbuild2",
"dtlink2",
"dtrun2",
"dtlink1",
"dtlink3",
"dtlink4",
"dtrun1",
"dtlink5",
"dtrun3",
"dtbuild3",
],
),
(
"dependencies",
[
"dtbuild1",
"dtbuild2",
"dtlink2",
"dtrun2",
"dtlink1",
"dtlink3",
"dtlink4",
"dtrun1",
"dtlink5",
"dtrun3",
"dtbuild3",
],
),
("package", ["dttop"]),
],
)
def test_correct_specs_are_pushed(
things_to_install, expected, tmpdir, monkeypatch, default_mock_concretization, temporary_store
):
# Concretize dttop and add it to the temporary database (without prefixes)
spec = default_mock_concretization("dttop")
temporary_store.db.add(spec, directory_layout=None)
slash_hash = "/{0}".format(spec.dag_hash())
packages_to_push = []
def fake_push(node, push_url, options):
assert isinstance(node, Spec)
packages_to_push.append(node.name)
monkeypatch.setattr(spack.binary_distribution, "push_or_raise", fake_push)
buildcache_create_args = ["create", "--unsigned"]
if things_to_install != "":
buildcache_create_args.extend(["--only", things_to_install])
buildcache_create_args.extend([str(tmpdir), slash_hash])
buildcache(*buildcache_create_args)
# Order is not guaranteed, so we can't just compare lists
assert set(packages_to_push) == set(expected)
# Ensure no duplicates
assert len(set(packages_to_push)) == len(packages_to_push)
@pytest.mark.parametrize("signed", [True, False])
def test_push_and_install_with_mirror_marked_unsigned_does_not_require_extra_flags(
tmp_path, mutable_database, mock_gnupghome, signed
):
"""Tests whether marking a mirror as unsigned makes it possible to push and install to/from
it without requiring extra flags on the command line (and no signing keys configured)."""
# Create a named mirror with signed set to True or False
add_flag = "--signed" if signed else "--unsigned"
mirror("add", add_flag, "my-mirror", str(tmp_path))
spec = mutable_database.query_local("libelf", installed=True)[0]
# Push
if signed:
# Need to pass "--unsigned" to override the mirror's default
args = ["push", "--update-index", "--unsigned", "my-mirror", f"/{spec.dag_hash()}"]
else:
# No need to pass "--unsigned" if the mirror is unsigned
args = ["push", "--update-index", "my-mirror", f"/{spec.dag_hash()}"]
buildcache(*args)
# Install
if signed:
# Need to pass "--no-check-signature" to avoid install errors
kwargs = {"cache_only": True, "unsigned": True}
else:
# No need to pass "--no-check-signature" if the mirror is unsigned
kwargs = {"cache_only": True}
spec.package.do_uninstall(force=True)
spec.package.do_install(**kwargs)