path: root/lib/spack/spack/test/cmd/buildcache.py

# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

import errno
import os
import shutil

import pytest

import spack.binary_distribution
import spack.cmd.buildcache
import spack.environment as ev
import spack.main
import spack.spec
import spack.util.url
from spack.spec import Spec

buildcache = spack.main.SpackCommand("buildcache")
install = spack.main.SpackCommand("install")
env = spack.main.SpackCommand("env")
add = spack.main.SpackCommand("add")
gpg = spack.main.SpackCommand("gpg")
mirror = spack.main.SpackCommand("mirror")
uninstall = spack.main.SpackCommand("uninstall")

pytestmark = pytest.mark.not_on_windows("does not run on windows")


@pytest.fixture()
def mock_get_specs(database, monkeypatch):
    specs = database.query_local()
    monkeypatch.setattr(spack.binary_distribution, "update_cache_and_get_specs", lambda: specs)


@pytest.fixture()
def mock_get_specs_multiarch(database, monkeypatch):
    specs = [spec.copy() for spec in database.query_local()]

    # make one spec that is NOT the test architecture
    for spec in specs:
        if spec.name == "mpileaks":
            spec.architecture = spack.spec.ArchSpec("linux-rhel7-x86_64")
            break

    monkeypatch.setattr(spack.binary_distribution, "update_cache_and_get_specs", lambda: specs)


def test_buildcache_preview_just_runs():
    # TODO: remove in Spack 0.21
    buildcache("preview", "mpileaks")


@pytest.mark.db
@pytest.mark.regression("13757")
def test_buildcache_list_duplicates(mock_get_specs, capsys):
    with capsys.disabled():
        output = buildcache("list", "mpileaks", "@2.3")

    assert output.count("mpileaks") == 3


@pytest.mark.db
@pytest.mark.regression("17827")
def test_buildcache_list_allarch(database, mock_get_specs_multiarch, capsys):
    with capsys.disabled():
        output = buildcache("list", "--allarch")

    assert output.count("mpileaks") == 3

    with capsys.disabled():
        output = buildcache("list")

    assert output.count("mpileaks") == 2


def tests_buildcache_create(install_mockery, mock_fetch, monkeypatch, tmpdir):
    """ "Ensure that buildcache create creates output files"""
    pkg = "trivial-install-test-package"
    install(pkg)

    buildcache("push", "--unsigned", str(tmpdir), pkg)

    spec = Spec(pkg).concretized()
    tarball_path = spack.binary_distribution.tarball_path_name(spec, ".spack")
    tarball = spack.binary_distribution.tarball_name(spec, ".spec.json")
    assert os.path.exists(os.path.join(str(tmpdir), "build_cache", tarball_path))
    assert os.path.exists(os.path.join(str(tmpdir), "build_cache", tarball))


def tests_buildcache_create_env(
    install_mockery, mock_fetch, monkeypatch, tmpdir, mutable_mock_env_path
):
    """ "Ensure that buildcache create creates output files from env"""
    pkg = "trivial-install-test-package"

    env("create", "test")
    with ev.read("test"):
        add(pkg)
        install()

        buildcache("push", "--unsigned", str(tmpdir))

    spec = Spec(pkg).concretized()
    tarball_path = spack.binary_distribution.tarball_path_name(spec, ".spack")
    tarball = spack.binary_distribution.tarball_name(spec, ".spec.json")
    assert os.path.exists(os.path.join(str(tmpdir), "build_cache", tarball_path))
    assert os.path.exists(os.path.join(str(tmpdir), "build_cache", tarball))


def test_buildcache_create_fails_on_noargs(tmpdir):
    """Ensure that buildcache create fails when given no args or
    environment."""
    with pytest.raises(spack.main.SpackCommandError):
        buildcache("push", "--unsigned", str(tmpdir))


def test_buildcache_create_fail_on_perm_denied(install_mockery, mock_fetch, monkeypatch, tmpdir):
    """Ensure that buildcache create fails on permission denied error."""
    install("trivial-install-test-package")

    tmpdir.chmod(0)
    with pytest.raises(OSError) as error:
        buildcache("push", "--unsigned", str(tmpdir), "trivial-install-test-package")
    assert error.value.errno == errno.EACCES
    tmpdir.chmod(0o700)


def test_update_key_index(
    tmpdir,
    mutable_mock_env_path,
    install_mockery,
    mock_packages,
    mock_fetch,
    mock_stage,
    mock_gnupghome,
):
    """Test the update-index command with the --keys option"""
    working_dir = tmpdir.join("working_dir")

    mirror_dir = working_dir.join("mirror")
    mirror_url = "file://{0}".format(mirror_dir.strpath)

    mirror("add", "test-mirror", mirror_url)

    gpg("create", "Test Signing Key", "nobody@nowhere.com")

    s = Spec("libdwarf").concretized()

    # Install a package
    install(s.name)

    # Put installed package in the buildcache, which, because we're signing
    # it, should result in the public key getting pushed to the buildcache
    # as well.
    buildcache("push", mirror_dir.strpath, s.name)

    # Now make sure that when we pass the "--keys" argument to update-index
    # it causes the index to get update.
    buildcache("update-index", "--keys", mirror_dir.strpath)

    key_dir_list = os.listdir(os.path.join(mirror_dir.strpath, "build_cache", "_pgp"))

    uninstall("-y", s.name)
    mirror("rm", "test-mirror")

    assert "index.json" in key_dir_list


def test_buildcache_sync(
    mutable_mock_env_path,
    install_mockery_mutable_config,
    mock_packages,
    mock_fetch,
    mock_stage,
    tmpdir,
):
    """
    Make sure buildcache sync works in an environment-aware manner, ignoring
    any specs that may be in the mirror but not in the environment.
    """
    working_dir = tmpdir.join("working_dir")

    src_mirror_dir = working_dir.join("src_mirror").strpath
    src_mirror_url = "file://{0}".format(src_mirror_dir)

    dest_mirror_dir = working_dir.join("dest_mirror").strpath
    dest_mirror_url = "file://{0}".format(dest_mirror_dir)

    in_env_pkg = "trivial-install-test-package"
    out_env_pkg = "libdwarf"

    def verify_mirror_contents():
        dest_list = os.listdir(os.path.join(dest_mirror_dir, "build_cache"))

        found_pkg = False

        for p in dest_list:
            assert out_env_pkg not in p
            if in_env_pkg in p:
                found_pkg = True

        if not found_pkg:
            print("Expected to find {0} in {1}".format(in_env_pkg, dest_mirror_dir))
            assert False

    # Install a package and put it in the buildcache
    s = Spec(out_env_pkg).concretized()
    install(s.name)
    buildcache("push", "-u", "-f", src_mirror_url, s.name)

    env("create", "test")
    with ev.read("test"):
        add(in_env_pkg)
        install()
        buildcache("push", "-u", "-f", src_mirror_url, in_env_pkg)

        # Now run the spack buildcache sync command with all the various options
        # for specifying mirrors

        # Use urls to specify mirrors
        buildcache("sync", src_mirror_url, dest_mirror_url)

        verify_mirror_contents()
        shutil.rmtree(dest_mirror_dir)

        # Use local directory paths to specify fs locations
        buildcache("sync", src_mirror_dir, dest_mirror_dir)

        verify_mirror_contents()
        shutil.rmtree(dest_mirror_dir)

        # Use mirror names to specify mirrors
        mirror("add", "src", src_mirror_url)
        mirror("add", "dest", dest_mirror_url)

        buildcache("sync", "src", "dest")

        verify_mirror_contents()


def test_buildcache_create_install(
    mutable_mock_env_path,
    install_mockery_mutable_config,
    mock_packages,
    mock_fetch,
    mock_stage,
    monkeypatch,
    tmpdir,
):
    """ "Ensure that buildcache create creates output files"""
    pkg = "trivial-install-test-package"
    install(pkg)

    buildcache("push", "--unsigned", str(tmpdir), pkg)

    spec = Spec(pkg).concretized()
    tarball_path = spack.binary_distribution.tarball_path_name(spec, ".spack")
    tarball = spack.binary_distribution.tarball_name(spec, ".spec.json")
    assert os.path.exists(os.path.join(str(tmpdir), "build_cache", tarball_path))
    assert os.path.exists(os.path.join(str(tmpdir), "build_cache", tarball))


@pytest.mark.parametrize(
    "things_to_install,expected",
    [
        (
            "",
            [
                "dttop",
                "dtbuild1",
                "dtbuild2",
                "dtlink2",
                "dtrun2",
                "dtlink1",
                "dtlink3",
                "dtlink4",
                "dtrun1",
                "dtlink5",
                "dtrun3",
                "dtbuild3",
            ],
        ),
        (
            "dependencies",
            [
                "dtbuild1",
                "dtbuild2",
                "dtlink2",
                "dtrun2",
                "dtlink1",
                "dtlink3",
                "dtlink4",
                "dtrun1",
                "dtlink5",
                "dtrun3",
                "dtbuild3",
            ],
        ),
        ("package", ["dttop"]),
    ],
)
def test_correct_specs_are_pushed(
    things_to_install, expected, tmpdir, monkeypatch, default_mock_concretization, temporary_store
):
    # Concretize dttop and add it to the temporary database (without prefixes)
    spec = default_mock_concretization("dttop")
    temporary_store.db.add(spec, directory_layout=None)
    slash_hash = "/{0}".format(spec.dag_hash())

    packages_to_push = []

    def fake_push(node, push_url, options):
        assert isinstance(node, Spec)
        packages_to_push.append(node.name)

    monkeypatch.setattr(spack.binary_distribution, "push_or_raise", fake_push)

    buildcache_create_args = ["create", "--unsigned"]

    if things_to_install != "":
        buildcache_create_args.extend(["--only", things_to_install])

    buildcache_create_args.extend([str(tmpdir), slash_hash])

    buildcache(*buildcache_create_args)

    # Order is not guaranteed, so we can't just compare lists
    assert set(packages_to_push) == set(expected)

    # Ensure no duplicates
    assert len(set(packages_to_push)) == len(packages_to_push)


@pytest.mark.parametrize("signed", [True, False])
def test_push_and_install_with_mirror_marked_unsigned_does_not_require_extra_flags(
    tmp_path, mutable_database, mock_gnupghome, signed
):
    """Tests whether marking a mirror as unsigned makes it possible to push and install to/from
    it without requiring extra flags on the command line (and no signing keys configured)."""

    # Create a named mirror with signed set to True or False
    add_flag = "--signed" if signed else "--unsigned"
    mirror("add", add_flag, "my-mirror", str(tmp_path))
    spec = mutable_database.query_local("libelf", installed=True)[0]

    # Push
    if signed:
        # Need to pass "--unsigned" to override the mirror's default
        args = ["push", "--update-index", "--unsigned", "my-mirror", f"/{spec.dag_hash()}"]
    else:
        # No need to pass "--unsigned" if the mirror is unsigned
        args = ["push", "--update-index", "my-mirror", f"/{spec.dag_hash()}"]

    buildcache(*args)

    # Install
    if signed:
        # Need to pass "--no-check-signature" to avoid install errors
        kwargs = {"cache_only": True, "unsigned": True}
    else:
        # No need to pass "--no-check-signature" if the mirror is unsigned
        kwargs = {"cache_only": True}

    spec.package.do_uninstall(force=True)
    spec.package.do_install(**kwargs)