From 88d8ca9b6504dca4973784932c0ea90fe8cd3f83 Mon Sep 17 00:00:00 2001 From: Nathan Hanford <8302958+nhanford@users.noreply.github.com> Date: Mon, 4 Apr 2022 14:45:35 -0700 Subject: rewiring of spliced specs (#26873) * tests for rewiring pure specs to spliced specs * relocate text, binaries, and links * using llnl.util.symlink for windows compat. Note: This does not include CLI hooks for relocation. Co-authored-by: Nathan Hanford --- lib/spack/spack/rewiring.py | 128 +++++++++++++++++++++++++++++++++++ lib/spack/spack/test/rewiring.py | 142 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 270 insertions(+) create mode 100644 lib/spack/spack/rewiring.py create mode 100644 lib/spack/spack/test/rewiring.py (limited to 'lib') diff --git a/lib/spack/spack/rewiring.py b/lib/spack/spack/rewiring.py new file mode 100644 index 0000000000..de7114dd97 --- /dev/null +++ b/lib/spack/spack/rewiring.py @@ -0,0 +1,128 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import re +import shutil +import tempfile +from collections import OrderedDict + +from llnl.util.symlink import symlink + +import spack.binary_distribution as bindist +import spack.error +import spack.hooks +import spack.paths +import spack.relocate as relocate +import spack.stage +import spack.store + + +def _relocate_spliced_links(links, orig_prefix, new_prefix): + """Re-linking function which differs from `relocate.relocate_links` by + reading the old link rather than the new link, since the latter wasn't moved + in our case. This still needs to be called after the copy to destination + because it expects the new directory structure to be in place.""" + for link in links: + link_target = os.readlink(os.path.join(orig_prefix, link)) + link_target = re.sub('^' + orig_prefix, new_prefix, link_target) + new_link_path = os.path.join(new_prefix, link) + os.unlink(new_link_path) + symlink(link_target, new_link_path) + + +def rewire(spliced_spec): + """Given a spliced spec, this function conducts all the rewiring on all + nodes in the DAG of that spec.""" + assert spliced_spec.spliced + for spec in spliced_spec.traverse(order='post', root=True): + if not spec.build_spec.package.installed: + # TODO: May want to change this at least for the root spec... + # spec.build_spec.package.do_install(force=True) + raise PackageNotInstalledError(spliced_spec, + spec.build_spec, + spec) + if spec.build_spec is not spec and not spec.package.installed: + explicit = spec is spliced_spec + rewire_node(spec, explicit) + + +def rewire_node(spec, explicit): + """This function rewires a single node, worrying only about references to + its subgraph. Binaries, text, and links are all changed in accordance with + the splice. The resulting package is then 'installed.'""" + tempdir = tempfile.mkdtemp() + # copy anything installed to a temporary directory + shutil.copytree(spec.build_spec.prefix, + os.path.join(tempdir, spec.dag_hash())) + + spack.hooks.pre_install(spec) + # compute prefix-to-prefix for every node from the build spec to the spliced + # spec + prefix_to_prefix = OrderedDict({spec.build_spec.prefix: spec.prefix}) + for build_dep in spec.build_spec.traverse(root=False): + prefix_to_prefix[build_dep.prefix] = spec[build_dep.name].prefix + + manifest = bindist.get_buildfile_manifest(spec.build_spec) + platform = spack.platforms.by_name(spec.platform) + + text_to_relocate = [os.path.join(tempdir, spec.dag_hash(), rel_path) + for rel_path in manifest.get('text_to_relocate', [])] + if text_to_relocate: + relocate.relocate_text(files=text_to_relocate, + prefixes=prefix_to_prefix) + + bins_to_relocate = [os.path.join(tempdir, spec.dag_hash(), rel_path) + for rel_path in manifest.get('binary_to_relocate', [])] + if bins_to_relocate: + if 'macho' in platform.binary_formats: + relocate.relocate_macho_binaries(bins_to_relocate, + str(spack.store.layout.root), + str(spack.store.layout.root), + prefix_to_prefix, + False, + spec.build_spec.prefix, + spec.prefix) + if 'elf' in platform.binary_formats: + relocate.relocate_elf_binaries(bins_to_relocate, + str(spack.store.layout.root), + str(spack.store.layout.root), + prefix_to_prefix, + False, + spec.build_spec.prefix, + spec.prefix) + relocate.relocate_text_bin(binaries=bins_to_relocate, + prefixes=prefix_to_prefix) + # copy package into place (shutil.copytree) + shutil.copytree(os.path.join(tempdir, spec.dag_hash()), spec.prefix, + ignore=shutil.ignore_patterns('spec.json', + 'install_manifest.json')) + if manifest.get('link_to_relocate'): + _relocate_spliced_links(manifest.get('link_to_relocate'), + spec.build_spec.prefix, + spec.prefix) + shutil.rmtree(tempdir) + # handle all metadata changes; don't copy over spec.json file in .spack/ + spack.store.layout.write_spec(spec, spack.store.layout.spec_file_path(spec)) + # add to database, not sure about explicit + spack.store.db.add(spec, spack.store.layout, explicit=explicit) + + # run post install hooks + spack.hooks.post_install(spec) + + +class RewireError(spack.error.SpackError): + """Raised when something goes wrong with rewiring.""" + def __init__(self, message, long_msg=None): + super(RewireError, self).__init__(message, long_msg) + + +class PackageNotInstalledError(RewireError): + """Raised when the build_spec for a splice was not installed.""" + def __init__(self, spliced_spec, build_spec, dep): + super(PackageNotInstalledError, self).__init__( + """Rewire of {0} + failed due to missing install of build spec {1} + for spec {2}""".format(spliced_spec, build_spec, dep)) diff --git a/lib/spack/spack/test/rewiring.py b/lib/spack/spack/test/rewiring.py new file mode 100644 index 0000000000..f467c8e239 --- /dev/null +++ b/lib/spack/spack/test/rewiring.py @@ -0,0 +1,142 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import filecmp +import os +import sys + +import pytest + +import spack.rewiring +import spack.store +from spack.spec import Spec +from spack.test.relocate import text_in_bin + +args = ['strings', 'file'] +if sys.platform == 'darwin': + args.extend(['/usr/bin/clang++', 'install_name_tool']) +else: + args.extend(['/usr/bin/g++', 'patchelf']) + + +@pytest.mark.requires_executables(*args) +@pytest.mark.parametrize('transitive', [True, False]) +def test_rewire(mock_fetch, install_mockery, transitive): + spec = Spec('splice-t^splice-h~foo').concretized() + dep = Spec('splice-h+foo').concretized() + spec.package.do_install() + dep.package.do_install() + spliced_spec = spec.splice(dep, transitive=transitive) + assert spec.dag_hash() != spliced_spec.dag_hash() + + spack.rewiring.rewire(spliced_spec) + + # check that the prefix exists + assert os.path.exists(spliced_spec.prefix) + + # test that it made it into the database + rec = spack.store.db.get_record(spliced_spec) + installed_in_db = rec.installed if rec else False + assert installed_in_db + + # check the file in the prefix has the correct paths + for node in spliced_spec.traverse(root=True): + text_file_path = os.path.join(node.prefix, node.name) + with open(text_file_path, 'r') as f: + text = f.read() + for modded_spec in node.traverse(root=True): + assert modded_spec.prefix in text + + +@pytest.mark.requires_executables(*args) +@pytest.mark.parametrize('transitive', [True, False]) +def test_rewire_bin(mock_fetch, install_mockery, transitive): + spec = Spec('quux').concretized() + dep = Spec('garply cflags=-g').concretized() + spec.package.do_install() + dep.package.do_install() + spliced_spec = spec.splice(dep, transitive=transitive) + assert spec.dag_hash() != spliced_spec.dag_hash() + + spack.rewiring.rewire(spliced_spec) + + # check that the prefix exists + assert os.path.exists(spliced_spec.prefix) + + # test that it made it into the database + rec = spack.store.db.get_record(spliced_spec) + installed_in_db = rec.installed if rec else False + assert installed_in_db + + # check the file in the prefix has the correct paths + bin_names = {'garply': 'garplinator', + 'corge': 'corgegator', + 'quux': 'quuxifier'} + for node in spliced_spec.traverse(root=True): + for dep in node.traverse(root=True): + bin_file_path = os.path.join(dep.prefix.bin, bin_names[dep.name]) + assert text_in_bin(dep.prefix, bin_file_path) + + +@pytest.mark.requires_executables(*args) +def test_rewire_writes_new_metadata(mock_fetch, install_mockery): + # check for spec.json and install_manifest.json and that they are new + # for a simple case. + spec = Spec('quux').concretized() + dep = Spec('garply cflags=-g').concretized() + spec.package.do_install() + dep.package.do_install() + spliced_spec = spec.splice(dep, transitive=True) + spack.rewiring.rewire(spliced_spec) + + # test install manifests + for node in spliced_spec.traverse(root=True): + spack.store.layout.ensure_installed(node) + manifest_file_path = os.path.join(node.prefix, + spack.store.layout.metadata_dir, + spack.store.layout.manifest_file_name) + assert os.path.exists(manifest_file_path) + orig_node = spec[node.name] + orig_manifest_file_path = os.path.join(orig_node.prefix, + spack.store.layout.metadata_dir, + spack.store.layout.manifest_file_name) + assert os.path.exists(orig_manifest_file_path) + assert not filecmp.cmp(orig_manifest_file_path, manifest_file_path, + shallow=False) + specfile_path = os.path.join(node.prefix, + spack.store.layout.metadata_dir, + spack.store.layout.spec_file_name) + assert os.path.exists(specfile_path) + orig_specfile_path = os.path.join(orig_node.prefix, + spack.store.layout.metadata_dir, + spack.store.layout.spec_file_name) + assert os.path.exists(orig_specfile_path) + assert not filecmp.cmp(orig_specfile_path, specfile_path, + shallow=False) + + +@pytest.mark.requires_executables(*args) +@pytest.mark.parametrize('transitive', [True, False]) +def test_uninstall_rewired_spec(mock_fetch, install_mockery, transitive): + # Test that rewired packages can be uninstalled as normal. + spec = Spec('quux').concretized() + dep = Spec('garply cflags=-g').concretized() + spec.package.do_install() + dep.package.do_install() + spliced_spec = spec.splice(dep, transitive=transitive) + spack.rewiring.rewire(spliced_spec) + spliced_spec.package.do_uninstall() + assert len(spack.store.db.query(spliced_spec)) == 0 + assert not os.path.exists(spliced_spec.prefix) + + +@pytest.mark.requires_executables(*args) +def test_rewire_not_installed_fails(mock_fetch, install_mockery): + spec = Spec('quux').concretized() + dep = Spec('garply cflags=-g').concretized() + spliced_spec = spec.splice(dep, False) + with pytest.raises(spack.rewiring.PackageNotInstalledError, + match="failed due to missing install of build spec"): + spack.rewiring.rewire(spliced_spec) -- cgit v1.2.3-60-g2f50