rewiring of spliced specs (#26873)

* tests for rewiring pure specs to spliced specs

* relocate text, binaries, and links

* using llnl.util.symlink for windows compat.

Note: This does not include CLI hooks for relocation.

Co-authored-by: Nathan Hanford <hanford1@llnl.gov>

2 files changed, 270 insertions, 0 deletions

diff --git a/lib/spack/spack/rewiring.py b/lib/spack/spack/rewiring.py
new file mode 100644
index 0000000000..de7114dd97
--- /dev/null
+++ b/lib/spack/spack/rewiring.py
@@ -0,0 +1,128 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+import re
+import shutil
+import tempfile
+from collections import OrderedDict
+
+from llnl.util.symlink import symlink
+
+import spack.binary_distribution as bindist
+import spack.error
+import spack.hooks
+import spack.paths
+import spack.relocate as relocate
+import spack.stage
+import spack.store
+
+
+def _relocate_spliced_links(links, orig_prefix, new_prefix):
+    """Re-linking function which differs from `relocate.relocate_links` by
+    reading the old link rather than the new link, since the latter wasn't moved
+    in our case. This still needs to be called after the copy to destination
+    because it expects the new directory structure to be in place."""
+    for link in links:
+        link_target = os.readlink(os.path.join(orig_prefix, link))
+        link_target = re.sub('^' + orig_prefix, new_prefix, link_target)
+        new_link_path = os.path.join(new_prefix, link)
+        os.unlink(new_link_path)
+        symlink(link_target, new_link_path)
+
+
+def rewire(spliced_spec):
+    """Given a spliced spec, this function conducts all the rewiring on all
+    nodes in the DAG of that spec."""
+    assert spliced_spec.spliced
+    for spec in spliced_spec.traverse(order='post', root=True):
+        if not spec.build_spec.package.installed:
+            # TODO: May want to change this at least for the root spec...
+            # spec.build_spec.package.do_install(force=True)
+            raise PackageNotInstalledError(spliced_spec,
+                                           spec.build_spec,
+                                           spec)
+        if spec.build_spec is not spec and not spec.package.installed:
+            explicit = spec is spliced_spec
+            rewire_node(spec, explicit)
+
+
+def rewire_node(spec, explicit):
+    """This function rewires a single node, worrying only about references to
+    its subgraph. Binaries, text, and links are all changed in accordance with
+    the splice. The resulting package is then 'installed.'"""
+    tempdir = tempfile.mkdtemp()
+    # copy anything installed to a temporary directory
+    shutil.copytree(spec.build_spec.prefix,
+                    os.path.join(tempdir, spec.dag_hash()))
+
+    spack.hooks.pre_install(spec)
+    # compute prefix-to-prefix for every node from the build spec to the spliced
+    # spec
+    prefix_to_prefix = OrderedDict({spec.build_spec.prefix: spec.prefix})
+    for build_dep in spec.build_spec.traverse(root=False):
+        prefix_to_prefix[build_dep.prefix] = spec[build_dep.name].prefix
+
+    manifest = bindist.get_buildfile_manifest(spec.build_spec)
+    platform = spack.platforms.by_name(spec.platform)
+
+    text_to_relocate = [os.path.join(tempdir, spec.dag_hash(), rel_path)
+                        for rel_path in manifest.get('text_to_relocate', [])]
+    if text_to_relocate:
+        relocate.relocate_text(files=text_to_relocate,
+                               prefixes=prefix_to_prefix)
+
+    bins_to_relocate = [os.path.join(tempdir, spec.dag_hash(), rel_path)
+                        for rel_path in manifest.get('binary_to_relocate', [])]
+    if bins_to_relocate:
+        if 'macho' in platform.binary_formats:
+            relocate.relocate_macho_binaries(bins_to_relocate,
+                                             str(spack.store.layout.root),
+                                             str(spack.store.layout.root),
+                                             prefix_to_prefix,
+                                             False,
+                                             spec.build_spec.prefix,
+                                             spec.prefix)
+        if 'elf' in platform.binary_formats:
+            relocate.relocate_elf_binaries(bins_to_relocate,
+                                           str(spack.store.layout.root),
+                                           str(spack.store.layout.root),
+                                           prefix_to_prefix,
+                                           False,
+                                           spec.build_spec.prefix,
+                                           spec.prefix)
+        relocate.relocate_text_bin(binaries=bins_to_relocate,
+                                   prefixes=prefix_to_prefix)
+    # copy package into place (shutil.copytree)
+    shutil.copytree(os.path.join(tempdir, spec.dag_hash()), spec.prefix,
+                    ignore=shutil.ignore_patterns('spec.json',
+                                                  'install_manifest.json'))
+    if manifest.get('link_to_relocate'):
+        _relocate_spliced_links(manifest.get('link_to_relocate'),
+                                spec.build_spec.prefix,
+                                spec.prefix)
+    shutil.rmtree(tempdir)
+    # handle all metadata changes; don't copy over spec.json file in .spack/
+    spack.store.layout.write_spec(spec, spack.store.layout.spec_file_path(spec))
+    # add to database, not sure about explicit
+    spack.store.db.add(spec, spack.store.layout, explicit=explicit)
+
+    # run post install hooks
+    spack.hooks.post_install(spec)
+
+
+class RewireError(spack.error.SpackError):
+    """Raised when something goes wrong with rewiring."""
+    def __init__(self, message, long_msg=None):
+        super(RewireError, self).__init__(message, long_msg)
+
+
+class PackageNotInstalledError(RewireError):
+    """Raised when the build_spec for a splice was not installed."""
+    def __init__(self, spliced_spec, build_spec, dep):
+        super(PackageNotInstalledError, self).__init__(
+            """Rewire of {0}
+            failed due to missing install of build spec {1}
+            for spec {2}""".format(spliced_spec, build_spec, dep))
diff --git a/lib/spack/spack/test/rewiring.py b/lib/spack/spack/test/rewiring.py
new file mode 100644
index 0000000000..f467c8e239
--- /dev/null
+++ b/lib/spack/spack/test/rewiring.py
@@ -0,0 +1,142 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import filecmp
+import os
+import sys
+
+import pytest
+
+import spack.rewiring
+import spack.store
+from spack.spec import Spec
+from spack.test.relocate import text_in_bin
+
+args = ['strings', 'file']
+if sys.platform == 'darwin':
+    args.extend(['/usr/bin/clang++', 'install_name_tool'])
+else:
+    args.extend(['/usr/bin/g++', 'patchelf'])
+
+
+@pytest.mark.requires_executables(*args)
+@pytest.mark.parametrize('transitive', [True, False])
+def test_rewire(mock_fetch, install_mockery, transitive):
+    spec = Spec('splice-t^splice-h~foo').concretized()
+    dep = Spec('splice-h+foo').concretized()
+    spec.package.do_install()
+    dep.package.do_install()
+    spliced_spec = spec.splice(dep, transitive=transitive)
+    assert spec.dag_hash() != spliced_spec.dag_hash()
+
+    spack.rewiring.rewire(spliced_spec)
+
+    # check that the prefix exists
+    assert os.path.exists(spliced_spec.prefix)
+
+    # test that it made it into the database
+    rec = spack.store.db.get_record(spliced_spec)
+    installed_in_db = rec.installed if rec else False
+    assert installed_in_db
+
+    # check the file in the prefix has the correct paths
+    for node in spliced_spec.traverse(root=True):
+        text_file_path = os.path.join(node.prefix, node.name)
+        with open(text_file_path, 'r') as f:
+            text = f.read()
+            for modded_spec in node.traverse(root=True):
+                assert modded_spec.prefix in text
+
+
+@pytest.mark.requires_executables(*args)
+@pytest.mark.parametrize('transitive', [True, False])
+def test_rewire_bin(mock_fetch, install_mockery, transitive):
+    spec = Spec('quux').concretized()
+    dep = Spec('garply cflags=-g').concretized()
+    spec.package.do_install()
+    dep.package.do_install()
+    spliced_spec = spec.splice(dep, transitive=transitive)
+    assert spec.dag_hash() != spliced_spec.dag_hash()
+
+    spack.rewiring.rewire(spliced_spec)
+
+    # check that the prefix exists
+    assert os.path.exists(spliced_spec.prefix)
+
+    # test that it made it into the database
+    rec = spack.store.db.get_record(spliced_spec)
+    installed_in_db = rec.installed if rec else False
+    assert installed_in_db
+
+    # check the file in the prefix has the correct paths
+    bin_names = {'garply': 'garplinator',
+                 'corge': 'corgegator',
+                 'quux': 'quuxifier'}
+    for node in spliced_spec.traverse(root=True):
+        for dep in node.traverse(root=True):
+            bin_file_path = os.path.join(dep.prefix.bin, bin_names[dep.name])
+            assert text_in_bin(dep.prefix, bin_file_path)
+
+
+@pytest.mark.requires_executables(*args)
+def test_rewire_writes_new_metadata(mock_fetch, install_mockery):
+    # check for spec.json and install_manifest.json and that they are new
+    # for a simple case.
+    spec = Spec('quux').concretized()
+    dep = Spec('garply cflags=-g').concretized()
+    spec.package.do_install()
+    dep.package.do_install()
+    spliced_spec = spec.splice(dep, transitive=True)
+    spack.rewiring.rewire(spliced_spec)
+
+    # test install manifests
+    for node in spliced_spec.traverse(root=True):
+        spack.store.layout.ensure_installed(node)
+        manifest_file_path = os.path.join(node.prefix,
+                                          spack.store.layout.metadata_dir,
+                                          spack.store.layout.manifest_file_name)
+        assert os.path.exists(manifest_file_path)
+        orig_node = spec[node.name]
+        orig_manifest_file_path = os.path.join(orig_node.prefix,
+                                               spack.store.layout.metadata_dir,
+                                               spack.store.layout.manifest_file_name)
+        assert os.path.exists(orig_manifest_file_path)
+        assert not filecmp.cmp(orig_manifest_file_path, manifest_file_path,
+                               shallow=False)
+        specfile_path = os.path.join(node.prefix,
+                                     spack.store.layout.metadata_dir,
+                                     spack.store.layout.spec_file_name)
+        assert os.path.exists(specfile_path)
+        orig_specfile_path = os.path.join(orig_node.prefix,
+                                          spack.store.layout.metadata_dir,
+                                          spack.store.layout.spec_file_name)
+        assert os.path.exists(orig_specfile_path)
+        assert not filecmp.cmp(orig_specfile_path, specfile_path,
+                               shallow=False)
+
+
+@pytest.mark.requires_executables(*args)
+@pytest.mark.parametrize('transitive', [True, False])
+def test_uninstall_rewired_spec(mock_fetch, install_mockery, transitive):
+    # Test that rewired packages can be uninstalled as normal.
+    spec = Spec('quux').concretized()
+    dep = Spec('garply cflags=-g').concretized()
+    spec.package.do_install()
+    dep.package.do_install()
+    spliced_spec = spec.splice(dep, transitive=transitive)
+    spack.rewiring.rewire(spliced_spec)
+    spliced_spec.package.do_uninstall()
+    assert len(spack.store.db.query(spliced_spec)) == 0
+    assert not os.path.exists(spliced_spec.prefix)
+
+
+@pytest.mark.requires_executables(*args)
+def test_rewire_not_installed_fails(mock_fetch, install_mockery):
+    spec = Spec('quux').concretized()
+    dep = Spec('garply cflags=-g').concretized()
+    spliced_spec = spec.splice(dep, False)
+    with pytest.raises(spack.rewiring.PackageNotInstalledError,
+                       match="failed due to missing install of build spec"):
+        spack.rewiring.rewire(spliced_spec)