From 88473a8da307368cca1bb07ba92f3e312f1823a7 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Fri, 31 May 2019 11:33:29 -0500 Subject: Build cache: relocate path to spack/bin/sbang in text files. (#11592) * Build cache: relocate path to spack/bin/sbang in text files. * Found in testing. * update packaging test * Make sbang replacement including #!/bin/bash. Add an additional spack prefix replacement to fix stage directory references. * flake8 * Use buildinfo.get() so old buildcaches without buildinfo['spackprefix'] can be read. --- lib/spack/spack/binary_distribution.py | 12 +++++++++--- lib/spack/spack/relocate.py | 33 ++++++++++++++++++++++++--------- lib/spack/spack/test/packaging.py | 3 ++- 3 files changed, 35 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 1d9acd1af9..602e9605de 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -171,6 +171,7 @@ def write_buildinfo_file(prefix, workdir, rel=False): buildinfo = {} buildinfo['relative_rpaths'] = rel buildinfo['buildpath'] = spack.store.layout.root + buildinfo['spackprefix'] = spack.paths.prefix buildinfo['relative_prefix'] = os.path.relpath( prefix, spack.store.layout.root) buildinfo['relocate_textfiles'] = text_to_relocate @@ -459,7 +460,9 @@ def relocate_package(workdir, allow_root): """ buildinfo = read_buildinfo_file(workdir) new_path = spack.store.layout.root + new_prefix = spack.paths.prefix old_path = buildinfo['buildpath'] + old_prefix = buildinfo.get('spackprefix', '/not/in/buildinfo/dictionary') rel = buildinfo.get('relative_rpaths', False) if rel: return @@ -472,7 +475,9 @@ def relocate_package(workdir, allow_root): # Don't add backup files generated by filter_file during install step. if not path_name.endswith('~'): path_names.add(path_name) - relocate.relocate_text(path_names, old_path, new_path) + relocate.relocate_text(path_names, oldpath=old_path, + newpath=new_path, oldprefix=old_prefix, + newprefix=new_prefix) # If the binary files in the package were not edited to use # relative RPATHs, then the RPATHs need to be relocated if not rel: @@ -602,7 +607,8 @@ def get_specs(force=False): urls = set() for mirror_name, mirror_url in mirrors.items(): if mirror_url.startswith('file'): - mirror = mirror_url.replace('file://', '') + "/" + _build_cache_relative_path + mirror = mirror_url.replace( + 'file://', '') + "/" + _build_cache_relative_path tty.msg("Finding buildcaches in %s" % mirror) if os.path.exists(mirror): files = os.listdir(mirror) @@ -731,7 +737,7 @@ def needs_rebuild(spec, mirror_url, rebuild_on_errors=False): # just rebuild. This can be simplified once the dag_hash and the # full_hash become the same thing. if ('full_hash' not in spec_yaml or - spec_yaml['full_hash'] != pkg_full_hash): + spec_yaml['full_hash'] != pkg_full_hash): if 'full_hash' in spec_yaml: reason = 'hash mismatch, remote = {0}, local = {1}'.format( spec_yaml['full_hash'], pkg_full_hash) diff --git a/lib/spack/spack/relocate.py b/lib/spack/spack/relocate.py index 4f4626e0cb..fc653e825d 100644 --- a/lib/spack/spack/relocate.py +++ b/lib/spack/spack/relocate.py @@ -19,6 +19,7 @@ class InstallRootStringException(spack.error.SpackError): """ Raised when the relocated binary still has the install root string. """ + def __init__(self, file_path, root_path): super(InstallRootStringException, self).__init__( "\n %s \ncontains string\n %s \n" @@ -249,7 +250,7 @@ def strings_contains_installroot(path_name, root_dir): strings = Executable('strings') output = strings('%s' % path_name, output=str, err=str) - return (root_dir in output) + return (root_dir in output or spack.paths.prefix in output) def modify_elf_object(path_name, new_rpaths): @@ -276,7 +277,7 @@ def needs_binary_relocation(m_type, m_subtype): """ if m_type == 'application': if (m_subtype == 'x-executable' or m_subtype == 'x-sharedlib' or - m_subtype == 'x-mach-binary'): + m_subtype == 'x-mach-binary'): return True return False @@ -334,7 +335,7 @@ def relocate_binary(path_names, old_dir, new_dir, allow_root): modify_elf_object(path_name, new_rpaths) if (not allow_root and old_dir != new_dir and - not file_is_relocatable(path_name)): + not file_is_relocatable(path_name)): raise InstallRootStringException(path_name, old_dir) else: tty.die("Relocation not implemented for %s" % platform.system()) @@ -367,7 +368,7 @@ def make_binary_relative(cur_path_names, orig_path_names, old_dir, allow_root): rpaths, deps, idpath, new_rpaths, new_deps, new_idpath) if (not allow_root and - not file_is_relocatable(cur_path, old_dir)): + not file_is_relocatable(cur_path, old_dir)): raise InstallRootStringException(cur_path, old_dir) elif platform.system() == 'Linux': for cur_path, orig_path in zip(cur_path_names, orig_path_names): @@ -390,13 +391,13 @@ def make_binary_placeholder(cur_path_names, allow_root): if platform.system() == 'Darwin': for cur_path in cur_path_names: if (not allow_root and - not file_is_relocatable(cur_path)): + not file_is_relocatable(cur_path)): raise InstallRootStringException( cur_path, spack.store.layout.root) elif platform.system() == 'Linux': for cur_path in cur_path_names: if (not allow_root and - not file_is_relocatable(cur_path)): + not file_is_relocatable(cur_path)): raise InstallRootStringException( cur_path, spack.store.layout.root) else: @@ -438,11 +439,18 @@ def relocate_links(path_names, old_dir, new_dir): os.symlink(new_src, path_name) -def relocate_text(path_names, old_dir, new_dir): +def relocate_text(path_names, oldpath, newpath, oldprefix, newprefix): """ Replace old path with new path in text file path_name """ - fs.filter_file('%s' % old_dir, '%s' % new_dir, *path_names, backup=False) + fs.filter_file('%s' % oldpath, '%s' % newpath, *path_names, + backup=False, string=True) + sbangre = '#!/bin/bash %s/bin/sbang' % oldprefix + sbangnew = '#!/bin/bash %s/bin/sbang' % newprefix + fs.filter_file(sbangre, sbangnew, *path_names, + backup=False, string=True) + fs.filter_file(oldprefix, newprefix, *path_names, + backup=False, string=True) def substitute_rpath(orig_rpath, topdir, new_root_path): @@ -530,7 +538,7 @@ def file_is_relocatable(file): set_of_strings.discard(rpaths.strip()) if platform.system().lower() == 'darwin': if m_subtype == 'x-mach-binary': - rpaths, deps, idpath = macho_get_paths(file) + rpaths, deps, idpath = macho_get_paths(file) set_of_strings.discard(set(rpaths)) set_of_strings.discard(set(deps)) if idpath is not None: @@ -543,6 +551,13 @@ def file_is_relocatable(file): tty.debug(msg.format(spack.store.layout.root, file)) return False + if any(spack.paths.prefix in x for x in set_of_strings): + # One binary has the root folder not in the RPATH, + # meaning that this spec is not relocatable + msg = 'Found "{0}" in {1} strings' + tty.debug(msg.format(spack.paths.prefix, file)) + return False + return True diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py index c83ebc30ea..e865cac32e 100644 --- a/lib/spack/spack/test/packaging.py +++ b/lib/spack/spack/test/packaging.py @@ -245,7 +245,8 @@ def test_relocate_text(tmpdir): script.close() filenames = [filename] new_dir = '/opt/rh/devtoolset/' - relocate_text(filenames, old_dir, new_dir) + relocate_text(filenames, oldpath=old_dir, newpath=new_dir, + oldprefix=old_dir, newprefix=new_dir) with open(filename, "r")as script: for line in script: assert(new_dir in line) -- cgit v1.2.3-70-g09d2