diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/spack/detection/path.py | 29 | ||||
| -rw-r--r-- | lib/spack/spack/test/util/ld_so_conf.py | 54 | ||||
| -rw-r--r-- | lib/spack/spack/util/ld_so_conf.py | 140 |
3 files changed, 211 insertions, 12 deletions
diff --git a/lib/spack/spack/detection/path.py b/lib/spack/spack/detection/path.py index b7a7f6702d..bf42fa6144 100644 --- a/lib/spack/spack/detection/path.py +++ b/lib/spack/spack/detection/path.py @@ -17,6 +17,7 @@ import llnl.util.tty import spack.operating_systems.windows_os as winOs import spack.util.environment +import spack.util.ld_so_conf from .common import ( DetectedPackage, @@ -75,9 +76,10 @@ def executables_in_path(path_hints=None): return path_to_exe -def libraries_in_ld_library_path(path_hints=None): +def libraries_in_ld_and_system_library_path(path_hints=None): """Get the paths of all libraries available from LD_LIBRARY_PATH, - LIBRARY_PATH, DYLD_LIBRARY_PATH, and DYLD_FALLBACK_LIBRARY_PATH. + LIBRARY_PATH, DYLD_LIBRARY_PATH, DYLD_FALLBACK_LIBRARY_PATH, and + standard system library paths. For convenience, this is constructed as a dictionary where the keys are the library paths and the values are the names of the libraries @@ -90,14 +92,14 @@ def libraries_in_ld_library_path(path_hints=None): path_hints (list): list of paths to be searched. If None the list will be constructed based on the set of LD_LIBRARY_PATH, LIBRARY_PATH, DYLD_LIBRARY_PATH, and DYLD_FALLBACK_LIBRARY_PATH environment - variables. + variables as well as the standard system library paths. """ - path_hints = path_hints or spack.util.environment.get_path( - "LIBRARY_PATH" - ) + spack.util.environment.get_path("LD_LIBRARY_PATH") + spack.util.environment.get_path( - "DYLD_LIBRARY_PATH" - ) + spack.util.environment.get_path( - "DYLD_FALLBACK_LIBRARY_PATH" + path_hints = ( + path_hints + or spack.util.environment.get_path("LD_LIBRARY_PATH") + + spack.util.environment.get_path("DYLD_LIBRARY_PATH") + + spack.util.environment.get_path("DYLD_FALLBACK_LIBRARY_PATH") + + spack.util.ld_so_conf.host_dynamic_linker_search_paths() ) search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints) @@ -129,14 +131,17 @@ def by_library(packages_to_check, path_hints=None): # Other libraries could use the strings function to extract it as described # in https://unix.stackexchange.com/questions/58846/viewing-linux-library-executable-version-info """Return the list of packages that have been detected on the system, - searching by LD_LIBRARY_PATH. + searching by LD_LIBRARY_PATH, LIBRARY_PATH, DYLD_LIBRARY_PATH, + DYLD_FALLBACK_LIBRARY_PATH, and standard system library paths. Args: packages_to_check (list): list of packages to be detected path_hints (list): list of paths to be searched. If None the list will be - constructed based on the LD_LIBRARY_PATH environment variable. + constructed based on the LD_LIBRARY_PATH, LIBRARY_PATH, + DYLD_LIBRARY_PATH, DYLD_FALLBACK_LIBRARY_PATH environment variables + and standard system library paths. """ - path_to_lib_name = libraries_in_ld_library_path(path_hints=path_hints) + path_to_lib_name = libraries_in_ld_and_system_library_path(path_hints=path_hints) lib_pattern_to_pkgs = collections.defaultdict(list) for pkg in packages_to_check: if hasattr(pkg, "libraries"): diff --git a/lib/spack/spack/test/util/ld_so_conf.py b/lib/spack/spack/test/util/ld_so_conf.py new file mode 100644 index 0000000000..ea1c86e896 --- /dev/null +++ b/lib/spack/spack/test/util/ld_so_conf.py @@ -0,0 +1,54 @@ +# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os + +import spack.util.ld_so_conf as ld_so_conf + + +def test_ld_so_conf_parsing(tmpdir): + cwd = os.getcwd() + tmpdir.ensure("subdir", dir=True) + + # Entrypoint config file + with open(str(tmpdir.join("main.conf")), "wb") as f: + f.write(b" \n") + f.write(b"include subdir/*.conf\n") + f.write(b"include non-existent/file\n") + f.write(b"include #nope\n") + f.write(b"include \n") + f.write(b"include\t\n") + f.write(b"include\n") + f.write(b"/main.conf/lib # and a comment\n") + f.write(b"relative/path\n\n") + f.write(b"#/skip/me\n") + + # Should be parsed: subdir/first.conf + with open(str(tmpdir.join("subdir", "first.conf")), "wb") as f: + f.write(b"/first.conf/lib") + + # Should be parsed: subdir/second.conf + with open(str(tmpdir.join("subdir", "second.conf")), "wb") as f: + f.write(b"/second.conf/lib") + + # Not matching subdir/*.conf + with open(str(tmpdir.join("subdir", "third")), "wb") as f: + f.write(b"/third/lib") + + paths = ld_so_conf.parse_ld_so_conf(str(tmpdir.join("main.conf"))) + + assert len(paths) == 3 + assert "/main.conf/lib" in paths + assert "/first.conf/lib" in paths + assert "/second.conf/lib" in paths + + # Make sure globbing didn't change the working dir + assert os.getcwd() == cwd + + +def test_host_dynamic_linker_search_paths(): + assert {"/usr/lib", "/usr/lib64", "/lib", "/lib64"}.issubset( + ld_so_conf.host_dynamic_linker_search_paths() + ) diff --git a/lib/spack/spack/util/ld_so_conf.py b/lib/spack/spack/util/ld_so_conf.py new file mode 100644 index 0000000000..dd7f0e0ee6 --- /dev/null +++ b/lib/spack/spack/util/ld_so_conf.py @@ -0,0 +1,140 @@ +# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import glob +import os +import re +import sys + +from llnl.util.lang import dedupe + +import spack.util.elf as elf_utils + + +def parse_ld_so_conf(conf_file="/etc/ld.so.conf"): + """Parse glibc style ld.so.conf file, which specifies default search paths for the + dynamic linker. This can in principle also be used for musl libc. + + Arguments: + conf_file (str or bytes): Path to config file + + Returns: + list: List of absolute search paths + """ + # Parse in binary mode since it's faster + is_bytes = isinstance(conf_file, bytes) + if not is_bytes: + conf_file = conf_file.encode("utf-8") + + # For globbing in Python2 we need to chdir. + cwd = os.getcwd() + try: + paths = _process_ld_so_conf_queue([conf_file]) + finally: + os.chdir(cwd) + + return list(paths) if is_bytes else [p.decode("utf-8") for p in paths] + + +def _process_ld_so_conf_queue(queue): + include_regex = re.compile(b"include\\s") + paths = [] + while queue: + p = queue.pop(0) + + try: + with open(p, "rb") as f: + lines = f.readlines() + except (IOError, OSError): + continue + + for line in lines: + # Strip comments + comment = line.find(b"#") + if comment != -1: + line = line[:comment] + + # Skip empty lines + line = line.strip() + if not line: + continue + + is_include = include_regex.match(line) is not None + + # If not an include, it's a literal path (no globbing here). + if not is_include: + # We only allow absolute search paths. + if os.path.isabs(line): + paths.append(line) + continue + + # Finally handle includes. + include_path = line[8:].strip() + if not include_path: + continue + + cwd = os.path.dirname(p) + os.chdir(cwd) + queue.extend(os.path.join(cwd, p) for p in glob.glob(include_path)) + + return dedupe(paths) + + +def get_conf_file_from_dynamic_linker(dynamic_linker_name): + # We basically assume everything is glibc, except musl. + if "ld-musl-" not in dynamic_linker_name: + return "ld.so.conf" + + # Musl has a dynamic loader of the form ld-musl-<arch>.so.1 + # and a corresponding config file ld-musl-<arch>.path + idx = dynamic_linker_name.find(".") + if idx != -1: + return dynamic_linker_name[:idx] + ".path" + + +def host_dynamic_linker_search_paths(): + """Retrieve the current host runtime search paths for shared libraries; + for GNU and musl Linux we try to retrieve the dynamic linker from the + current Python interpreter and then find the corresponding config file + (e.g. ld.so.conf or ld-musl-<arch>.path). Similar can be done for + BSD and others, but this is not implemented yet. The default paths + are always returned. We don't check if the listed directories exist.""" + default_paths = ["/usr/lib", "/usr/lib64", "/lib", "/lib64"] + + # Currently only for Linux (gnu/musl) + if not sys.platform.startswith("linux"): + return default_paths + + # If everything fails, try this standard glibc path. + conf_file = "/etc/ld.so.conf" + + # Try to improve on the default conf path by retrieving the location of the + # dynamic linker from our current Python interpreter, and figure out the + # config file location from there. + try: + with open(sys.executable, "rb") as f: + elf = elf_utils.parse_elf(f, dynamic_section=False, interpreter=True) + + # If we have a dynamic linker, try to retrieve the config file relative + # to its prefix. + if elf.has_pt_interp: + dynamic_linker = elf.pt_interp_str.decode("utf-8") + dynamic_linker_name = os.path.basename(dynamic_linker) + conf_name = get_conf_file_from_dynamic_linker(dynamic_linker_name) + + # Typically it is /lib/ld.so, but on Gentoo Prefix it is something + # like <long glibc prefix>/lib/ld.so. And on Debian /lib64 is actually + # a symlink to /usr/lib64. So, best effort attempt is to just strip + # two path components and join with etc/ld.so.conf. + possible_prefix = os.path.dirname(os.path.dirname(dynamic_linker)) + possible_conf = os.path.join(possible_prefix, "etc", conf_name) + + if os.path.exists(possible_conf): + conf_file = possible_conf + except (IOError, OSError, elf_utils.ElfParsingError): + pass + + # Note: ld_so_conf doesn't error if the file does not exist. + return list(dedupe(parse_ld_so_conf(conf_file) + default_paths)) |