summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/llnl/util/filesystem.py37
-rw-r--r--lib/spack/spack/cmd/external.py2
-rw-r--r--lib/spack/spack/detection/__init__.py3
-rw-r--r--lib/spack/spack/detection/common.py23
-rw-r--r--lib/spack/spack/detection/path.py129
-rw-r--r--lib/spack/spack/package.py34
6 files changed, 215 insertions, 13 deletions
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index e16db8ba8a..fbc6a3d7ce 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -1940,6 +1940,11 @@ def files_in(*search_paths):
return files
+def is_readable_file(file_path):
+ """Return True if the path passed as argument is readable"""
+ return os.path.isfile(file_path) and os.access(file_path, os.R_OK)
+
+
@system_path_filter
def search_paths_for_executables(*path_hints):
"""Given a list of path hints returns a list of paths where
@@ -1969,6 +1974,38 @@ def search_paths_for_executables(*path_hints):
@system_path_filter
+def search_paths_for_libraries(*path_hints):
+ """Given a list of path hints returns a list of paths where
+ to search for a shared library.
+
+ Args:
+ *path_hints (list of paths): list of paths taken into
+ consideration for a search
+
+ Returns:
+ A list containing the real path of every existing directory
+ in `path_hints` and its `lib` and `lib64` subdirectory if it exists.
+ """
+ library_paths = []
+ for path in path_hints:
+ if not os.path.isdir(path):
+ continue
+
+ path = os.path.abspath(path)
+ library_paths.append(path)
+
+ lib_dir = os.path.join(path, 'lib')
+ if os.path.isdir(lib_dir):
+ library_paths.append(lib_dir)
+
+ lib64_dir = os.path.join(path, 'lib64')
+ if os.path.isdir(lib64_dir):
+ library_paths.append(lib64_dir)
+
+ return library_paths
+
+
+@system_path_filter
def partition_path(path, entry=None):
"""
Split the prefixes of the path at the first occurrence of entry and
diff --git a/lib/spack/spack/cmd/external.py b/lib/spack/spack/cmd/external.py
index 7e73572028..42f033d979 100644
--- a/lib/spack/spack/cmd/external.py
+++ b/lib/spack/spack/cmd/external.py
@@ -91,6 +91,8 @@ def external_find(args):
packages_to_check = spack.repo.path.all_packages()
detected_packages = spack.detection.by_executable(packages_to_check)
+ detected_packages.update(spack.detection.by_library(packages_to_check))
+
new_entries = spack.detection.update_configuration(
detected_packages, scope=args.scope, buildable=not args.not_buildable
)
diff --git a/lib/spack/spack/detection/__init__.py b/lib/spack/spack/detection/__init__.py
index e2976dacdd..586f39fd92 100644
--- a/lib/spack/spack/detection/__init__.py
+++ b/lib/spack/spack/detection/__init__.py
@@ -3,10 +3,11 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from .common import DetectedPackage, executable_prefix, update_configuration
-from .path import by_executable, executables_in_path
+from .path import by_executable, by_library, executables_in_path
__all__ = [
'DetectedPackage',
+ 'by_library',
'by_executable',
'executables_in_path',
'executable_prefix',
diff --git a/lib/spack/spack/detection/common.py b/lib/spack/spack/detection/common.py
index 4f56a526aa..a8f0e82a87 100644
--- a/lib/spack/spack/detection/common.py
+++ b/lib/spack/spack/detection/common.py
@@ -150,6 +150,29 @@ def executable_prefix(executable_dir):
return os.sep.join(components[:idx])
+def library_prefix(library_dir):
+ """Given a directory where an library is found, guess the prefix
+ (i.e. the "root" directory of that installation) and return it.
+
+ Args:
+ library_dir: directory where an library is found
+ """
+ # Given a prefix where an library is found, assuming that prefix
+ # contains /lib/ or /lib64/, strip off the 'lib' or 'lib64' directory
+ # to get a Spack-compatible prefix
+ assert os.path.isdir(library_dir)
+
+ components = library_dir.split(os.sep)
+ if 'lib64' in components:
+ idx = components.index('lib64')
+ return os.sep.join(components[:idx])
+ elif 'lib' in components:
+ idx = components.index('lib')
+ return os.sep.join(components[:idx])
+ else:
+ return library_dir
+
+
def update_configuration(detected_packages, scope=None, buildable=True):
"""Add the packages passed as arguments to packages.yaml
diff --git a/lib/spack/spack/detection/path.py b/lib/spack/spack/detection/path.py
index ef0bffad67..7933114827 100644
--- a/lib/spack/spack/detection/path.py
+++ b/lib/spack/spack/detection/path.py
@@ -25,6 +25,7 @@ from .common import (
executable_prefix,
find_win32_additional_install_paths,
is_executable,
+ library_prefix,
)
@@ -72,6 +73,34 @@ def executables_in_path(path_hints=None):
return path_to_exe
+def libraries_in_ld_library_path(path_hints=None):
+ """Get the paths of all libraries available from LD_LIBRARY_PATH.
+
+ For convenience, this is constructed as a dictionary where the keys are
+ the library paths and the values are the names of the libraries
+ (i.e. the basename of the library path).
+
+ There may be multiple paths with the same basename. In this case it is
+ assumed there are two different instances of the library.
+
+ Args:
+ path_hints (list): list of paths to be searched. If None the list will be
+ constructed based on the LD_LIBRARY_PATH environment variable.
+ """
+ path_hints = path_hints or spack.util.environment.get_path('LD_LIBRARY_PATH')
+ search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints)
+
+ path_to_lib = {}
+ # Reverse order of search directories so that a lib in the first
+ # LD_LIBRARY_PATH entry overrides later entries
+ for search_path in reversed(search_paths):
+ for lib in os.listdir(search_path):
+ lib_path = os.path.join(search_path, lib)
+ if llnl.util.filesystem.is_readable_file(lib_path):
+ path_to_lib[lib_path] = lib
+ return path_to_lib
+
+
def _group_by_prefix(paths):
groups = collections.defaultdict(set)
for p in paths:
@@ -79,6 +108,106 @@ def _group_by_prefix(paths):
return groups.items()
+# TODO consolidate this with by_executable
+# Packages should be able to define both .libraries and .executables in the future
+# determine_spec_details should get all relevant libraries and executables in one call
+def by_library(packages_to_check, path_hints=None):
+ # Techniques for finding libraries is determined on a per recipe basis in
+ # the determine_version class method. Some packages will extract the
+ # version number from a shared libraries filename.
+ # Other libraries could use the strings function to extract it as described
+ # in https://unix.stackexchange.com/questions/58846/viewing-linux-library-executable-version-info
+ """Return the list of packages that have been detected on the system,
+ searching by LD_LIBRARY_PATH.
+
+ Args:
+ packages_to_check (list): list of packages to be detected
+ path_hints (list): list of paths to be searched. If None the list will be
+ constructed based on the LD_LIBRARY_PATH environment variable.
+ """
+ path_to_lib_name = libraries_in_ld_library_path(path_hints=path_hints)
+ lib_pattern_to_pkgs = collections.defaultdict(list)
+ for pkg in packages_to_check:
+ if hasattr(pkg, 'libraries'):
+ for lib in pkg.libraries:
+ lib_pattern_to_pkgs[lib].append(pkg)
+
+ pkg_to_found_libs = collections.defaultdict(set)
+ for lib_pattern, pkgs in lib_pattern_to_pkgs.items():
+ compiled_re = re.compile(lib_pattern)
+ for path, lib in path_to_lib_name.items():
+ if compiled_re.search(lib):
+ for pkg in pkgs:
+ pkg_to_found_libs[pkg].add(path)
+
+ pkg_to_entries = collections.defaultdict(list)
+ resolved_specs = {} # spec -> lib found for the spec
+
+ for pkg, libs in pkg_to_found_libs.items():
+ if not hasattr(pkg, 'determine_spec_details'):
+ llnl.util.tty.warn(
+ "{0} must define 'determine_spec_details' in order"
+ " for Spack to detect externally-provided instances"
+ " of the package.".format(pkg.name))
+ continue
+
+ for prefix, libs_in_prefix in sorted(_group_by_prefix(libs)):
+ try:
+ specs = _convert_to_iterable(
+ pkg.determine_spec_details(prefix, libs_in_prefix)
+ )
+ except Exception as e:
+ specs = []
+ msg = 'error detecting "{0}" from prefix {1} [{2}]'
+ warnings.warn(msg.format(pkg.name, prefix, str(e)))
+
+ if not specs:
+ llnl.util.tty.debug(
+ 'The following libraries in {0} were decidedly not '
+ 'part of the package {1}: {2}'
+ .format(prefix, pkg.name, ', '.join(
+ _convert_to_iterable(libs_in_prefix)))
+ )
+
+ for spec in specs:
+ pkg_prefix = library_prefix(prefix)
+
+ if not pkg_prefix:
+ msg = "no lib/ or lib64/ dir found in {0}. Cannot "
+ "add it as a Spack package"
+ llnl.util.tty.debug(msg.format(prefix))
+ continue
+
+ if spec in resolved_specs:
+ prior_prefix = ', '.join(
+ _convert_to_iterable(resolved_specs[spec]))
+
+ llnl.util.tty.debug(
+ "Libraries in {0} and {1} are both associated"
+ " with the same spec {2}"
+ .format(prefix, prior_prefix, str(spec)))
+ continue
+ else:
+ resolved_specs[spec] = prefix
+
+ try:
+ spec.validate_detection()
+ except Exception as e:
+ msg = ('"{0}" has been detected on the system but will '
+ 'not be added to packages.yaml [reason={1}]')
+ llnl.util.tty.warn(msg.format(spec, str(e)))
+ continue
+
+ if spec.external_path:
+ pkg_prefix = spec.external_path
+
+ pkg_to_entries[pkg.name].append(
+ DetectedPackage(spec=spec, prefix=pkg_prefix)
+ )
+
+ return pkg_to_entries
+
+
def by_executable(packages_to_check, path_hints=None):
"""Return the list of packages that have been detected on the system,
searching by path.
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 532a9ff318..96ad5cab8c 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -177,13 +177,21 @@ class DetectablePackageMeta(object):
for the detection function.
"""
def __init__(cls, name, bases, attr_dict):
+ if hasattr(cls, 'executables') and hasattr(cls, 'libraries'):
+ msg = "a package can have either an 'executables' or 'libraries' attribute"
+ msg += " [package '{0.name}' defines both]"
+ raise spack.error.SpackError(msg.format(cls))
+
# On windows, extend the list of regular expressions to look for
# filenames ending with ".exe"
# (in some cases these regular expressions include "$" to avoid
# pulling in filenames with unexpected suffixes, but this allows
# for example detecting "foo.exe" when the package writer specified
# that "foo" was a possible executable.
- if hasattr(cls, 'executables'):
+
+ # If a package has the executables or libraries attribute then it's
+ # assumed to be detectable
+ if hasattr(cls, 'executables') or hasattr(cls, 'libraries'):
@property
def platform_executables(self):
def to_windows_exe(exe):
@@ -201,35 +209,37 @@ class DetectablePackageMeta(object):
return plat_exe
@classmethod
- def determine_spec_details(cls, prefix, exes_in_prefix):
+ def determine_spec_details(cls, prefix, objs_in_prefix):
"""Allow ``spack external find ...`` to locate installations.
Args:
prefix (str): the directory containing the executables
- exes_in_prefix (set): the executables that match the regex
+ or libraries
+ objs_in_prefix (set): the executables or libraries that
+ match the regex
Returns:
The list of detected specs for this package
"""
- exes_by_version = collections.defaultdict(list)
+ objs_by_version = collections.defaultdict(list)
# The default filter function is the identity function for the
# list of executables
filter_fn = getattr(cls, 'filter_detected_exes',
lambda x, exes: exes)
- exes_in_prefix = filter_fn(prefix, exes_in_prefix)
- for exe in exes_in_prefix:
+ objs_in_prefix = filter_fn(prefix, objs_in_prefix)
+ for obj in objs_in_prefix:
try:
- version_str = cls.determine_version(exe)
+ version_str = cls.determine_version(obj)
if version_str:
- exes_by_version[version_str].append(exe)
+ objs_by_version[version_str].append(obj)
except Exception as e:
msg = ('An error occurred when trying to detect '
'the version of "{0}" [{1}]')
- tty.debug(msg.format(exe, str(e)))
+ tty.debug(msg.format(obj, str(e)))
specs = []
- for version_str, exes in exes_by_version.items():
- variants = cls.determine_variants(exes, version_str)
+ for version_str, objs in objs_by_version.items():
+ variants = cls.determine_variants(objs, version_str)
# Normalize output to list
if not isinstance(variants, list):
variants = [variants]
@@ -265,7 +275,7 @@ class DetectablePackageMeta(object):
return sorted(specs)
@classmethod
- def determine_variants(cls, exes, version_str):
+ def determine_variants(cls, objs, version_str):
return ''
# Register the class as a detectable package
generated by cgit v1.2.3-60-g2f50 (git 2.42.0) at 2024-07-04 17:37:45 +0000