"spack external find": also find library-only packages (#28005)

Update "spack external find --all" to also find library-only packages.
A Package can add a ".libraries" attribute, which is a list of regular
expressions to use to find libraries associated with the Package.
"spack external find --all" will search LD_LIBRARY_PATH for potential
libraries.

This PR adds examples for NCCL, RCCL, and hipblas packages. These
examples specify the suffix ".so" for the regular expressions used
to find libraries, so generally are only useful for detecting library
packages on Linux.

6 files changed, 215 insertions, 13 deletions

diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index e16db8ba8a..fbc6a3d7ce 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -1940,6 +1940,11 @@ def files_in(*search_paths):
     return files
 
 
+def is_readable_file(file_path):
+    """Return True if the path passed as argument is readable"""
+    return os.path.isfile(file_path) and os.access(file_path, os.R_OK)
+
+
 @system_path_filter
 def search_paths_for_executables(*path_hints):
     """Given a list of path hints returns a list of paths where
@@ -1969,6 +1974,38 @@ def search_paths_for_executables(*path_hints):
 
 
 @system_path_filter
+def search_paths_for_libraries(*path_hints):
+    """Given a list of path hints returns a list of paths where
+    to search for a shared library.
+
+    Args:
+        *path_hints (list of paths): list of paths taken into
+            consideration for a search
+
+    Returns:
+        A list containing the real path of every existing directory
+        in `path_hints` and its `lib` and `lib64` subdirectory if it exists.
+    """
+    library_paths = []
+    for path in path_hints:
+        if not os.path.isdir(path):
+            continue
+
+        path = os.path.abspath(path)
+        library_paths.append(path)
+
+        lib_dir = os.path.join(path, 'lib')
+        if os.path.isdir(lib_dir):
+            library_paths.append(lib_dir)
+
+        lib64_dir = os.path.join(path, 'lib64')
+        if os.path.isdir(lib64_dir):
+            library_paths.append(lib64_dir)
+
+    return library_paths
+
+
+@system_path_filter
 def partition_path(path, entry=None):
     """
     Split the prefixes of the path at the first occurrence of entry and
diff --git a/lib/spack/spack/cmd/external.py b/lib/spack/spack/cmd/external.py
index 7e73572028..42f033d979 100644
--- a/lib/spack/spack/cmd/external.py
+++ b/lib/spack/spack/cmd/external.py
@@ -91,6 +91,8 @@ def external_find(args):
         packages_to_check = spack.repo.path.all_packages()
 
     detected_packages = spack.detection.by_executable(packages_to_check)
+    detected_packages.update(spack.detection.by_library(packages_to_check))
+
     new_entries = spack.detection.update_configuration(
         detected_packages, scope=args.scope, buildable=not args.not_buildable
     )
diff --git a/lib/spack/spack/detection/__init__.py b/lib/spack/spack/detection/__init__.py
index e2976dacdd..586f39fd92 100644
--- a/lib/spack/spack/detection/__init__.py
+++ b/lib/spack/spack/detection/__init__.py
@@ -3,10 +3,11 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 from .common import DetectedPackage, executable_prefix, update_configuration
-from .path import by_executable, executables_in_path
+from .path import by_executable, by_library, executables_in_path
 
 __all__ = [
     'DetectedPackage',
+    'by_library',
     'by_executable',
     'executables_in_path',
     'executable_prefix',
diff --git a/lib/spack/spack/detection/common.py b/lib/spack/spack/detection/common.py
index 4f56a526aa..a8f0e82a87 100644
--- a/lib/spack/spack/detection/common.py
+++ b/lib/spack/spack/detection/common.py
@@ -150,6 +150,29 @@ def executable_prefix(executable_dir):
     return os.sep.join(components[:idx])
 
 
+def library_prefix(library_dir):
+    """Given a directory where an library is found, guess the prefix
+    (i.e. the "root" directory of that installation) and return it.
+
+    Args:
+        library_dir: directory where an library is found
+    """
+    # Given a prefix where an library is found, assuming that prefix
+    # contains /lib/ or /lib64/, strip off the 'lib' or 'lib64' directory
+    # to get a Spack-compatible prefix
+    assert os.path.isdir(library_dir)
+
+    components = library_dir.split(os.sep)
+    if 'lib64' in components:
+        idx = components.index('lib64')
+        return os.sep.join(components[:idx])
+    elif 'lib' in components:
+        idx = components.index('lib')
+        return os.sep.join(components[:idx])
+    else:
+        return library_dir
+
+
 def update_configuration(detected_packages, scope=None, buildable=True):
     """Add the packages passed as arguments to packages.yaml
 
diff --git a/lib/spack/spack/detection/path.py b/lib/spack/spack/detection/path.py
index ef0bffad67..7933114827 100644
--- a/lib/spack/spack/detection/path.py
+++ b/lib/spack/spack/detection/path.py
@@ -25,6 +25,7 @@ from .common import (
     executable_prefix,
     find_win32_additional_install_paths,
     is_executable,
+    library_prefix,
 )
 
 
@@ -72,6 +73,34 @@ def executables_in_path(path_hints=None):
     return path_to_exe
 
 
+def libraries_in_ld_library_path(path_hints=None):
+    """Get the paths of all libraries available from LD_LIBRARY_PATH.
+
+    For convenience, this is constructed as a dictionary where the keys are
+    the library paths and the values are the names of the libraries
+    (i.e. the basename of the library path).
+
+    There may be multiple paths with the same basename. In this case it is
+    assumed there are two different instances of the library.
+
+    Args:
+        path_hints (list): list of paths to be searched. If None the list will be
+            constructed based on the LD_LIBRARY_PATH environment variable.
+    """
+    path_hints = path_hints or spack.util.environment.get_path('LD_LIBRARY_PATH')
+    search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints)
+
+    path_to_lib = {}
+    # Reverse order of search directories so that a lib in the first
+    # LD_LIBRARY_PATH entry overrides later entries
+    for search_path in reversed(search_paths):
+        for lib in os.listdir(search_path):
+            lib_path = os.path.join(search_path, lib)
+            if llnl.util.filesystem.is_readable_file(lib_path):
+                path_to_lib[lib_path] = lib
+    return path_to_lib
+
+
 def _group_by_prefix(paths):
     groups = collections.defaultdict(set)
     for p in paths:
@@ -79,6 +108,106 @@ def _group_by_prefix(paths):
     return groups.items()
 
 
+# TODO consolidate this with by_executable
+# Packages should be able to define both .libraries and .executables in the future
+# determine_spec_details should get all relevant libraries and executables in one call
+def by_library(packages_to_check, path_hints=None):
+    # Techniques for finding libraries is determined on a per recipe basis in
+    # the determine_version class method. Some packages will extract the
+    # version number from a shared libraries filename.
+    # Other libraries could use the strings function to extract it as described
+    # in https://unix.stackexchange.com/questions/58846/viewing-linux-library-executable-version-info
+    """Return the list of packages that have been detected on the system,
+    searching by LD_LIBRARY_PATH.
+
+    Args:
+        packages_to_check (list): list of packages to be detected
+        path_hints (list): list of paths to be searched. If None the list will be
+            constructed based on the LD_LIBRARY_PATH environment variable.
+    """
+    path_to_lib_name = libraries_in_ld_library_path(path_hints=path_hints)
+    lib_pattern_to_pkgs = collections.defaultdict(list)
+    for pkg in packages_to_check:
+        if hasattr(pkg, 'libraries'):
+            for lib in pkg.libraries:
+                lib_pattern_to_pkgs[lib].append(pkg)
+
+    pkg_to_found_libs = collections.defaultdict(set)
+    for lib_pattern, pkgs in lib_pattern_to_pkgs.items():
+        compiled_re = re.compile(lib_pattern)
+        for path, lib in path_to_lib_name.items():
+            if compiled_re.search(lib):
+                for pkg in pkgs:
+                    pkg_to_found_libs[pkg].add(path)
+
+    pkg_to_entries = collections.defaultdict(list)
+    resolved_specs = {}  # spec -> lib found for the spec
+
+    for pkg, libs in pkg_to_found_libs.items():
+        if not hasattr(pkg, 'determine_spec_details'):
+            llnl.util.tty.warn(
+                "{0} must define 'determine_spec_details' in order"
+                " for Spack to detect externally-provided instances"
+                " of the package.".format(pkg.name))
+            continue
+
+        for prefix, libs_in_prefix in sorted(_group_by_prefix(libs)):
+            try:
+                specs = _convert_to_iterable(
+                    pkg.determine_spec_details(prefix, libs_in_prefix)
+                )
+            except Exception as e:
+                specs = []
+                msg = 'error detecting "{0}" from prefix {1} [{2}]'
+                warnings.warn(msg.format(pkg.name, prefix, str(e)))
+
+            if not specs:
+                llnl.util.tty.debug(
+                    'The following libraries in {0} were decidedly not '
+                    'part of the package {1}: {2}'
+                    .format(prefix, pkg.name, ', '.join(
+                        _convert_to_iterable(libs_in_prefix)))
+                )
+
+            for spec in specs:
+                pkg_prefix = library_prefix(prefix)
+
+                if not pkg_prefix:
+                    msg = "no lib/ or lib64/ dir found in {0}. Cannot "
+                    "add it as a Spack package"
+                    llnl.util.tty.debug(msg.format(prefix))
+                    continue
+
+                if spec in resolved_specs:
+                    prior_prefix = ', '.join(
+                        _convert_to_iterable(resolved_specs[spec]))
+
+                    llnl.util.tty.debug(
+                        "Libraries in {0} and {1} are both associated"
+                        " with the same spec {2}"
+                        .format(prefix, prior_prefix, str(spec)))
+                    continue
+                else:
+                    resolved_specs[spec] = prefix
+
+                try:
+                    spec.validate_detection()
+                except Exception as e:
+                    msg = ('"{0}" has been detected on the system but will '
+                           'not be added to packages.yaml [reason={1}]')
+                    llnl.util.tty.warn(msg.format(spec, str(e)))
+                    continue
+
+                if spec.external_path:
+                    pkg_prefix = spec.external_path
+
+                pkg_to_entries[pkg.name].append(
+                    DetectedPackage(spec=spec, prefix=pkg_prefix)
+                )
+
+    return pkg_to_entries
+
+
 def by_executable(packages_to_check, path_hints=None):
     """Return the list of packages that have been detected on the system,
     searching by path.
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 532a9ff318..96ad5cab8c 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -177,13 +177,21 @@ class DetectablePackageMeta(object):
     for the detection function.
     """
     def __init__(cls, name, bases, attr_dict):
+        if hasattr(cls, 'executables') and hasattr(cls, 'libraries'):
+            msg = "a package can have either an 'executables' or 'libraries' attribute"
+            msg += " [package '{0.name}' defines both]"
+            raise spack.error.SpackError(msg.format(cls))
+
         # On windows, extend the list of regular expressions to look for
         # filenames ending with ".exe"
         # (in some cases these regular expressions include "$" to avoid
         # pulling in filenames with unexpected suffixes, but this allows
         # for example detecting "foo.exe" when the package writer specified
         # that "foo" was a possible executable.
-        if hasattr(cls, 'executables'):
+
+        # If a package has the executables or libraries  attribute then it's
+        # assumed to be detectable
+        if hasattr(cls, 'executables') or hasattr(cls, 'libraries'):
             @property
             def platform_executables(self):
                 def to_windows_exe(exe):
@@ -201,35 +209,37 @@ class DetectablePackageMeta(object):
                 return plat_exe
 
             @classmethod
-            def determine_spec_details(cls, prefix, exes_in_prefix):
+            def determine_spec_details(cls, prefix, objs_in_prefix):
                 """Allow ``spack external find ...`` to locate installations.
 
                 Args:
                     prefix (str): the directory containing the executables
-                    exes_in_prefix (set): the executables that match the regex
+                                  or libraries
+                    objs_in_prefix (set): the executables or libraries that
+                                          match the regex
 
                 Returns:
                     The list of detected specs for this package
                 """
-                exes_by_version = collections.defaultdict(list)
+                objs_by_version = collections.defaultdict(list)
                 # The default filter function is the identity function for the
                 # list of executables
                 filter_fn = getattr(cls, 'filter_detected_exes',
                                     lambda x, exes: exes)
-                exes_in_prefix = filter_fn(prefix, exes_in_prefix)
-                for exe in exes_in_prefix:
+                objs_in_prefix = filter_fn(prefix, objs_in_prefix)
+                for obj in objs_in_prefix:
                     try:
-                        version_str = cls.determine_version(exe)
+                        version_str = cls.determine_version(obj)
                         if version_str:
-                            exes_by_version[version_str].append(exe)
+                            objs_by_version[version_str].append(obj)
                     except Exception as e:
                         msg = ('An error occurred when trying to detect '
                                'the version of "{0}" [{1}]')
-                        tty.debug(msg.format(exe, str(e)))
+                        tty.debug(msg.format(obj, str(e)))
 
                 specs = []
-                for version_str, exes in exes_by_version.items():
-                    variants = cls.determine_variants(exes, version_str)
+                for version_str, objs in objs_by_version.items():
+                    variants = cls.determine_variants(objs, version_str)
                     # Normalize output to list
                     if not isinstance(variants, list):
                         variants = [variants]
@@ -265,7 +275,7 @@ class DetectablePackageMeta(object):
                 return sorted(specs)
 
             @classmethod
-            def determine_variants(cls, exes, version_str):
+            def determine_variants(cls, objs, version_str):
                 return ''
 
             # Register the class as a detectable package