Automatically find externals (#15158)

Add a `spack external find` command that tries to populate
`packages.yaml` with external packages from the user's `$PATH`. This
focuses on finding build dependencies. Currently, support has only been
added for `cmake`.

For a package to be discoverable with `spack external find`, it must define:
  * an `executables` class attribute containing a list of
    regular expressions that match executable names.
  * a `determine_spec_details(prefix, specs_in_prefix)` method

Spack will call `determine_spec_details()` once for each prefix where
executables are found, passing in the path to the prefix and the path to
all found executables. The package is responsible for invoking the
executables and figuring out what type of installation(s) are in the
prefix, and returning one or more specs (each with version, variants or
whatever else the user decides to include in the spec).

The found specs and prefixes will be added to the user's `packages.yaml`
file. Providing the `--not-buildable` option will mark all generated
entries in `packages.yaml` as `buildable: False`

3 files changed, 479 insertions, 0 deletions

diff --git a/lib/spack/docs/build_settings.rst b/lib/spack/docs/build_settings.rst
index cfd850af28..dee892f272 100644
--- a/lib/spack/docs/build_settings.rst
+++ b/lib/spack/docs/build_settings.rst
@@ -158,6 +158,37 @@ Spack can then use any of the listed external implementations of MPI
 to satisfy a dependency, and will choose depending on the compiler and
 architecture.
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Automatically Find External Packages
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A user can run the :ref:`spack external find <spack-external-find>` command
+to search for system-provided packages and add them to ``packages.yaml``.
+After running this command your ``packages.yaml`` may include new entries:
+
+.. code-block:: yaml
+
+   packages:
+     cmake:
+       paths:
+         cmake@3.17.2: /usr
+
+Generally this is useful for detecting a small set of commonly-used packages;
+for now this is generally limited to finding build-only dependencies.
+Specific limitations include:
+
+* A package must define ``executables`` and ``determine_spec_details``
+  for Spack to locate instances of that package.
+* This is currently intended to find build dependencies rather than
+  library packages.
+* Spack does not overwrite existing entries in the package configuration:
+  If there is an external defined for a spec at any configuration scope,
+  then Spack will not add a new external entry (``spack config blame packages``
+  can help locate all external entries).
+* Currently this logic is focused on examining ``PATH`` and does not
+  search through modules (although it should find the package if a
+  module is loaded for it).
+
 .. _concretization-preferences:
 
 --------------------------
diff --git a/lib/spack/spack/cmd/external.py b/lib/spack/spack/cmd/external.py
new file mode 100644
index 0000000000..f93deaba03
--- /dev/null
+++ b/lib/spack/spack/cmd/external.py
@@ -0,0 +1,271 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from __future__ import print_function
+from collections import defaultdict, namedtuple
+import argparse
+import os
+import re
+import six
+
+import spack
+import spack.error
+import llnl.util.tty as tty
+import spack.util.spack_yaml as syaml
+import spack.util.environment
+import llnl.util.filesystem
+
+description = "add external packages to Spack configuration"
+section = "config"
+level = "short"
+
+
+def setup_parser(subparser):
+    sp = subparser.add_subparsers(
+        metavar='SUBCOMMAND', dest='external_command')
+
+    find_parser = sp.add_parser('find', help=external_find.__doc__)
+    find_parser.add_argument(
+        '--not-buildable', action='store_true', default=False,
+        help="packages with detected externals won't be built with Spack")
+    find_parser.add_argument('packages', nargs=argparse.REMAINDER)
+
+
+def is_executable(path):
+    return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def _get_system_executables():
+    """Get the paths of all executables available from the current PATH.
+
+    For convenience, this is constructed as a dictionary where the keys are
+    the executable paths and the values are the names of the executables
+    (i.e. the basename of the executable path).
+
+    There may be multiple paths with the same basename. In this case it is
+    assumed there are two different instances of the executable.
+    """
+    path_hints = spack.util.environment.get_path('PATH')
+    search_paths = llnl.util.filesystem.search_paths_for_executables(
+        *path_hints)
+
+    path_to_exe = {}
+    # Reverse order of search directories so that an exe in the first PATH
+    # entry overrides later entries
+    for search_path in reversed(search_paths):
+        for exe in os.listdir(search_path):
+            exe_path = os.path.join(search_path, exe)
+            if is_executable(exe_path):
+                path_to_exe[exe_path] = exe
+    return path_to_exe
+
+
+ExternalPackageEntry = namedtuple(
+    'ExternalPackageEntry',
+    ['spec', 'base_dir'])
+
+
+def _generate_pkg_config(external_pkg_entries):
+    """Generate config according to the packages.yaml schema for a single
+    package.
+
+    This does not generate the entire packages.yaml. For example, given some
+    external entries for the CMake package, this could return::
+
+       { 'paths': {
+             'cmake@3.17.1': '/opt/cmake-3.17.1/',
+             'cmake@3.16.5': '/opt/cmake-3.16.5/'
+         }
+       }
+    """
+    paths_dict = syaml.syaml_dict()
+    for e in external_pkg_entries:
+        if not _spec_is_valid(e.spec):
+            continue
+        paths_dict[str(e.spec)] = e.base_dir
+    pkg_dict = syaml.syaml_dict()
+    pkg_dict['paths'] = paths_dict
+
+    return pkg_dict
+
+
+def _spec_is_valid(spec):
+    try:
+        str(spec)
+    except spack.error.SpackError:
+        # It is assumed here that we can at least extract the package name from
+        # the spec so we can look up the implementation of
+        # determine_spec_details
+        tty.warn('Constructed spec for {0} does not have a string'
+                 ' representation'.format(spec.name))
+        return False
+
+    try:
+        spack.spec.Spec(str(spec))
+    except spack.error.SpackError:
+        tty.warn('Constructed spec has a string representation but the string'
+                 ' representation does not evaluate to a valid spec: {0}'
+                 .format(str(spec)))
+        return False
+
+    return True
+
+
+def external_find(args):
+    if args.packages:
+        packages_to_check = list(spack.repo.get(pkg) for pkg in args.packages)
+    else:
+        packages_to_check = spack.repo.path.all_packages()
+
+    pkg_to_entries = _get_external_packages(packages_to_check)
+    _update_pkg_config(pkg_to_entries, args.not_buildable)
+
+
+def _group_by_prefix(paths):
+    groups = defaultdict(set)
+    for p in paths:
+        groups[os.path.dirname(p)].add(p)
+    return groups.items()
+
+
+def _convert_to_iterable(single_val_or_multiple):
+    x = single_val_or_multiple
+    if x is None:
+        return []
+    elif isinstance(x, six.string_types):
+        return [x]
+    elif isinstance(x, spack.spec.Spec):
+        # Specs are iterable, but a single spec should be converted to a list
+        return [x]
+
+    try:
+        iter(x)
+        return x
+    except TypeError:
+        return [x]
+
+
+def _determine_base_dir(prefix):
+    # Given a prefix where an executable is found, assuming that prefix ends
+    # with /bin/, strip off the 'bin' directory to get a Spack-compatible
+    # prefix
+    assert os.path.isdir(prefix)
+    if os.path.basename(prefix) == 'bin':
+        return os.path.dirname(prefix)
+
+
+def _get_predefined_externals():
+    # Pull from all scopes when looking for preexisting external package
+    # entries
+    pkg_config = spack.config.get('packages')
+    already_defined_specs = set()
+    for pkg_name, per_pkg_cfg in pkg_config.items():
+        paths = per_pkg_cfg.get('paths', {})
+        already_defined_specs.update(spack.spec.Spec(k) for k in paths)
+        modules = per_pkg_cfg.get('modules', {})
+        already_defined_specs.update(spack.spec.Spec(k) for k in modules)
+    return already_defined_specs
+
+
+def _update_pkg_config(pkg_to_entries, not_buildable):
+    predefined_external_specs = _get_predefined_externals()
+
+    pkg_to_cfg = {}
+    for pkg_name, ext_pkg_entries in pkg_to_entries.items():
+        new_entries = list(
+            e for e in ext_pkg_entries
+            if (e.spec not in predefined_external_specs))
+
+        pkg_config = _generate_pkg_config(new_entries)
+        if not_buildable:
+            pkg_config['buildable'] = False
+        pkg_to_cfg[pkg_name] = pkg_config
+
+    cfg_scope = spack.config.default_modify_scope()
+    pkgs_cfg = spack.config.get('packages', scope=cfg_scope)
+
+    spack.config._merge_yaml(pkgs_cfg, pkg_to_cfg)
+    spack.config.set('packages', pkgs_cfg, scope=cfg_scope)
+
+
+def _get_external_packages(packages_to_check, system_path_to_exe=None):
+    if not system_path_to_exe:
+        system_path_to_exe = _get_system_executables()
+
+    exe_pattern_to_pkgs = defaultdict(list)
+    for pkg in packages_to_check:
+        if hasattr(pkg, 'executables'):
+            for exe in pkg.executables:
+                exe_pattern_to_pkgs[exe].append(pkg)
+
+    pkg_to_found_exes = defaultdict(set)
+    for exe_pattern, pkgs in exe_pattern_to_pkgs.items():
+        compiled_re = re.compile(exe_pattern)
+        for path, exe in system_path_to_exe.items():
+            if compiled_re.search(exe):
+                for pkg in pkgs:
+                    pkg_to_found_exes[pkg].add(path)
+
+    pkg_to_entries = defaultdict(list)
+    resolved_specs = {}  # spec -> exe found for the spec
+
+    for pkg, exes in pkg_to_found_exes.items():
+        if not hasattr(pkg, 'determine_spec_details'):
+            tty.warn("{0} must define 'determine_spec_details' in order"
+                     " for Spack to detect externally-provided instances"
+                     " of the package.".format(pkg.name))
+            continue
+
+        # TODO: iterate through this in a predetermined order (e.g. by package
+        # name) to get repeatable results when there are conflicts. Note that
+        # if we take the prefixes returned by _group_by_prefix, then consider
+        # them in the order that they appear in PATH, this should be sufficient
+        # to get repeatable results.
+        for prefix, exes_in_prefix in _group_by_prefix(exes):
+            # TODO: multiple instances of a package can live in the same
+            # prefix, and a package implementation can return multiple specs
+            # for one prefix, but without additional details (e.g. about the
+            # naming scheme which differentiates them), the spec won't be
+            # usable.
+            specs = _convert_to_iterable(
+                pkg.determine_spec_details(prefix, exes_in_prefix))
+
+            if not specs:
+                tty.debug(
+                    'The following executables in {0} were decidedly not'
+                    'part of the package {1}: {2}'
+                    .format(prefix, pkg.name, ', '.join(exes_in_prefix))
+                )
+
+            for spec in specs:
+                pkg_prefix = _determine_base_dir(prefix)
+
+                if not pkg_prefix:
+                    tty.debug("{0} does not end with a 'bin/' directory: it"
+                              " cannot be added as a Spack package"
+                              .format(prefix))
+                    continue
+
+                if spec in resolved_specs:
+                    prior_prefix = ', '.join(resolved_specs[spec])
+
+                    tty.debug(
+                        "Executables in {0} and {1} are both associated"
+                        " with the same spec {2}"
+                        .format(prefix, prior_prefix, str(spec)))
+                    continue
+                else:
+                    resolved_specs[spec] = prefix
+
+                pkg_to_entries[pkg.name].append(
+                    ExternalPackageEntry(spec=spec, base_dir=pkg_prefix))
+
+    return pkg_to_entries
+
+
+def external(parser, args):
+    action = {'find': external_find}
+
+    action[args.external_command](args)
diff --git a/lib/spack/spack/test/cmd/external.py b/lib/spack/spack/test/cmd/external.py
new file mode 100644
index 0000000000..0bdf67fe3e
--- /dev/null
+++ b/lib/spack/spack/test/cmd/external.py
@@ -0,0 +1,177 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import pytest
+import os
+import stat
+
+import spack
+from spack.spec import Spec
+from spack.cmd.external import ExternalPackageEntry
+from spack.main import SpackCommand
+
+
+@pytest.fixture()
+def create_exe(tmpdir_factory):
+    def _create_exe(exe_name, content):
+        base_prefix = tmpdir_factory.mktemp('base-prefix')
+        base_prefix.ensure('bin', dir=True)
+        exe_path = str(base_prefix.join('bin', exe_name))
+        with open(exe_path, 'w') as f:
+            f.write("""\
+#!/bin/bash
+
+echo "{0}"
+""".format(content))
+
+        st = os.stat(exe_path)
+        os.chmod(exe_path, st.st_mode | stat.S_IEXEC)
+        return exe_path
+
+    yield _create_exe
+
+
+def test_find_external_single_package(create_exe):
+    pkgs_to_check = [spack.repo.get('cmake')]
+
+    cmake_path = create_exe("cmake", "cmake version 1.foo")
+    system_path_to_exe = {cmake_path: 'cmake'}
+
+    pkg_to_entries = spack.cmd.external._get_external_packages(
+        pkgs_to_check, system_path_to_exe)
+
+    pkg, entries = next(iter(pkg_to_entries.items()))
+    single_entry = next(iter(entries))
+
+    assert single_entry.spec == Spec('cmake@1.foo')
+
+
+def test_find_external_two_instances_same_package(create_exe):
+    pkgs_to_check = [spack.repo.get('cmake')]
+
+    # Each of these cmake instances is created in a different prefix
+    cmake_path1 = create_exe("cmake", "cmake version 1.foo")
+    cmake_path2 = create_exe("cmake", "cmake version 3.17.2")
+    system_path_to_exe = {
+        cmake_path1: 'cmake',
+        cmake_path2: 'cmake'}
+
+    pkg_to_entries = spack.cmd.external._get_external_packages(
+        pkgs_to_check, system_path_to_exe)
+
+    pkg, entries = next(iter(pkg_to_entries.items()))
+    spec_to_path = dict((e.spec, e.base_dir) for e in entries)
+    assert spec_to_path[Spec('cmake@1.foo')] == (
+        spack.cmd.external._determine_base_dir(os.path.dirname(cmake_path1)))
+    assert spec_to_path[Spec('cmake@3.17.2')] == (
+        spack.cmd.external._determine_base_dir(os.path.dirname(cmake_path2)))
+
+
+def test_find_external_update_config(mutable_config):
+    pkg_to_entries = {
+        'cmake': [
+            ExternalPackageEntry(Spec('cmake@1.foo'), '/x/y1/'),
+            ExternalPackageEntry(Spec('cmake@3.17.2'), '/x/y2/'),
+        ]
+    }
+
+    spack.cmd.external._update_pkg_config(pkg_to_entries, False)
+
+    pkgs_cfg = spack.config.get('packages')
+    cmake_cfg = pkgs_cfg['cmake']
+    cmake_paths_cfg = cmake_cfg['paths']
+
+    assert cmake_paths_cfg['cmake@1.foo'] == '/x/y1/'
+    assert cmake_paths_cfg['cmake@3.17.2'] == '/x/y2/'
+
+
+def test_get_executables(working_env, create_exe):
+    cmake_path1 = create_exe("cmake", "cmake version 1.foo")
+
+    os.environ['PATH'] = ':'.join([os.path.dirname(cmake_path1)])
+    path_to_exe = spack.cmd.external._get_system_executables()
+    assert path_to_exe[cmake_path1] == 'cmake'
+
+
+external = SpackCommand('external')
+
+
+def test_find_external_cmd(mutable_config, working_env, create_exe):
+    """Test invoking 'spack external find' with additional package arguments,
+    which restricts the set of packages that Spack looks for.
+    """
+    cmake_path1 = create_exe("cmake", "cmake version 1.foo")
+
+    os.environ['PATH'] = ':'.join([os.path.dirname(cmake_path1)])
+    external('find', 'cmake')
+
+    pkgs_cfg = spack.config.get('packages')
+    cmake_cfg = pkgs_cfg['cmake']
+    cmake_paths_cfg = cmake_cfg['paths']
+
+    assert 'cmake@1.foo' in cmake_paths_cfg
+
+
+def test_find_external_cmd_not_buildable(
+        mutable_config, working_env, create_exe):
+    """When the user invokes 'spack external find --not-buildable', the config
+    for any package where Spack finds an external version should be marked as
+    not buildable.
+    """
+    cmake_path1 = create_exe("cmake", "cmake version 1.foo")
+    os.environ['PATH'] = ':'.join([os.path.dirname(cmake_path1)])
+    external('find', '--not-buildable', 'cmake')
+    pkgs_cfg = spack.config.get('packages')
+    assert not pkgs_cfg['cmake']['buildable']
+
+
+def test_find_external_cmd_full_repo(
+        mutable_config, working_env, create_exe, mutable_mock_repo):
+    """Test invoking 'spack external find' with no additional arguments, which
+    iterates through each package in the repository.
+    """
+
+    exe_path1 = create_exe(
+        "find-externals1-exe", "find-externals1 version 1.foo")
+
+    os.environ['PATH'] = ':'.join([os.path.dirname(exe_path1)])
+    external('find')
+
+    pkgs_cfg = spack.config.get('packages')
+    pkg_cfg = pkgs_cfg['find-externals1']
+    pkg_paths_cfg = pkg_cfg['paths']
+
+    assert 'find-externals1@1.foo' in pkg_paths_cfg
+
+
+def test_find_external_merge(mutable_config, mutable_mock_repo):
+    """Check that 'spack find external' doesn't overwrite an existing spec
+    entry in packages.yaml.
+    """
+    pkgs_cfg_init = {
+        'find-externals1': {
+            'paths': {
+                'find-externals1@1.1': '/preexisting-prefix/'
+            },
+            'buildable': False
+        }
+    }
+
+    mutable_config.update_config('packages', pkgs_cfg_init)
+
+    pkg_to_entries = {
+        'find-externals1': [
+            ExternalPackageEntry(Spec('find-externals1@1.1'), '/x/y1/'),
+            ExternalPackageEntry(Spec('find-externals1@1.2'), '/x/y2/'),
+        ]
+    }
+    spack.cmd.external._update_pkg_config(pkg_to_entries, False)
+
+    pkgs_cfg = spack.config.get('packages')
+    pkg_cfg = pkgs_cfg['find-externals1']
+    pkg_paths_cfg = pkg_cfg['paths']
+
+    assert pkg_paths_cfg['find-externals1@1.1'] == '/preexisting-prefix/'
+    assert pkg_paths_cfg['find-externals1@1.2'] == '/x/y2/'