summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/docs/build_settings.rst31
-rw-r--r--lib/spack/spack/cmd/external.py271
-rw-r--r--lib/spack/spack/test/cmd/external.py177
3 files changed, 479 insertions, 0 deletions
diff --git a/lib/spack/docs/build_settings.rst b/lib/spack/docs/build_settings.rst
index cfd850af28..dee892f272 100644
--- a/lib/spack/docs/build_settings.rst
+++ b/lib/spack/docs/build_settings.rst
@@ -158,6 +158,37 @@ Spack can then use any of the listed external implementations of MPI
to satisfy a dependency, and will choose depending on the compiler and
architecture.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Automatically Find External Packages
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A user can run the :ref:`spack external find <spack-external-find>` command
+to search for system-provided packages and add them to ``packages.yaml``.
+After running this command your ``packages.yaml`` may include new entries:
+
+.. code-block:: yaml
+
+ packages:
+ cmake:
+ paths:
+ cmake@3.17.2: /usr
+
+Generally this is useful for detecting a small set of commonly-used packages;
+for now this is generally limited to finding build-only dependencies.
+Specific limitations include:
+
+* A package must define ``executables`` and ``determine_spec_details``
+ for Spack to locate instances of that package.
+* This is currently intended to find build dependencies rather than
+ library packages.
+* Spack does not overwrite existing entries in the package configuration:
+ If there is an external defined for a spec at any configuration scope,
+ then Spack will not add a new external entry (``spack config blame packages``
+ can help locate all external entries).
+* Currently this logic is focused on examining ``PATH`` and does not
+ search through modules (although it should find the package if a
+ module is loaded for it).
+
.. _concretization-preferences:
--------------------------
diff --git a/lib/spack/spack/cmd/external.py b/lib/spack/spack/cmd/external.py
new file mode 100644
index 0000000000..f93deaba03
--- /dev/null
+++ b/lib/spack/spack/cmd/external.py
@@ -0,0 +1,271 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from __future__ import print_function
+from collections import defaultdict, namedtuple
+import argparse
+import os
+import re
+import six
+
+import spack
+import spack.error
+import llnl.util.tty as tty
+import spack.util.spack_yaml as syaml
+import spack.util.environment
+import llnl.util.filesystem
+
+description = "add external packages to Spack configuration"
+section = "config"
+level = "short"
+
+
+def setup_parser(subparser):
+ sp = subparser.add_subparsers(
+ metavar='SUBCOMMAND', dest='external_command')
+
+ find_parser = sp.add_parser('find', help=external_find.__doc__)
+ find_parser.add_argument(
+ '--not-buildable', action='store_true', default=False,
+ help="packages with detected externals won't be built with Spack")
+ find_parser.add_argument('packages', nargs=argparse.REMAINDER)
+
+
+def is_executable(path):
+ return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def _get_system_executables():
+ """Get the paths of all executables available from the current PATH.
+
+ For convenience, this is constructed as a dictionary where the keys are
+ the executable paths and the values are the names of the executables
+ (i.e. the basename of the executable path).
+
+ There may be multiple paths with the same basename. In this case it is
+ assumed there are two different instances of the executable.
+ """
+ path_hints = spack.util.environment.get_path('PATH')
+ search_paths = llnl.util.filesystem.search_paths_for_executables(
+ *path_hints)
+
+ path_to_exe = {}
+ # Reverse order of search directories so that an exe in the first PATH
+ # entry overrides later entries
+ for search_path in reversed(search_paths):
+ for exe in os.listdir(search_path):
+ exe_path = os.path.join(search_path, exe)
+ if is_executable(exe_path):
+ path_to_exe[exe_path] = exe
+ return path_to_exe
+
+
+ExternalPackageEntry = namedtuple(
+ 'ExternalPackageEntry',
+ ['spec', 'base_dir'])
+
+
+def _generate_pkg_config(external_pkg_entries):
+ """Generate config according to the packages.yaml schema for a single
+ package.
+
+ This does not generate the entire packages.yaml. For example, given some
+ external entries for the CMake package, this could return::
+
+ { 'paths': {
+ 'cmake@3.17.1': '/opt/cmake-3.17.1/',
+ 'cmake@3.16.5': '/opt/cmake-3.16.5/'
+ }
+ }
+ """
+ paths_dict = syaml.syaml_dict()
+ for e in external_pkg_entries:
+ if not _spec_is_valid(e.spec):
+ continue
+ paths_dict[str(e.spec)] = e.base_dir
+ pkg_dict = syaml.syaml_dict()
+ pkg_dict['paths'] = paths_dict
+
+ return pkg_dict
+
+
+def _spec_is_valid(spec):
+ try:
+ str(spec)
+ except spack.error.SpackError:
+ # It is assumed here that we can at least extract the package name from
+ # the spec so we can look up the implementation of
+ # determine_spec_details
+ tty.warn('Constructed spec for {0} does not have a string'
+ ' representation'.format(spec.name))
+ return False
+
+ try:
+ spack.spec.Spec(str(spec))
+ except spack.error.SpackError:
+ tty.warn('Constructed spec has a string representation but the string'
+ ' representation does not evaluate to a valid spec: {0}'
+ .format(str(spec)))
+ return False
+
+ return True
+
+
+def external_find(args):
+ if args.packages:
+ packages_to_check = list(spack.repo.get(pkg) for pkg in args.packages)
+ else:
+ packages_to_check = spack.repo.path.all_packages()
+
+ pkg_to_entries = _get_external_packages(packages_to_check)
+ _update_pkg_config(pkg_to_entries, args.not_buildable)
+
+
+def _group_by_prefix(paths):
+ groups = defaultdict(set)
+ for p in paths:
+ groups[os.path.dirname(p)].add(p)
+ return groups.items()
+
+
+def _convert_to_iterable(single_val_or_multiple):
+ x = single_val_or_multiple
+ if x is None:
+ return []
+ elif isinstance(x, six.string_types):
+ return [x]
+ elif isinstance(x, spack.spec.Spec):
+ # Specs are iterable, but a single spec should be converted to a list
+ return [x]
+
+ try:
+ iter(x)
+ return x
+ except TypeError:
+ return [x]
+
+
+def _determine_base_dir(prefix):
+ # Given a prefix where an executable is found, assuming that prefix ends
+ # with /bin/, strip off the 'bin' directory to get a Spack-compatible
+ # prefix
+ assert os.path.isdir(prefix)
+ if os.path.basename(prefix) == 'bin':
+ return os.path.dirname(prefix)
+
+
+def _get_predefined_externals():
+ # Pull from all scopes when looking for preexisting external package
+ # entries
+ pkg_config = spack.config.get('packages')
+ already_defined_specs = set()
+ for pkg_name, per_pkg_cfg in pkg_config.items():
+ paths = per_pkg_cfg.get('paths', {})
+ already_defined_specs.update(spack.spec.Spec(k) for k in paths)
+ modules = per_pkg_cfg.get('modules', {})
+ already_defined_specs.update(spack.spec.Spec(k) for k in modules)
+ return already_defined_specs
+
+
+def _update_pkg_config(pkg_to_entries, not_buildable):
+ predefined_external_specs = _get_predefined_externals()
+
+ pkg_to_cfg = {}
+ for pkg_name, ext_pkg_entries in pkg_to_entries.items():
+ new_entries = list(
+ e for e in ext_pkg_entries
+ if (e.spec not in predefined_external_specs))
+
+ pkg_config = _generate_pkg_config(new_entries)
+ if not_buildable:
+ pkg_config['buildable'] = False
+ pkg_to_cfg[pkg_name] = pkg_config
+
+ cfg_scope = spack.config.default_modify_scope()
+ pkgs_cfg = spack.config.get('packages', scope=cfg_scope)
+
+ spack.config._merge_yaml(pkgs_cfg, pkg_to_cfg)
+ spack.config.set('packages', pkgs_cfg, scope=cfg_scope)
+
+
+def _get_external_packages(packages_to_check, system_path_to_exe=None):
+ if not system_path_to_exe:
+ system_path_to_exe = _get_system_executables()
+
+ exe_pattern_to_pkgs = defaultdict(list)
+ for pkg in packages_to_check:
+ if hasattr(pkg, 'executables'):
+ for exe in pkg.executables:
+ exe_pattern_to_pkgs[exe].append(pkg)
+
+ pkg_to_found_exes = defaultdict(set)
+ for exe_pattern, pkgs in exe_pattern_to_pkgs.items():
+ compiled_re = re.compile(exe_pattern)
+ for path, exe in system_path_to_exe.items():
+ if compiled_re.search(exe):
+ for pkg in pkgs:
+ pkg_to_found_exes[pkg].add(path)
+
+ pkg_to_entries = defaultdict(list)
+ resolved_specs = {} # spec -> exe found for the spec
+
+ for pkg, exes in pkg_to_found_exes.items():
+ if not hasattr(pkg, 'determine_spec_details'):
+ tty.warn("{0} must define 'determine_spec_details' in order"
+ " for Spack to detect externally-provided instances"
+ " of the package.".format(pkg.name))
+ continue
+
+ # TODO: iterate through this in a predetermined order (e.g. by package
+ # name) to get repeatable results when there are conflicts. Note that
+ # if we take the prefixes returned by _group_by_prefix, then consider
+ # them in the order that they appear in PATH, this should be sufficient
+ # to get repeatable results.
+ for prefix, exes_in_prefix in _group_by_prefix(exes):
+ # TODO: multiple instances of a package can live in the same
+ # prefix, and a package implementation can return multiple specs
+ # for one prefix, but without additional details (e.g. about the
+ # naming scheme which differentiates them), the spec won't be
+ # usable.
+ specs = _convert_to_iterable(
+ pkg.determine_spec_details(prefix, exes_in_prefix))
+
+ if not specs:
+ tty.debug(
+ 'The following executables in {0} were decidedly not'
+ 'part of the package {1}: {2}'
+ .format(prefix, pkg.name, ', '.join(exes_in_prefix))
+ )
+
+ for spec in specs:
+ pkg_prefix = _determine_base_dir(prefix)
+
+ if not pkg_prefix:
+ tty.debug("{0} does not end with a 'bin/' directory: it"
+ " cannot be added as a Spack package"
+ .format(prefix))
+ continue
+
+ if spec in resolved_specs:
+ prior_prefix = ', '.join(resolved_specs[spec])
+
+ tty.debug(
+ "Executables in {0} and {1} are both associated"
+ " with the same spec {2}"
+ .format(prefix, prior_prefix, str(spec)))
+ continue
+ else:
+ resolved_specs[spec] = prefix
+
+ pkg_to_entries[pkg.name].append(
+ ExternalPackageEntry(spec=spec, base_dir=pkg_prefix))
+
+ return pkg_to_entries
+
+
+def external(parser, args):
+ action = {'find': external_find}
+
+ action[args.external_command](args)
diff --git a/lib/spack/spack/test/cmd/external.py b/lib/spack/spack/test/cmd/external.py
new file mode 100644
index 0000000000..0bdf67fe3e
--- /dev/null
+++ b/lib/spack/spack/test/cmd/external.py
@@ -0,0 +1,177 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import pytest
+import os
+import stat
+
+import spack
+from spack.spec import Spec
+from spack.cmd.external import ExternalPackageEntry
+from spack.main import SpackCommand
+
+
+@pytest.fixture()
+def create_exe(tmpdir_factory):
+ def _create_exe(exe_name, content):
+ base_prefix = tmpdir_factory.mktemp('base-prefix')
+ base_prefix.ensure('bin', dir=True)
+ exe_path = str(base_prefix.join('bin', exe_name))
+ with open(exe_path, 'w') as f:
+ f.write("""\
+#!/bin/bash
+
+echo "{0}"
+""".format(content))
+
+ st = os.stat(exe_path)
+ os.chmod(exe_path, st.st_mode | stat.S_IEXEC)
+ return exe_path
+
+ yield _create_exe
+
+
+def test_find_external_single_package(create_exe):
+ pkgs_to_check = [spack.repo.get('cmake')]
+
+ cmake_path = create_exe("cmake", "cmake version 1.foo")
+ system_path_to_exe = {cmake_path: 'cmake'}
+
+ pkg_to_entries = spack.cmd.external._get_external_packages(
+ pkgs_to_check, system_path_to_exe)
+
+ pkg, entries = next(iter(pkg_to_entries.items()))
+ single_entry = next(iter(entries))
+
+ assert single_entry.spec == Spec('cmake@1.foo')
+
+
+def test_find_external_two_instances_same_package(create_exe):
+ pkgs_to_check = [spack.repo.get('cmake')]
+
+ # Each of these cmake instances is created in a different prefix
+ cmake_path1 = create_exe("cmake", "cmake version 1.foo")
+ cmake_path2 = create_exe("cmake", "cmake version 3.17.2")
+ system_path_to_exe = {
+ cmake_path1: 'cmake',
+ cmake_path2: 'cmake'}
+
+ pkg_to_entries = spack.cmd.external._get_external_packages(
+ pkgs_to_check, system_path_to_exe)
+
+ pkg, entries = next(iter(pkg_to_entries.items()))
+ spec_to_path = dict((e.spec, e.base_dir) for e in entries)
+ assert spec_to_path[Spec('cmake@1.foo')] == (
+ spack.cmd.external._determine_base_dir(os.path.dirname(cmake_path1)))
+ assert spec_to_path[Spec('cmake@3.17.2')] == (
+ spack.cmd.external._determine_base_dir(os.path.dirname(cmake_path2)))
+
+
+def test_find_external_update_config(mutable_config):
+ pkg_to_entries = {
+ 'cmake': [
+ ExternalPackageEntry(Spec('cmake@1.foo'), '/x/y1/'),
+ ExternalPackageEntry(Spec('cmake@3.17.2'), '/x/y2/'),
+ ]
+ }
+
+ spack.cmd.external._update_pkg_config(pkg_to_entries, False)
+
+ pkgs_cfg = spack.config.get('packages')
+ cmake_cfg = pkgs_cfg['cmake']
+ cmake_paths_cfg = cmake_cfg['paths']
+
+ assert cmake_paths_cfg['cmake@1.foo'] == '/x/y1/'
+ assert cmake_paths_cfg['cmake@3.17.2'] == '/x/y2/'
+
+
+def test_get_executables(working_env, create_exe):
+ cmake_path1 = create_exe("cmake", "cmake version 1.foo")
+
+ os.environ['PATH'] = ':'.join([os.path.dirname(cmake_path1)])
+ path_to_exe = spack.cmd.external._get_system_executables()
+ assert path_to_exe[cmake_path1] == 'cmake'
+
+
+external = SpackCommand('external')
+
+
+def test_find_external_cmd(mutable_config, working_env, create_exe):
+ """Test invoking 'spack external find' with additional package arguments,
+ which restricts the set of packages that Spack looks for.
+ """
+ cmake_path1 = create_exe("cmake", "cmake version 1.foo")
+
+ os.environ['PATH'] = ':'.join([os.path.dirname(cmake_path1)])
+ external('find', 'cmake')
+
+ pkgs_cfg = spack.config.get('packages')
+ cmake_cfg = pkgs_cfg['cmake']
+ cmake_paths_cfg = cmake_cfg['paths']
+
+ assert 'cmake@1.foo' in cmake_paths_cfg
+
+
+def test_find_external_cmd_not_buildable(
+ mutable_config, working_env, create_exe):
+ """When the user invokes 'spack external find --not-buildable', the config
+ for any package where Spack finds an external version should be marked as
+ not buildable.
+ """
+ cmake_path1 = create_exe("cmake", "cmake version 1.foo")
+ os.environ['PATH'] = ':'.join([os.path.dirname(cmake_path1)])
+ external('find', '--not-buildable', 'cmake')
+ pkgs_cfg = spack.config.get('packages')
+ assert not pkgs_cfg['cmake']['buildable']
+
+
+def test_find_external_cmd_full_repo(
+ mutable_config, working_env, create_exe, mutable_mock_repo):
+ """Test invoking 'spack external find' with no additional arguments, which
+ iterates through each package in the repository.
+ """
+
+ exe_path1 = create_exe(
+ "find-externals1-exe", "find-externals1 version 1.foo")
+
+ os.environ['PATH'] = ':'.join([os.path.dirname(exe_path1)])
+ external('find')
+
+ pkgs_cfg = spack.config.get('packages')
+ pkg_cfg = pkgs_cfg['find-externals1']
+ pkg_paths_cfg = pkg_cfg['paths']
+
+ assert 'find-externals1@1.foo' in pkg_paths_cfg
+
+
+def test_find_external_merge(mutable_config, mutable_mock_repo):
+ """Check that 'spack find external' doesn't overwrite an existing spec
+ entry in packages.yaml.
+ """
+ pkgs_cfg_init = {
+ 'find-externals1': {
+ 'paths': {
+ 'find-externals1@1.1': '/preexisting-prefix/'
+ },
+ 'buildable': False
+ }
+ }
+
+ mutable_config.update_config('packages', pkgs_cfg_init)
+
+ pkg_to_entries = {
+ 'find-externals1': [
+ ExternalPackageEntry(Spec('find-externals1@1.1'), '/x/y1/'),
+ ExternalPackageEntry(Spec('find-externals1@1.2'), '/x/y2/'),
+ ]
+ }
+ spack.cmd.external._update_pkg_config(pkg_to_entries, False)
+
+ pkgs_cfg = spack.config.get('packages')
+ pkg_cfg = pkgs_cfg['find-externals1']
+ pkg_paths_cfg = pkg_cfg['paths']
+
+ assert pkg_paths_cfg['find-externals1@1.1'] == '/preexisting-prefix/'
+ assert pkg_paths_cfg['find-externals1@1.2'] == '/x/y2/'
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-02-18 05:19:10 +0000