diff options
| -rw-r--r-- | lib/spack/docs/build_settings.rst | 31 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/external.py | 271 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/external.py | 177 | ||||
| -rwxr-xr-x | share/spack/spack-completion.bash | 20 | ||||
| -rw-r--r-- | var/spack/repos/builtin.mock/packages/find-externals1/package.py | 34 | ||||
| -rw-r--r-- | var/spack/repos/builtin/packages/automake/package.py | 21 | ||||
| -rw-r--r-- | var/spack/repos/builtin/packages/cmake/package.py | 21 |
7 files changed, 574 insertions, 1 deletions
diff --git a/lib/spack/docs/build_settings.rst b/lib/spack/docs/build_settings.rst index cfd850af28..dee892f272 100644 --- a/lib/spack/docs/build_settings.rst +++ b/lib/spack/docs/build_settings.rst @@ -158,6 +158,37 @@ Spack can then use any of the listed external implementations of MPI to satisfy a dependency, and will choose depending on the compiler and architecture. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Automatically Find External Packages +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A user can run the :ref:`spack external find <spack-external-find>` command +to search for system-provided packages and add them to ``packages.yaml``. +After running this command your ``packages.yaml`` may include new entries: + +.. code-block:: yaml + + packages: + cmake: + paths: + cmake@3.17.2: /usr + +Generally this is useful for detecting a small set of commonly-used packages; +for now this is generally limited to finding build-only dependencies. +Specific limitations include: + +* A package must define ``executables`` and ``determine_spec_details`` + for Spack to locate instances of that package. +* This is currently intended to find build dependencies rather than + library packages. +* Spack does not overwrite existing entries in the package configuration: + If there is an external defined for a spec at any configuration scope, + then Spack will not add a new external entry (``spack config blame packages`` + can help locate all external entries). +* Currently this logic is focused on examining ``PATH`` and does not + search through modules (although it should find the package if a + module is loaded for it). + .. _concretization-preferences: -------------------------- diff --git a/lib/spack/spack/cmd/external.py b/lib/spack/spack/cmd/external.py new file mode 100644 index 0000000000..f93deaba03 --- /dev/null +++ b/lib/spack/spack/cmd/external.py @@ -0,0 +1,271 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from __future__ import print_function +from collections import defaultdict, namedtuple +import argparse +import os +import re +import six + +import spack +import spack.error +import llnl.util.tty as tty +import spack.util.spack_yaml as syaml +import spack.util.environment +import llnl.util.filesystem + +description = "add external packages to Spack configuration" +section = "config" +level = "short" + + +def setup_parser(subparser): + sp = subparser.add_subparsers( + metavar='SUBCOMMAND', dest='external_command') + + find_parser = sp.add_parser('find', help=external_find.__doc__) + find_parser.add_argument( + '--not-buildable', action='store_true', default=False, + help="packages with detected externals won't be built with Spack") + find_parser.add_argument('packages', nargs=argparse.REMAINDER) + + +def is_executable(path): + return os.path.isfile(path) and os.access(path, os.X_OK) + + +def _get_system_executables(): + """Get the paths of all executables available from the current PATH. + + For convenience, this is constructed as a dictionary where the keys are + the executable paths and the values are the names of the executables + (i.e. the basename of the executable path). + + There may be multiple paths with the same basename. In this case it is + assumed there are two different instances of the executable. + """ + path_hints = spack.util.environment.get_path('PATH') + search_paths = llnl.util.filesystem.search_paths_for_executables( + *path_hints) + + path_to_exe = {} + # Reverse order of search directories so that an exe in the first PATH + # entry overrides later entries + for search_path in reversed(search_paths): + for exe in os.listdir(search_path): + exe_path = os.path.join(search_path, exe) + if is_executable(exe_path): + path_to_exe[exe_path] = exe + return path_to_exe + + +ExternalPackageEntry = namedtuple( + 'ExternalPackageEntry', + ['spec', 'base_dir']) + + +def _generate_pkg_config(external_pkg_entries): + """Generate config according to the packages.yaml schema for a single + package. + + This does not generate the entire packages.yaml. For example, given some + external entries for the CMake package, this could return:: + + { 'paths': { + 'cmake@3.17.1': '/opt/cmake-3.17.1/', + 'cmake@3.16.5': '/opt/cmake-3.16.5/' + } + } + """ + paths_dict = syaml.syaml_dict() + for e in external_pkg_entries: + if not _spec_is_valid(e.spec): + continue + paths_dict[str(e.spec)] = e.base_dir + pkg_dict = syaml.syaml_dict() + pkg_dict['paths'] = paths_dict + + return pkg_dict + + +def _spec_is_valid(spec): + try: + str(spec) + except spack.error.SpackError: + # It is assumed here that we can at least extract the package name from + # the spec so we can look up the implementation of + # determine_spec_details + tty.warn('Constructed spec for {0} does not have a string' + ' representation'.format(spec.name)) + return False + + try: + spack.spec.Spec(str(spec)) + except spack.error.SpackError: + tty.warn('Constructed spec has a string representation but the string' + ' representation does not evaluate to a valid spec: {0}' + .format(str(spec))) + return False + + return True + + +def external_find(args): + if args.packages: + packages_to_check = list(spack.repo.get(pkg) for pkg in args.packages) + else: + packages_to_check = spack.repo.path.all_packages() + + pkg_to_entries = _get_external_packages(packages_to_check) + _update_pkg_config(pkg_to_entries, args.not_buildable) + + +def _group_by_prefix(paths): + groups = defaultdict(set) + for p in paths: + groups[os.path.dirname(p)].add(p) + return groups.items() + + +def _convert_to_iterable(single_val_or_multiple): + x = single_val_or_multiple + if x is None: + return [] + elif isinstance(x, six.string_types): + return [x] + elif isinstance(x, spack.spec.Spec): + # Specs are iterable, but a single spec should be converted to a list + return [x] + + try: + iter(x) + return x + except TypeError: + return [x] + + +def _determine_base_dir(prefix): + # Given a prefix where an executable is found, assuming that prefix ends + # with /bin/, strip off the 'bin' directory to get a Spack-compatible + # prefix + assert os.path.isdir(prefix) + if os.path.basename(prefix) == 'bin': + return os.path.dirname(prefix) + + +def _get_predefined_externals(): + # Pull from all scopes when looking for preexisting external package + # entries + pkg_config = spack.config.get('packages') + already_defined_specs = set() + for pkg_name, per_pkg_cfg in pkg_config.items(): + paths = per_pkg_cfg.get('paths', {}) + already_defined_specs.update(spack.spec.Spec(k) for k in paths) + modules = per_pkg_cfg.get('modules', {}) + already_defined_specs.update(spack.spec.Spec(k) for k in modules) + return already_defined_specs + + +def _update_pkg_config(pkg_to_entries, not_buildable): + predefined_external_specs = _get_predefined_externals() + + pkg_to_cfg = {} + for pkg_name, ext_pkg_entries in pkg_to_entries.items(): + new_entries = list( + e for e in ext_pkg_entries + if (e.spec not in predefined_external_specs)) + + pkg_config = _generate_pkg_config(new_entries) + if not_buildable: + pkg_config['buildable'] = False + pkg_to_cfg[pkg_name] = pkg_config + + cfg_scope = spack.config.default_modify_scope() + pkgs_cfg = spack.config.get('packages', scope=cfg_scope) + + spack.config._merge_yaml(pkgs_cfg, pkg_to_cfg) + spack.config.set('packages', pkgs_cfg, scope=cfg_scope) + + +def _get_external_packages(packages_to_check, system_path_to_exe=None): + if not system_path_to_exe: + system_path_to_exe = _get_system_executables() + + exe_pattern_to_pkgs = defaultdict(list) + for pkg in packages_to_check: + if hasattr(pkg, 'executables'): + for exe in pkg.executables: + exe_pattern_to_pkgs[exe].append(pkg) + + pkg_to_found_exes = defaultdict(set) + for exe_pattern, pkgs in exe_pattern_to_pkgs.items(): + compiled_re = re.compile(exe_pattern) + for path, exe in system_path_to_exe.items(): + if compiled_re.search(exe): + for pkg in pkgs: + pkg_to_found_exes[pkg].add(path) + + pkg_to_entries = defaultdict(list) + resolved_specs = {} # spec -> exe found for the spec + + for pkg, exes in pkg_to_found_exes.items(): + if not hasattr(pkg, 'determine_spec_details'): + tty.warn("{0} must define 'determine_spec_details' in order" + " for Spack to detect externally-provided instances" + " of the package.".format(pkg.name)) + continue + + # TODO: iterate through this in a predetermined order (e.g. by package + # name) to get repeatable results when there are conflicts. Note that + # if we take the prefixes returned by _group_by_prefix, then consider + # them in the order that they appear in PATH, this should be sufficient + # to get repeatable results. + for prefix, exes_in_prefix in _group_by_prefix(exes): + # TODO: multiple instances of a package can live in the same + # prefix, and a package implementation can return multiple specs + # for one prefix, but without additional details (e.g. about the + # naming scheme which differentiates them), the spec won't be + # usable. + specs = _convert_to_iterable( + pkg.determine_spec_details(prefix, exes_in_prefix)) + + if not specs: + tty.debug( + 'The following executables in {0} were decidedly not' + 'part of the package {1}: {2}' + .format(prefix, pkg.name, ', '.join(exes_in_prefix)) + ) + + for spec in specs: + pkg_prefix = _determine_base_dir(prefix) + + if not pkg_prefix: + tty.debug("{0} does not end with a 'bin/' directory: it" + " cannot be added as a Spack package" + .format(prefix)) + continue + + if spec in resolved_specs: + prior_prefix = ', '.join(resolved_specs[spec]) + + tty.debug( + "Executables in {0} and {1} are both associated" + " with the same spec {2}" + .format(prefix, prior_prefix, str(spec))) + continue + else: + resolved_specs[spec] = prefix + + pkg_to_entries[pkg.name].append( + ExternalPackageEntry(spec=spec, base_dir=pkg_prefix)) + + return pkg_to_entries + + +def external(parser, args): + action = {'find': external_find} + + action[args.external_command](args) diff --git a/lib/spack/spack/test/cmd/external.py b/lib/spack/spack/test/cmd/external.py new file mode 100644 index 0000000000..0bdf67fe3e --- /dev/null +++ b/lib/spack/spack/test/cmd/external.py @@ -0,0 +1,177 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import pytest +import os +import stat + +import spack +from spack.spec import Spec +from spack.cmd.external import ExternalPackageEntry +from spack.main import SpackCommand + + +@pytest.fixture() +def create_exe(tmpdir_factory): + def _create_exe(exe_name, content): + base_prefix = tmpdir_factory.mktemp('base-prefix') + base_prefix.ensure('bin', dir=True) + exe_path = str(base_prefix.join('bin', exe_name)) + with open(exe_path, 'w') as f: + f.write("""\ +#!/bin/bash + +echo "{0}" +""".format(content)) + + st = os.stat(exe_path) + os.chmod(exe_path, st.st_mode | stat.S_IEXEC) + return exe_path + + yield _create_exe + + +def test_find_external_single_package(create_exe): + pkgs_to_check = [spack.repo.get('cmake')] + + cmake_path = create_exe("cmake", "cmake version 1.foo") + system_path_to_exe = {cmake_path: 'cmake'} + + pkg_to_entries = spack.cmd.external._get_external_packages( + pkgs_to_check, system_path_to_exe) + + pkg, entries = next(iter(pkg_to_entries.items())) + single_entry = next(iter(entries)) + + assert single_entry.spec == Spec('cmake@1.foo') + + +def test_find_external_two_instances_same_package(create_exe): + pkgs_to_check = [spack.repo.get('cmake')] + + # Each of these cmake instances is created in a different prefix + cmake_path1 = create_exe("cmake", "cmake version 1.foo") + cmake_path2 = create_exe("cmake", "cmake version 3.17.2") + system_path_to_exe = { + cmake_path1: 'cmake', + cmake_path2: 'cmake'} + + pkg_to_entries = spack.cmd.external._get_external_packages( + pkgs_to_check, system_path_to_exe) + + pkg, entries = next(iter(pkg_to_entries.items())) + spec_to_path = dict((e.spec, e.base_dir) for e in entries) + assert spec_to_path[Spec('cmake@1.foo')] == ( + spack.cmd.external._determine_base_dir(os.path.dirname(cmake_path1))) + assert spec_to_path[Spec('cmake@3.17.2')] == ( + spack.cmd.external._determine_base_dir(os.path.dirname(cmake_path2))) + + +def test_find_external_update_config(mutable_config): + pkg_to_entries = { + 'cmake': [ + ExternalPackageEntry(Spec('cmake@1.foo'), '/x/y1/'), + ExternalPackageEntry(Spec('cmake@3.17.2'), '/x/y2/'), + ] + } + + spack.cmd.external._update_pkg_config(pkg_to_entries, False) + + pkgs_cfg = spack.config.get('packages') + cmake_cfg = pkgs_cfg['cmake'] + cmake_paths_cfg = cmake_cfg['paths'] + + assert cmake_paths_cfg['cmake@1.foo'] == '/x/y1/' + assert cmake_paths_cfg['cmake@3.17.2'] == '/x/y2/' + + +def test_get_executables(working_env, create_exe): + cmake_path1 = create_exe("cmake", "cmake version 1.foo") + + os.environ['PATH'] = ':'.join([os.path.dirname(cmake_path1)]) + path_to_exe = spack.cmd.external._get_system_executables() + assert path_to_exe[cmake_path1] == 'cmake' + + +external = SpackCommand('external') + + +def test_find_external_cmd(mutable_config, working_env, create_exe): + """Test invoking 'spack external find' with additional package arguments, + which restricts the set of packages that Spack looks for. + """ + cmake_path1 = create_exe("cmake", "cmake version 1.foo") + + os.environ['PATH'] = ':'.join([os.path.dirname(cmake_path1)]) + external('find', 'cmake') + + pkgs_cfg = spack.config.get('packages') + cmake_cfg = pkgs_cfg['cmake'] + cmake_paths_cfg = cmake_cfg['paths'] + + assert 'cmake@1.foo' in cmake_paths_cfg + + +def test_find_external_cmd_not_buildable( + mutable_config, working_env, create_exe): + """When the user invokes 'spack external find --not-buildable', the config + for any package where Spack finds an external version should be marked as + not buildable. + """ + cmake_path1 = create_exe("cmake", "cmake version 1.foo") + os.environ['PATH'] = ':'.join([os.path.dirname(cmake_path1)]) + external('find', '--not-buildable', 'cmake') + pkgs_cfg = spack.config.get('packages') + assert not pkgs_cfg['cmake']['buildable'] + + +def test_find_external_cmd_full_repo( + mutable_config, working_env, create_exe, mutable_mock_repo): + """Test invoking 'spack external find' with no additional arguments, which + iterates through each package in the repository. + """ + + exe_path1 = create_exe( + "find-externals1-exe", "find-externals1 version 1.foo") + + os.environ['PATH'] = ':'.join([os.path.dirname(exe_path1)]) + external('find') + + pkgs_cfg = spack.config.get('packages') + pkg_cfg = pkgs_cfg['find-externals1'] + pkg_paths_cfg = pkg_cfg['paths'] + + assert 'find-externals1@1.foo' in pkg_paths_cfg + + +def test_find_external_merge(mutable_config, mutable_mock_repo): + """Check that 'spack find external' doesn't overwrite an existing spec + entry in packages.yaml. + """ + pkgs_cfg_init = { + 'find-externals1': { + 'paths': { + 'find-externals1@1.1': '/preexisting-prefix/' + }, + 'buildable': False + } + } + + mutable_config.update_config('packages', pkgs_cfg_init) + + pkg_to_entries = { + 'find-externals1': [ + ExternalPackageEntry(Spec('find-externals1@1.1'), '/x/y1/'), + ExternalPackageEntry(Spec('find-externals1@1.2'), '/x/y2/'), + ] + } + spack.cmd.external._update_pkg_config(pkg_to_entries, False) + + pkgs_cfg = spack.config.get('packages') + pkg_cfg = pkgs_cfg['find-externals1'] + pkg_paths_cfg = pkg_cfg['paths'] + + assert pkg_paths_cfg['find-externals1@1.1'] == '/preexisting-prefix/' + assert pkg_paths_cfg['find-externals1@1.2'] == '/x/y2/' diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index fce5360ff5..12d4c599b8 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -313,7 +313,7 @@ _spack() { then SPACK_COMPREPLY="-h --help -H --all-help --color -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars" else - SPACK_COMPREPLY="activate add arch blame bootstrap build build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config configure containerize create deactivate debug dependencies dependents deprecate dev-build diy docs edit env extensions fetch find flake8 gc gpg graph help info install license list load location log-parse maintainers mirror module patch pkg providers pydoc python reindex remove rm repo resource restage setup spec stage test uninstall unload upload-s3 url verify versions view" + SPACK_COMPREPLY="activate add arch blame bootstrap build build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config configure containerize create deactivate debug dependencies dependents deprecate dev-build diy docs edit env extensions external fetch find flake8 gc gpg graph help info install license list load location log-parse maintainers mirror module patch pkg providers pydoc python reindex remove rm repo resource restage setup spec stage test uninstall unload upload-s3 url verify versions view" fi } @@ -817,6 +817,24 @@ _spack_extensions() { fi } +_spack_external() { + if $list_options + then + SPACK_COMPREPLY="-h --help" + else + SPACK_COMPREPLY="find" + fi +} + +_spack_external_find() { + if $list_options + then + SPACK_COMPREPLY="-h --help --not-buildable" + else + _all_packages + fi +} + _spack_fetch() { if $list_options then diff --git a/var/spack/repos/builtin.mock/packages/find-externals1/package.py b/var/spack/repos/builtin.mock/packages/find-externals1/package.py new file mode 100644 index 0000000000..25e26dcced --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/find-externals1/package.py @@ -0,0 +1,34 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + +import os +import re + + +class FindExternals1(AutotoolsPackage): + executables = ['find-externals1-exe'] + + url = "http://www.example.com/find-externals-1.0.tar.gz" + + version('1.0', 'hash-1.0') + + @classmethod + def determine_spec_details(cls, prefix, exes_in_prefix): + exe_to_path = dict( + (os.path.basename(p), p) for p in exes_in_prefix + ) + if 'find-externals1-exe' not in exe_to_path: + return None + + exe = spack.util.executable.Executable( + exe_to_path['find-externals1-exe']) + output = exe('--version', output=str) + if output: + match = re.search(r'find-externals1.*version\s+(\S+)', output) + if match: + version_str = match.group(1) + return Spec('find-externals1@{0}'.format(version_str)) diff --git a/var/spack/repos/builtin/packages/automake/package.py b/var/spack/repos/builtin/packages/automake/package.py index 5327c90daf..0e9d22cb37 100644 --- a/var/spack/repos/builtin/packages/automake/package.py +++ b/var/spack/repos/builtin/packages/automake/package.py @@ -5,6 +5,9 @@ from spack import * +import os +import re + class Automake(AutotoolsPackage, GNUMirrorPackage): """Automake -- make file builder part of autotools""" @@ -25,6 +28,24 @@ class Automake(AutotoolsPackage, GNUMirrorPackage): build_directory = 'spack-build' + executables = ['automake'] + + @classmethod + def determine_spec_details(cls, prefix, exes_in_prefix): + exe_to_path = dict( + (os.path.basename(p), p) for p in exes_in_prefix + ) + if 'automake' not in exe_to_path: + return None + + exe = spack.util.executable.Executable(exe_to_path['automake']) + output = exe('--version', output=str) + if output: + match = re.search(r'GNU automake\)\s+(\S+)', output) + if match: + version_str = match.group(1) + return Spec('automake@{0}'.format(version_str)) + def patch(self): # The full perl shebang might be too long files_to_be_patched_fmt = 'bin/{0}.in' diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index acf5b1fcdc..cfc13c436d 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -5,6 +5,9 @@ from spack import * +import re +import os + class Cmake(Package): """A cross-platform, open-source build system. CMake is a family of @@ -13,6 +16,8 @@ class Cmake(Package): url = 'https://github.com/Kitware/CMake/releases/download/v3.15.5/cmake-3.15.5.tar.gz' maintainers = ['chuckatkins'] + executables = ['cmake'] + version('3.17.1', sha256='3aa9114485da39cbd9665a0bfe986894a282d5f0882b1dea960a739496620727') version('3.17.0', sha256='b74c05b55115eacc4fa2b77a814981dbda05cdc95a53e279fe16b7b272f00847') version('3.16.5', sha256='5f760b50b8ecc9c0c37135fae5fbf00a2fef617059aa9d61c1bb91653e5a8bfc') @@ -146,6 +151,22 @@ class Cmake(Package): phases = ['bootstrap', 'build', 'install'] + @classmethod + def determine_spec_details(cls, prefix, exes_in_prefix): + exe_to_path = dict( + (os.path.basename(p), p) for p in exes_in_prefix + ) + if 'cmake' not in exe_to_path: + return None + + cmake = spack.util.executable.Executable(exe_to_path['cmake']) + output = cmake('--version', output=str) + if output: + match = re.search(r'cmake.*version\s+(\S+)', output) + if match: + version_str = match.group(1) + return Spec('cmake@{0}'.format(version_str)) + def flag_handler(self, name, flags): if name == 'cxxflags' and self.compiler.name == 'fj': cxx11plus_flags = (self.compiler.cxx11_flag, |