From 694d633a2c46505b5bcb2e57a52966e37c36477e Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Thu, 4 Feb 2021 13:17:32 +0100 Subject: spack external find: allow to search by tags (#21407) This commit adds an option to the `external find` command that allows it to search by tags. In this way group of executables with common purposes can be grouped under a single name and a simple command can be used to detect all of them. As an example introduce the 'build-tools' tag to search for common development tools on a system --- lib/spack/spack/cmd/common/arguments.py | 4 ++-- lib/spack/spack/cmd/external.py | 23 +++++++++++++++++++- lib/spack/spack/test/cmd/external.py | 25 ++++++++++++++++++++++ lib/spack/spack/test/cmd/find.py | 6 +++--- lib/spack/spack/test/cmd/list.py | 6 +++--- share/spack/spack-completion.bash | 6 +++--- .../repos/builtin/packages/autoconf/package.py | 2 ++ .../repos/builtin/packages/automake/package.py | 2 ++ var/spack/repos/builtin/packages/cmake/package.py | 2 ++ var/spack/repos/builtin/packages/flex/package.py | 2 ++ var/spack/repos/builtin/packages/gmake/package.py | 2 ++ .../repos/builtin/packages/libtool/package.py | 2 ++ var/spack/repos/builtin/packages/m4/package.py | 2 ++ .../repos/builtin/packages/pkg-config/package.py | 2 ++ .../repos/builtin/packages/pkgconf/package.py | 2 ++ 15 files changed, 76 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/cmd/common/arguments.py b/lib/spack/spack/cmd/common/arguments.py index c09f890114..f83708a7dd 100644 --- a/lib/spack/spack/cmd/common/arguments.py +++ b/lib/spack/spack/cmd/common/arguments.py @@ -250,8 +250,8 @@ def very_long(): @arg def tags(): return Args( - '-t', '--tags', action='append', - help='filter a package query by tags') + '-t', '--tag', action='append', dest='tags', metavar='TAG', + help='filter a package query by tag (multiple use allowed)') @arg diff --git a/lib/spack/spack/cmd/external.py b/lib/spack/spack/cmd/external.py index c4f3c0e854..c7d045e66d 100644 --- a/lib/spack/spack/cmd/external.py +++ b/lib/spack/spack/cmd/external.py @@ -16,6 +16,7 @@ import llnl.util.tty.colify as colify import six import spack import spack.cmd +import spack.cmd.common.arguments import spack.error import spack.util.environment import spack.util.spack_yaml as syaml @@ -42,6 +43,7 @@ def setup_parser(subparser): '--scope', choices=scopes, metavar=scopes_metavar, default=spack.config.default_modify_scope('packages'), help="configuration scope to modify") + spack.cmd.common.arguments.add_common_arguments(find_parser, ['tags']) find_parser.add_argument('packages', nargs=argparse.REMAINDER) sp.add_parser( @@ -148,9 +150,28 @@ def _spec_is_valid(spec): def external_find(args): + # Construct the list of possible packages to be detected + packages_to_check = [] + + # Add the packages that have been required explicitly if args.packages: packages_to_check = list(spack.repo.get(pkg) for pkg in args.packages) - else: + if args.tags: + allowed = set(spack.repo.path.packages_with_tags(*args.tags)) + packages_to_check = [x for x in packages_to_check if x in allowed] + + if args.tags and not packages_to_check: + # If we arrived here we didn't have any explicit package passed + # as argument, which means to search all packages. + # Since tags are cached it's much faster to construct what we need + # to search directly, rather than filtering after the fact + packages_to_check = [ + spack.repo.get(pkg) for pkg in + spack.repo.path.packages_with_tags(*args.tags) + ] + + # If the list of packages is empty, search for every possible package + if not args.tags and not packages_to_check: packages_to_check = spack.repo.path.all_packages() pkg_to_entries = _get_external_packages(packages_to_check) diff --git a/lib/spack/spack/test/cmd/external.py b/lib/spack/spack/test/cmd/external.py index 5a94164e61..3249a92894 100644 --- a/lib/spack/spack/test/cmd/external.py +++ b/lib/spack/spack/test/cmd/external.py @@ -2,6 +2,8 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import pytest + import os import os.path @@ -242,3 +244,26 @@ def test_new_entries_are_reported_correctly( # has been found output = external('find', 'gcc') assert 'No new external packages detected' in output + + +@pytest.mark.parametrize('command_args', [ + ('-t', 'build-tools'), + ('-t', 'build-tools', 'cmake'), +]) +def test_use_tags_for_detection( + command_args, mock_executable, mutable_config, monkeypatch +): + # Prepare an environment to detect a fake cmake + cmake_exe = mock_executable('cmake', output="echo cmake version 3.19.1") + prefix = os.path.dirname(cmake_exe) + monkeypatch.setenv('PATH', prefix) + + openssl_exe = mock_executable('openssl', output="OpenSSL 2.8.3") + prefix = os.path.dirname(openssl_exe) + monkeypatch.setenv('PATH', prefix) + + # Test that we detect specs + output = external('find', *command_args) + assert 'The following specs have been' in output + assert 'cmake' in output + assert 'openssl' not in output diff --git a/lib/spack/spack/test/cmd/find.py b/lib/spack/spack/test/cmd/find.py index ef73b629e0..762578a028 100644 --- a/lib/spack/spack/test/cmd/find.py +++ b/lib/spack/spack/test/cmd/find.py @@ -89,7 +89,7 @@ def test_query_arguments(): @pytest.mark.usefixtures('database', 'mock_display') def test_tag1(parser, specs): - args = parser.parse_args(['--tags', 'tag1']) + args = parser.parse_args(['--tag', 'tag1']) spack.cmd.find.find(parser, args) assert len(specs) == 2 @@ -100,7 +100,7 @@ def test_tag1(parser, specs): @pytest.mark.db @pytest.mark.usefixtures('database', 'mock_display') def test_tag2(parser, specs): - args = parser.parse_args(['--tags', 'tag2']) + args = parser.parse_args(['--tag', 'tag2']) spack.cmd.find.find(parser, args) assert len(specs) == 1 @@ -110,7 +110,7 @@ def test_tag2(parser, specs): @pytest.mark.db @pytest.mark.usefixtures('database', 'mock_display') def test_tag2_tag3(parser, specs): - args = parser.parse_args(['--tags', 'tag2', '--tags', 'tag3']) + args = parser.parse_args(['--tag', 'tag2', '--tag', 'tag3']) spack.cmd.find.find(parser, args) assert len(specs) == 0 diff --git a/lib/spack/spack/test/cmd/list.py b/lib/spack/spack/test/cmd/list.py index afa71de90e..f99b7420a4 100644 --- a/lib/spack/spack/test/cmd/list.py +++ b/lib/spack/spack/test/cmd/list.py @@ -33,15 +33,15 @@ def test_list_search_description(): def test_list_tags(): - output = list('--tags', 'proxy-app') + output = list('--tag', 'proxy-app') assert 'cloverleaf3d' in output assert 'hdf5' not in output - output = list('--tags', 'hpc') + output = list('--tag', 'hpc') assert 'nek5000' in output assert 'mfem' in output - output = list('--tags', 'HPC') + output = list('--tag', 'HPC') assert 'nek5000' in output assert 'mfem' in output diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 09e33ef394..4a76c2898c 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -882,7 +882,7 @@ _spack_external() { _spack_external_find() { if $list_options then - SPACK_COMPREPLY="-h --help --not-buildable --scope" + SPACK_COMPREPLY="-h --help --not-buildable --scope -t --tag" else _all_packages fi @@ -904,7 +904,7 @@ _spack_fetch() { _spack_find() { if $list_options then - SPACK_COMPREPLY="-h --help --format --json -d --deps -p --paths --groups --no-groups -l --long -L --very-long -t --tags -c --show-concretized -f --show-flags --show-full-compiler -x --explicit -X --implicit -u --unknown -m --missing -v --variants --loaded -M --only-missing --deprecated --only-deprecated -N --namespace --start-date --end-date" + SPACK_COMPREPLY="-h --help --format --json -d --deps -p --paths --groups --no-groups -l --long -L --very-long -t --tag -c --show-concretized -f --show-flags --show-full-compiler -x --explicit -X --implicit -u --unknown -m --missing -v --variants --loaded -M --only-missing --deprecated --only-deprecated -N --namespace --start-date --end-date" else _installed_packages fi @@ -1063,7 +1063,7 @@ _spack_license_update_copyright_year() { _spack_list() { if $list_options then - SPACK_COMPREPLY="-h --help -d --search-description --format --update -v --virtuals -t --tags" + SPACK_COMPREPLY="-h --help -d --search-description --format --update -v --virtuals -t --tag" else _all_packages fi diff --git a/var/spack/repos/builtin/packages/autoconf/package.py b/var/spack/repos/builtin/packages/autoconf/package.py index 1ba77a4240..0a307e00c1 100644 --- a/var/spack/repos/builtin/packages/autoconf/package.py +++ b/var/spack/repos/builtin/packages/autoconf/package.py @@ -31,6 +31,8 @@ class Autoconf(AutotoolsPackage, GNUMirrorPackage): build_directory = 'spack-build' + tags = ['build-tools'] + executables = [ '^autoconf$', '^autoheader$', '^autom4te$', '^autoreconf$', '^autoscan$', '^autoupdate$', '^ifnames$' diff --git a/var/spack/repos/builtin/packages/automake/package.py b/var/spack/repos/builtin/packages/automake/package.py index 33d8abd5aa..e364fe07cb 100644 --- a/var/spack/repos/builtin/packages/automake/package.py +++ b/var/spack/repos/builtin/packages/automake/package.py @@ -25,6 +25,8 @@ class Automake(AutotoolsPackage, GNUMirrorPackage): build_directory = 'spack-build' + tags = ['build-tools'] + executables = ['^automake$'] @classmethod diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index 8278482e99..556a2117ce 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -14,6 +14,8 @@ class Cmake(Package): url = 'https://github.com/Kitware/CMake/releases/download/v3.19.0/cmake-3.19.0.tar.gz' maintainers = ['chuckatkins'] + tags = ['build-tools'] + executables = ['^cmake$'] version('3.19.2', sha256='e3e0fd3b23b7fb13e1a856581078e0776ffa2df4e9d3164039c36d3315e0c7f0') diff --git a/var/spack/repos/builtin/packages/flex/package.py b/var/spack/repos/builtin/packages/flex/package.py index 6ae8ddbce1..a3398e6566 100644 --- a/var/spack/repos/builtin/packages/flex/package.py +++ b/var/spack/repos/builtin/packages/flex/package.py @@ -14,6 +14,8 @@ class Flex(AutotoolsPackage): homepage = "https://github.com/westes/flex" url = "https://github.com/westes/flex/releases/download/v2.6.1/flex-2.6.1.tar.gz" + tags = ['build-tools'] + executables = ['^flex$'] version('2.6.4', sha256='e87aae032bf07c26f85ac0ed3250998c37621d95f8bd748b31f15b33c45ee995') diff --git a/var/spack/repos/builtin/packages/gmake/package.py b/var/spack/repos/builtin/packages/gmake/package.py index 3c01635d8e..0073ccb5f1 100644 --- a/var/spack/repos/builtin/packages/gmake/package.py +++ b/var/spack/repos/builtin/packages/gmake/package.py @@ -29,6 +29,8 @@ class Gmake(AutotoolsPackage, GNUMirrorPackage): patch('https://src.fedoraproject.org/rpms/make/raw/519a7c5bcbead22e6ea2d2c2341d981ef9e25c0d/f/make-4.2.1-glob-fix-2.patch', level=1, sha256='fe5b60d091c33f169740df8cb718bf4259f84528b42435194ffe0dd5b79cd125', when='@4.2.1') patch('https://src.fedoraproject.org/rpms/make/raw/519a7c5bcbead22e6ea2d2c2341d981ef9e25c0d/f/make-4.2.1-glob-fix-3.patch', level=1, sha256='ca60bd9c1a1b35bc0dc58b6a4a19d5c2651f7a94a4b22b2c5ea001a1ca7a8a7f', when='@:4.2.1') + tags = ['build-tools'] + executables = ['^g?make$'] @classmethod diff --git a/var/spack/repos/builtin/packages/libtool/package.py b/var/spack/repos/builtin/packages/libtool/package.py index b840688885..001327c78f 100644 --- a/var/spack/repos/builtin/packages/libtool/package.py +++ b/var/spack/repos/builtin/packages/libtool/package.py @@ -30,6 +30,8 @@ class Libtool(AutotoolsPackage, GNUMirrorPackage): build_directory = 'spack-build' + tags = ['build-tools'] + executables = ['^g?libtool(ize)?$'] @classmethod diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index 78ea153adc..23f802d9d1 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -33,6 +33,8 @@ class M4(AutotoolsPackage, GNUMirrorPackage): build_directory = 'spack-build' + tags = ['build-tools'] + executables = ['^g?m4$'] @classmethod diff --git a/var/spack/repos/builtin/packages/pkg-config/package.py b/var/spack/repos/builtin/packages/pkg-config/package.py index 89950cc7bd..f0f06d81ed 100644 --- a/var/spack/repos/builtin/packages/pkg-config/package.py +++ b/var/spack/repos/builtin/packages/pkg-config/package.py @@ -31,6 +31,8 @@ class PkgConfig(AutotoolsPackage): parallel = False + tags = ['build-tools'] + executables = ['^pkg-config$'] @classmethod diff --git a/var/spack/repos/builtin/packages/pkgconf/package.py b/var/spack/repos/builtin/packages/pkgconf/package.py index c6109d33cc..fdcddf7b56 100644 --- a/var/spack/repos/builtin/packages/pkgconf/package.py +++ b/var/spack/repos/builtin/packages/pkgconf/package.py @@ -37,6 +37,8 @@ class Pkgconf(AutotoolsPackage): # https://github.com/spack/spack/issues/3525 conflicts('%pgi') + tags = ['build-tools'] + executables = ['^pkgconf$', '^pkg-config$'] @classmethod -- cgit v1.2.3-60-g2f50