feature: add "spack tags" command (#26136)

This PR adds a "spack tags" command to output package tags or 
(available) packages with those tags. It also ensures each package
is listed in the tag cache ONLY ONCE per tag.

9 files changed, 490 insertions, 77 deletions

diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py
index b85ecc6057..edff8ce66e 100644
--- a/lib/spack/spack/cmd/list.py
+++ b/lib/spack/spack/cmd/list.py
@@ -16,7 +16,6 @@ import sys
 import llnl.util.tty as tty
 from llnl.util.tty.colify import colify
 
-import spack.cmd.common.arguments as arguments
 import spack.dependency
 import spack.repo
 from spack.version import VersionList
@@ -57,8 +56,6 @@ def setup_parser(subparser):
         '-v', '--virtuals', action='store_true', default=False,
         help='include virtual packages in list')
 
-    arguments.add_common_arguments(subparser, ['tags'])
-
 
 def filter_by_name(pkgs, args):
     """
@@ -277,13 +274,6 @@ def list(parser, args):
     # Filter the set appropriately
     sorted_packages = filter_by_name(pkgs, args)
 
-    # Filter by tags
-    if args.tags:
-        packages_with_tags = set(
-            spack.repo.path.packages_with_tags(*args.tags))
-        sorted_packages = set(sorted_packages) & packages_with_tags
-        sorted_packages = sorted(sorted_packages)
-
     if args.update:
         # change output stream if user asked for update
         if os.path.exists(args.update):
diff --git a/lib/spack/spack/cmd/tags.py b/lib/spack/spack/cmd/tags.py
new file mode 100644
index 0000000000..d0f3951687
--- /dev/null
+++ b/lib/spack/spack/cmd/tags.py
@@ -0,0 +1,107 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import sys
+
+import six
+
+import llnl.util.tty as tty
+import llnl.util.tty.colify as colify
+
+import spack.repo
+import spack.store
+import spack.tag
+
+description = "Show package tags and associated packages"
+section = "basic"
+level = "long"
+
+
+def report_tags(category, tags):
+    buffer = six.StringIO()
+    isatty = sys.stdout.isatty()
+
+    if isatty:
+        num = len(tags)
+        fmt = '{0} package tag'.format(category)
+        buffer.write("{0}:\n".format(spack.util.string.plural(num, fmt)))
+
+    if tags:
+        colify.colify(tags, output=buffer, tty=isatty, indent=4)
+    else:
+        buffer.write("    None\n")
+    print(buffer.getvalue())
+
+
+def setup_parser(subparser):
+    subparser.epilog = (
+        "Tags from known packages will be used if no tags are provided on "
+        "the command\nline. If tags are provided, packages with at least one "
+        "will be reported.\n\nYou are not allowed to provide tags and use "
+        "'--all' at the same time."
+    )
+    subparser.add_argument(
+        '-i', '--installed', action='store_true', default=False,
+        help="show information for installed packages only"
+    )
+    subparser.add_argument(
+        '-a', '--all', action='store_true', default=False,
+        help="show packages for all available tags"
+    )
+    subparser.add_argument(
+        'tag',
+        nargs='*',
+        help="show packages with the specified tag"
+    )
+
+
+def tags(parser, args):
+    # Disallow combining all option with (positional) tags to avoid confusion
+    if args.all and args.tag:
+        tty.die("Use the '--all' option OR provide tag(s) on the command line")
+
+    # Provide a nice, simple message if database is empty
+    if args.installed and not spack.environment.installed_specs():
+        tty.msg("No installed packages")
+        return
+
+    # unique list of available tags
+    available_tags = sorted(spack.repo.path.tag_index.keys())
+    if not available_tags:
+        tty.msg("No tagged packages")
+        return
+
+    show_packages = args.tag or args.all
+
+    # Only report relevant, available tags if no packages are to be shown
+    if not show_packages:
+        if not args.installed:
+            report_tags("available", available_tags)
+        else:
+            tag_pkgs = spack.tag.packages_with_tags(available_tags, True, True)
+            tags = tag_pkgs.keys() if tag_pkgs else []
+            report_tags("installed", tags)
+        return
+
+    # Report packages associated with tags
+    buffer = six.StringIO()
+    isatty = sys.stdout.isatty()
+
+    tags = args.tag if args.tag else available_tags
+    tag_pkgs = spack.tag.packages_with_tags(tags, args.installed, False)
+    missing = 'No installed packages' if args.installed else 'None'
+    for tag in sorted(tag_pkgs):
+        # TODO: Remove the sorting once we're sure noone has an old
+        # TODO: tag cache since it can accumulate duplicates.
+        packages = sorted(list(set(tag_pkgs[tag])))
+        if isatty:
+            buffer.write("{0}:\n".format(tag))
+
+        if packages:
+            colify.colify(packages, output=buffer, tty=isatty, indent=4)
+        else:
+            buffer.write("    {0}\n".format(missing))
+        buffer.write("\n")
+    print(buffer.getvalue())
diff --git a/lib/spack/spack/environment/__init__.py b/lib/spack/spack/environment/__init__.py
index 0f04162b35..63633bb03c 100644
--- a/lib/spack/spack/environment/__init__.py
+++ b/lib/spack/spack/environment/__init__.py
@@ -17,6 +17,7 @@ from .environment import (
     default_view_name,
     display_specs,
     exists,
+    installed_specs,
     is_env_dir,
     is_latest_format,
     lockfile_name,
@@ -44,6 +45,7 @@ __all__ = [
     'default_view_name',
     'display_specs',
     'exists',
+    'installed_specs',
     'is_env_dir',
     'is_latest_format',
     'lockfile_name',
diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py
index 5304f2b135..4ed3d2508e 100644
--- a/lib/spack/spack/environment/environment.py
+++ b/lib/spack/spack/environment/environment.py
@@ -102,6 +102,16 @@ default_view_name = 'default'
 default_view_link = 'all'
 
 
+def installed_specs():
+    """
+    Returns the specs of packages installed in the active environment or None
+    if no packages are installed.
+    """
+    env = spack.environment.active_environment()
+    hashes = env.all_hashes() if env else None
+    return spack.store.db.query(hashes=hashes)
+
+
 def valid_env_name(name):
     return re.match(valid_environment_name_re, name)
 
diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py
index 7dc269065f..ee415c480d 100644
--- a/lib/spack/spack/repo.py
+++ b/lib/spack/spack/repo.py
@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import abc
-import collections
 import contextlib
 import errno
 import functools
@@ -38,10 +37,10 @@ import spack.error
 import spack.patch
 import spack.provider_index
 import spack.spec
+import spack.tag
 import spack.util.imp as simp
 import spack.util.naming as nm
 import spack.util.path
-import spack.util.spack_json as sjson
 
 #: Super-namespace for all packages.
 #: Package modules are imported as spack.pkg.<namespace>.<pkg-name>.
@@ -219,55 +218,6 @@ class FastPackageChecker(Mapping):
         return len(self._packages_to_stats)
 
 
-class TagIndex(Mapping):
-    """Maps tags to list of packages."""
-
-    def __init__(self):
-        self._tag_dict = collections.defaultdict(list)
-
-    def to_json(self, stream):
-        sjson.dump({'tags': self._tag_dict}, stream)
-
-    @staticmethod
-    def from_json(stream):
-        d = sjson.load(stream)
-
-        r = TagIndex()
-
-        for tag, list in d['tags'].items():
-            r[tag].extend(list)
-
-        return r
-
-    def __getitem__(self, item):
-        return self._tag_dict[item]
-
-    def __iter__(self):
-        return iter(self._tag_dict)
-
-    def __len__(self):
-        return len(self._tag_dict)
-
-    def update_package(self, pkg_name):
-        """Updates a package in the tag index.
-
-        Args:
-            pkg_name (str): name of the package to be removed from the index
-
-        """
-        package = path.get(pkg_name)
-
-        # Remove the package from the list of packages, if present
-        for pkg_list in self._tag_dict.values():
-            if pkg_name in pkg_list:
-                pkg_list.remove(pkg_name)
-
-        # Add it again under the appropriate tags
-        for tag in getattr(package, 'tags', []):
-            tag = tag.lower()
-            self._tag_dict[tag].append(package.name)
-
-
 @six.add_metaclass(abc.ABCMeta)
 class Indexer(object):
     """Adaptor for indexes that need to be generated when repos are updated."""
@@ -311,10 +261,10 @@ class Indexer(object):
 class TagIndexer(Indexer):
     """Lifecycle methods for a TagIndex on a Repo."""
     def _create(self):
-        return TagIndex()
+        return spack.tag.TagIndex()
 
     def read(self, stream):
-        self.index = TagIndex.from_json(stream)
+        self.index = spack.tag.TagIndex.from_json(stream)
 
     def update(self, pkg_fullname):
         self.index.update_package(pkg_fullname)
@@ -475,6 +425,7 @@ class RepoPath(object):
 
         self._provider_index = None
         self._patch_index = None
+        self._tag_index = None
 
         # Add each repo to this path.
         for repo in repos:
@@ -580,6 +531,16 @@ class RepoPath(object):
         return self._provider_index
 
     @property
+    def tag_index(self):
+        """Merged TagIndex from all Repos in the RepoPath."""
+        if self._tag_index is None:
+            self._tag_index = spack.tag.TagIndex()
+            for repo in reversed(self.repos):
+                self._tag_index.merge(repo.tag_index)
+
+        return self._tag_index
+
+    @property
     def patch_index(self):
         """Merged PatchIndex from all Repos in the RepoPath."""
         if self._patch_index is None:
diff --git a/lib/spack/spack/tag.py b/lib/spack/spack/tag.py
new file mode 100644
index 0000000000..6e2eea6864
--- /dev/null
+++ b/lib/spack/spack/tag.py
@@ -0,0 +1,135 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Classes and functions to manage package tags"""
+import collections
+import copy
+import sys
+
+if sys.version_info >= (3, 5):
+    from collections.abc import Mapping  # novm
+else:
+    from collections import Mapping
+
+import spack.error
+import spack.util.spack_json as sjson
+
+
+def _get_installed_package_names():
+    """Returns names of packages installed in the active environment."""
+    specs = spack.environment.installed_specs()
+    return [spec.name for spec in specs]
+
+
+def packages_with_tags(tags, installed, skip_empty):
+    """
+    Returns a dict, indexed by tag, containing lists of names of packages
+    containing the tag or, if no tags, for all available tags.
+
+    Arguments:
+        tags (list or None): list of tags of interest or None for all
+        installed (bool): True if want names of packages that are installed;
+            otherwise, False if want all packages with the tag
+        skip_empty (bool): True if exclude tags with no associated packages;
+            otherwise, False if want entries for all tags even when no such
+            tagged packages
+    """
+    tag_pkgs = collections.defaultdict(lambda: list)
+    spec_names = _get_installed_package_names() if installed else []
+    keys = spack.repo.path.tag_index if tags is None else tags
+    for tag in keys:
+        packages = [name for name in spack.repo.path.tag_index[tag] if
+                    not installed or name in spec_names]
+        if packages or not skip_empty:
+            tag_pkgs[tag] = packages
+    return tag_pkgs
+
+
+class TagIndex(Mapping):
+    """Maps tags to list of packages."""
+
+    def __init__(self):
+        self._tag_dict = collections.defaultdict(list)
+
+    @property
+    def tags(self):
+        return self._tag_dict
+
+    def to_json(self, stream):
+        sjson.dump({'tags': self._tag_dict}, stream)
+
+    @staticmethod
+    def from_json(stream):
+        d = sjson.load(stream)
+
+        if not isinstance(d, dict):
+            raise TagIndexError("TagIndex data was not a dict.")
+
+        if 'tags' not in d:
+            raise TagIndexError("TagIndex data does not start with 'tags'")
+
+        r = TagIndex()
+
+        for tag, packages in d['tags'].items():
+            r[tag].extend(packages)
+
+        return r
+
+    def __getitem__(self, item):
+        return self._tag_dict[item]
+
+    def __iter__(self):
+        return iter(self._tag_dict)
+
+    def __len__(self):
+        return len(self._tag_dict)
+
+    def copy(self):
+        """Return a deep copy of this index."""
+        clone = TagIndex()
+        clone._tag_dict = copy.deepcopy(self._tag_dict)
+        return clone
+
+    def get_packages(self, tag):
+        """Returns all packages associated with the tag."""
+        return self.tags[tag] if tag in self.tags else []
+
+    def merge(self, other):
+        """Merge another tag index into this one.
+
+        Args:
+            other (TagIndex): tag index to be merged
+        """
+        other = other.copy()   # defensive copy.
+
+        for tag in other.tags:
+            if tag not in self.tags:
+                self.tags[tag] = other.tags[tag]
+                continue
+
+            spkgs, opkgs = self.tags[tag], other.tags[tag]
+            self.tags[tag] = sorted(list(set(spkgs + opkgs)))
+
+    def update_package(self, pkg_name):
+        """Updates a package in the tag index.
+
+        Args:
+            pkg_name (str): name of the package to be removed from the index
+
+        """
+        package = spack.repo.path.get(pkg_name)
+
+        # Remove the package from the list of packages, if present
+        for pkg_list in self._tag_dict.values():
+            if pkg_name in pkg_list:
+                pkg_list.remove(pkg_name)
+
+        # Add it again under the appropriate tags
+        for tag in getattr(package, 'tags', []):
+            tag = tag.lower()
+            self._tag_dict[tag].append(package.name)
+
+
+class TagIndexError(spack.error.SpackError):
+    """Raised when there is a problem with a TagIndex."""
diff --git a/lib/spack/spack/test/cmd/list.py b/lib/spack/spack/test/cmd/list.py
index b0adeb7e55..c23ac8f68e 100644
--- a/lib/spack/spack/test/cmd/list.py
+++ b/lib/spack/spack/test/cmd/list.py
@@ -35,20 +35,6 @@ def test_list_search_description(mock_packages):
     assert 'depb' in output
 
 
-def test_list_tags(mock_packages):
-    output = list('--tag', 'tag1')
-    assert 'mpich' in output
-    assert 'mpich2' in output
-
-    output = list('--tag', 'tag2')
-    assert 'mpich\n' in output
-    assert 'mpich2' not in output
-
-    output = list('--tag', 'tag3')
-    assert 'mpich\n' not in output
-    assert 'mpich2' in output
-
-
 def test_list_format_name_only(mock_packages):
     output = list('--format', 'name_only')
     assert 'zmpi' in output
diff --git a/lib/spack/spack/test/cmd/tags.py b/lib/spack/spack/test/cmd/tags.py
new file mode 100644
index 0000000000..53fc85efdd
--- /dev/null
+++ b/lib/spack/spack/test/cmd/tags.py
@@ -0,0 +1,62 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import spack.main
+import spack.repo
+import spack.spec
+
+tags = spack.main.SpackCommand('tags')
+
+
+def test_tags_bad_options():
+    out = tags('-a', 'tag1', fail_on_error=False)
+    assert "option OR provide" in out
+
+
+def test_tags_no_installed(install_mockery, mock_fetch):
+    out = tags('-i')
+    assert 'No installed' in out
+
+
+def test_tags_invalid_tag(mock_packages):
+    out = tags('nosuchtag')
+    assert 'None' in out
+
+
+def test_tags_all_mock_tags(mock_packages):
+    out = tags()
+    for tag in ['tag1', 'tag2', 'tag3']:
+        assert tag in out
+
+
+def test_tags_all_mock_tag_packages(mock_packages):
+    out = tags('-a')
+    for pkg in ['mpich\n', 'mpich2\n']:
+        assert pkg in out
+
+
+def test_tags_no_tags(monkeypatch):
+    class tag_path():
+        tag_index = dict()
+
+    monkeypatch.setattr(spack.repo, 'path', tag_path)
+    out = tags()
+    assert "No tagged" in out
+
+
+def test_tags_installed(install_mockery, mock_fetch):
+    spec = spack.spec.Spec('mpich').concretized()
+    pkg = spack.repo.get(spec)
+    pkg.do_install()
+
+    out = tags('-i')
+    for tag in ['tag1', 'tag2']:
+        assert tag in out
+
+    out = tags('-i', 'tag1')
+    assert 'mpich' in out
+
+    out = tags('-i', 'tag3')
+    assert 'No installed' in out
diff --git a/lib/spack/spack/test/tag.py b/lib/spack/spack/test/tag.py
new file mode 100644
index 0000000000..0a69989f90
--- /dev/null
+++ b/lib/spack/spack/test/tag.py
@@ -0,0 +1,160 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Tests for tag index cache files."""
+
+import pytest
+from six import StringIO
+
+import spack.cmd.install
+import spack.tag
+from spack.main import SpackCommand
+
+install = SpackCommand('install')
+
+# Alternate representation
+tags_json = \
+    """
+    {
+      "tags": {
+        "no-version": [
+          "noversion",
+          "noversion-bundle"
+        ],
+        "no-source": [
+          "nosource"
+        ]
+      }
+    }
+    """
+
+more_tags_json = \
+    """
+    {
+      "tags": {
+        "merge": [
+          "check"
+        ]
+      }
+    }
+    """
+
+
+def test_tag_copy(mock_packages):
+    index = spack.tag.TagIndex.from_json(StringIO(tags_json))
+    new_index = index.copy()
+
+    assert index.tags == new_index.tags
+
+
+def test_tag_get_all_available(mock_packages):
+    for skip in [False, True]:
+        all_pkgs = spack.tag.packages_with_tags(None, False, skip)
+        assert sorted(all_pkgs['tag1']) == ['mpich', 'mpich2']
+        assert all_pkgs['tag2'] == ['mpich']
+        assert all_pkgs['tag3'] == ['mpich2']
+
+
+def ensure_tags_results_equal(results, expected):
+    if expected:
+        assert sorted(results.keys()) == sorted(expected.keys())
+        for tag in results:
+            assert sorted(results[tag]) == sorted(expected[tag])
+    else:
+        assert results == expected
+
+
+@pytest.mark.parametrize('tags,expected', [
+    (['tag1'], {'tag1': ['mpich', 'mpich2']}),
+    (['tag2'], {'tag2': ['mpich']}),
+    (['tag3'], {'tag3': ['mpich2']}),
+    (['nosuchpackage'], {'nosuchpackage': {}}),
+])
+def test_tag_get_available(tags, expected, mock_packages):
+    # Ensure results for all tags
+    all_tag_pkgs = spack.tag.packages_with_tags(tags, False, False)
+    ensure_tags_results_equal(all_tag_pkgs, expected)
+
+    # Ensure results for tags expecting results since skipping otherwise
+    only_pkgs = spack.tag.packages_with_tags(tags, False, True)
+    if expected[tags[0]]:
+        ensure_tags_results_equal(only_pkgs, expected)
+    else:
+        assert not only_pkgs
+
+
+def test_tag_get_installed_packages(
+        mock_packages, mock_archive, mock_fetch, install_mockery):
+    install('mpich')
+
+    for skip in [False, True]:
+        all_pkgs = spack.tag.packages_with_tags(None, True, skip)
+        assert sorted(all_pkgs['tag1']) == ['mpich']
+        assert all_pkgs['tag2'] == ['mpich']
+        assert skip or all_pkgs['tag3'] == []
+
+
+def test_tag_index_round_trip(mock_packages):
+    # Assumes at least two packages -- mpich and mpich2 -- have tags
+    mock_index = spack.repo.path.tag_index
+    assert mock_index.tags
+
+    ostream = StringIO()
+    mock_index.to_json(ostream)
+
+    istream = StringIO(ostream.getvalue())
+    new_index = spack.tag.TagIndex.from_json(istream)
+
+    assert mock_index == new_index
+
+
+def test_tag_equal():
+    first_index = spack.tag.TagIndex.from_json(StringIO(tags_json))
+    second_index = spack.tag.TagIndex.from_json(StringIO(tags_json))
+
+    assert first_index == second_index
+
+
+def test_tag_merge():
+    first_index = spack.tag.TagIndex.from_json(StringIO(tags_json))
+    second_index = spack.tag.TagIndex.from_json(StringIO(more_tags_json))
+
+    assert first_index != second_index
+
+    tags1 = list(first_index.tags.keys())
+    tags2 = list(second_index.tags.keys())
+    all_tags = sorted(list(set(tags1 + tags2)))
+
+    first_index.merge(second_index)
+    tag_keys = sorted(first_index.tags.keys())
+    assert tag_keys == all_tags
+
+    # Merge again to make sure the index does not retain duplicates
+    first_index.merge(second_index)
+    tag_keys = sorted(first_index.tags.keys())
+    assert tag_keys == all_tags
+
+
+def test_tag_not_dict():
+    list_json = "[]"
+    with pytest.raises(spack.tag.TagIndexError) as e:
+        spack.tag.TagIndex.from_json(StringIO(list_json))
+        assert "not a dict" in str(e)
+
+
+def test_tag_no_tags():
+    pkg_json = "{\"packages\": []}"
+    with pytest.raises(spack.tag.TagIndexError) as e:
+        spack.tag.TagIndex.from_json(StringIO(pkg_json))
+        assert "does not start with" in str(e)
+
+
+def test_tag_update_package(mock_packages):
+    mock_index = spack.repo.path.tag_index
+
+    index = spack.tag.TagIndex()
+    for name in spack.repo.all_package_names():
+        index.update_package(name)
+
+    ensure_tags_results_equal(mock_index.tags, index.tags)