Rework command reference in docs, add `spack commands` command

- command reference now includes usage for all Spack commands as output
  by `spack help`.  Each command usage links to any related section in
  the docs.

- added `spack commands` command which can list command names,
  subcommands, and generate RST docs for commands.

- added `llnl.util.argparsewriter`, which analyzes an argparse parser and
  calls hooks for description, usage, options, and subcommands

7 files changed, 459 insertions, 20 deletions

diff --git a/lib/spack/docs/command_index.in b/lib/spack/docs/command_index.in
index 6520352b42..7f1f35c744 100644
--- a/lib/spack/docs/command_index.in
+++ b/lib/spack/docs/command_index.in
@@ -1,9 +1,9 @@
-=============
-Command Index
-=============
+=================
+Command Reference
+=================
 
-This is an alphabetical list of commands with links to the places they
-appear in the documentation.
+This is a reference for all commands in the Spack command line interface.
+The same information is available through :ref:`spack-help`.
 
-.. hlist::
-   :columns: 3
+Commands that also have sections in the main documentation have a link to
+"More documentation".
diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py
index 2d38211b4c..80e7f42bff 100644
--- a/lib/spack/docs/conf.py
+++ b/lib/spack/docs/conf.py
@@ -76,19 +76,23 @@ with open('package_list.html', 'w') as plist_file:
 #
 # Find all the `cmd-spack-*` references and add them to a command index
 #
-command_names = []
+import spack
+command_names = spack.cmd.all_commands
+documented_commands = set()
 for filename in glob('*rst'):
     with open(filename) as f:
         for line in f:
-            match = re.match('.. _(cmd-spack-.*):', line)
+            match = re.match('.. _cmd-(spack-.*):', line)
             if match:
-                command_names.append(match.group(1).strip())
+                documented_commands.add(match.group(1).strip())
 
+os.environ['COLUMNS'] = '120'
 shutil.copy('command_index.in', 'command_index.rst')
 with open('command_index.rst', 'a') as index:
-    index.write('\n')
-    for cmd in sorted(command_names):
-        index.write('   * :ref:`%s`\n' % cmd)
+    subprocess.Popen(
+        [spack_root + '/bin/spack', 'commands', '--format=rst'] + list(
+            documented_commands),
+        stdout=index)
 
 #
 # Run sphinx-apidoc
@@ -115,7 +119,7 @@ sphinx_apidoc(apidoc_args + ['../llnl'])
 # This also avoids issues where some of these symbols shadow core spack
 # modules.  Sphinx will complain about duplicate docs when this happens.
 #
-import fileinput, spack
+import fileinput
 handling_spack = False
 for line in fileinput.input('spack.rst', inplace=1):
     if handling_spack:
diff --git a/lib/spack/llnl/util/argparsewriter.py b/lib/spack/llnl/util/argparsewriter.py
new file mode 100644
index 0000000000..27439c2994
--- /dev/null
+++ b/lib/spack/llnl/util/argparsewriter.py
@@ -0,0 +1,222 @@
+##############################################################################
+# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/spack/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from __future__ import print_function
+import re
+import argparse
+import errno
+import sys
+
+
+class ArgparseWriter(object):
+    """Analyzes an argparse ArgumentParser for easy generation of help."""
+    def __init__(self):
+        self.level = 0
+
+    def _write(self, parser, root=True, level=0):
+        self.parser = parser
+        self.level = level
+        actions = parser._actions
+
+        # allow root level to be flattened with rest of commands
+        if type(root) == int:
+            self.level = root
+            root = True
+
+        # go through actions and split them into optionals, positionals,
+        # and subcommands
+        optionals = []
+        positionals = []
+        subcommands = []
+        for action in actions:
+            if action.option_strings:
+                optionals.append(action)
+            elif isinstance(action, argparse._SubParsersAction):
+                for subaction in action._choices_actions:
+                    subparser = action._name_parser_map[subaction.dest]
+                    subcommands.append(subparser)
+            else:
+                positionals.append(action)
+
+        groups = parser._mutually_exclusive_groups
+        fmt = parser._get_formatter()
+        description = parser.description
+
+        def action_group(function, actions):
+            for action in actions:
+                arg = fmt._format_action_invocation(action)
+                help = action.help if action.help else ''
+                function(arg, re.sub('\n', ' ', help))
+
+        if root:
+            self.begin_command(parser.prog)
+
+            if description:
+                self.description(parser.description)
+
+            usage = fmt._format_usage(None, actions, groups, '').strip()
+            self.usage(usage)
+
+            if positionals:
+                self.begin_positionals()
+                action_group(self.positional, positionals)
+                self.end_positionals()
+
+            if optionals:
+                self.begin_optionals()
+                action_group(self.optional, optionals)
+                self.end_optionals()
+
+        if subcommands:
+            self.begin_subcommands(subcommands)
+            for subparser in subcommands:
+                self._write(subparser, root=True, level=level + 1)
+            self.end_subcommands(subcommands)
+
+        if root:
+            self.end_command(parser.prog)
+
+    def write(self, parser, root=True):
+        """Write out details about an ArgumentParser.
+
+        Args:
+            parser (ArgumentParser): an ``argparse`` parser
+            root (bool or int): if bool, whether to include the root parser;
+                or ``1`` to flatten the root parser with first-level
+                subcommands
+        """
+        try:
+            self._write(parser, root, level=0)
+        except IOError as e:
+            # swallow pipe errors
+            if e.errno != errno.EPIPE:
+                raise
+
+    def begin_command(self, prog):
+        pass
+
+    def end_command(self, prog):
+        pass
+
+    def description(self, description):
+        pass
+
+    def usage(self, usage):
+        pass
+
+    def begin_positionals(self):
+        pass
+
+    def positional(self, name, help):
+        pass
+
+    def end_positionals(self):
+        pass
+
+    def begin_optionals(self):
+        pass
+
+    def optional(self, option, help):
+        pass
+
+    def end_optionals(self):
+        pass
+
+    def begin_subcommands(self, subcommands):
+        pass
+
+    def end_subcommands(self, subcommands):
+        pass
+
+
+_rst_levels = ['=', '-', '^', '~', ':', '`']
+
+
+class ArgparseRstWriter(ArgparseWriter):
+    """Write argparse output as rst sections."""
+
+    def __init__(self, out=sys.stdout, rst_levels=_rst_levels,
+                 strip_root_prog=True):
+        """Create a new ArgparseRstWriter.
+
+        Args:
+            out (file object): file to write to
+            rst_levels (list of str): list of characters
+                for rst section headings
+            strip_root_prog (bool): if ``True``, strip the base command name
+                from subcommands in output
+        """
+        super(ArgparseWriter, self).__init__()
+        self.out = out
+        self.rst_levels = rst_levels
+        self.strip_root_prog = strip_root_prog
+
+    def line(self, string=''):
+        self.out.write('%s\n' % string)
+
+    def begin_command(self, prog):
+        self.line()
+        self.line('----')
+        self.line()
+        self.line('.. _%s:\n' % prog.replace(' ', '-'))
+        self.line('%s' % prog)
+        self.line(self.rst_levels[self.level] * len(prog) + '\n')
+
+    def description(self, description):
+        self.line('%s\n' % description)
+
+    def usage(self, usage):
+        self.line('.. code-block:: console\n')
+        self.line('    %s\n' % usage)
+
+    def begin_positionals(self):
+        self.line()
+        self.line('**Positional arguments**\n')
+
+    def positional(self, name, help):
+        self.line(name)
+        self.line('  %s\n' % help)
+
+    def begin_optionals(self):
+        self.line()
+        self.line('**Optional arguments**\n')
+
+    def optional(self, opts, help):
+        self.line('``%s``' % opts)
+        self.line('  %s\n' % help)
+
+    def begin_subcommands(self, subcommands):
+        self.line()
+        self.line('**Subcommands**\n')
+        self.line('.. hlist::')
+        self.line('   :columns: 4\n')
+
+        for cmd in subcommands:
+            prog = cmd.prog
+            if self.strip_root_prog:
+                prog = re.sub(r'^[^ ]* ', '', prog)
+
+            self.line('   * :ref:`%s <%s>`'
+                      % (prog, cmd.prog.replace(' ', '-')))
+        self.line()
diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py
index 4114108fda..c161d548b4 100644
--- a/lib/spack/spack/cmd/__init__.py
+++ b/lib/spack/spack/cmd/__init__.py
@@ -45,6 +45,7 @@ import spack.store
 # Commands that modify configuration by default modify the *highest*
 # priority scope.
 default_modify_scope = spack.config.highest_precedence_scope().name
+
 # Commands that list configuration list *all* scopes by default.
 default_list_scope = None
 
@@ -60,7 +61,7 @@ DESCRIPTION = "description"
 command_path = os.path.join(spack.lib_path, "spack", "cmd")
 
 #: Names of all commands
-commands = []
+all_commands = []
 
 
 def python_name(cmd_name):
@@ -76,8 +77,8 @@ def cmd_name(python_name):
 for file in os.listdir(command_path):
     if file.endswith(".py") and not re.search(ignore_files, file):
         cmd = re.sub(r'.py$', '', file)
-        commands.append(cmd_name(cmd))
-commands.sort()
+        all_commands.append(cmd_name(cmd))
+all_commands.sort()
 
 
 def remove_options(parser, *options):
diff --git a/lib/spack/spack/cmd/commands.py b/lib/spack/spack/cmd/commands.py
new file mode 100644
index 0000000000..c8c0ab1e6c
--- /dev/null
+++ b/lib/spack/spack/cmd/commands.py
@@ -0,0 +1,142 @@
+##############################################################################
+# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/spack/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from __future__ import print_function
+
+import sys
+import re
+import argparse
+
+from llnl.util.argparsewriter import ArgparseWriter, ArgparseRstWriter
+
+import spack.main
+from spack.main import section_descriptions
+
+
+description = "list available spack commands"
+section = "developer"
+level = "long"
+
+
+#: list of command formatters
+formatters = {}
+
+
+def formatter(func):
+    """Decorator used to register formatters"""
+    formatters[func.__name__] = func
+    return func
+
+
+def setup_parser(subparser):
+    subparser.add_argument(
+        '--format', default='names', choices=formatters,
+        help='format to be used to print the output (default: names)')
+    subparser.add_argument(
+        'documented_commands', nargs=argparse.REMAINDER,
+        help='list of documented commands to cross-references')
+
+
+class SpackArgparseRstWriter(ArgparseRstWriter):
+    """RST writer tailored for spack documentation."""
+
+    def __init__(self, documented_commands, out=sys.stdout):
+        super(SpackArgparseRstWriter, self).__init__(out)
+        self.documented = documented_commands if documented_commands else []
+
+    def usage(self, *args):
+        super(SpackArgparseRstWriter, self).usage(*args)
+        cmd = re.sub(' ', '-', self.parser.prog)
+        if cmd in self.documented:
+            self.line()
+            self.line(':ref:`More documentation <cmd-%s>`' % cmd)
+
+
+class SubcommandWriter(ArgparseWriter):
+    def begin_command(self, prog):
+        print('    ' * self.level + prog)
+
+
+@formatter
+def subcommands(args):
+    parser = spack.main.make_argument_parser()
+    spack.main.add_all_commands(parser)
+    SubcommandWriter().write(parser)
+
+
+def rst_index(out=sys.stdout):
+    out.write('\n')
+
+    index = spack.main.index_commands()
+    sections = index['long']
+
+    dmax = max(len(section_descriptions.get(s, s)) for s in sections) + 2
+    cmax = max(len(c) for _, c in sections.items()) + 60
+
+    row = "%s  %s\n" % ('=' * dmax, '=' * cmax)
+    line = '%%-%ds  %%s\n' % dmax
+
+    out.write(row)
+    out.write(line % (" Category ", " Commands "))
+    out.write(row)
+    for section, commands in sorted(sections.items()):
+        description = section_descriptions.get(section, section)
+
+        for i, cmd in enumerate(sorted(commands)):
+            description = description.capitalize() if i == 0 else ''
+            ref = ':ref:`%s <spack-%s>`' % (cmd, cmd)
+            comma = ',' if i != len(commands) - 1 else ''
+            bar = '| ' if i % 8 == 0 else '  '
+            out.write(line % (description, bar + ref + comma))
+    out.write(row)
+
+
+@formatter
+def rst(args):
+    # print an index to each command
+    rst_index()
+    print()
+
+    # create a parser with all commands
+    parser = spack.main.make_argument_parser()
+    spack.main.add_all_commands(parser)
+
+    # get documented commands from the command line
+    documented_commands = set(args.documented_commands)
+
+    # print sections for each command and subcommand
+    SpackArgparseRstWriter(documented_commands).write(parser, root=1)
+
+
+@formatter
+def names(args):
+    for cmd in spack.cmd.all_commands:
+        print(cmd)
+
+
+def commands(parser, args):
+
+    # Print to stdout
+    formatters[args.format](args)
+    return
diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py
index f29a0a6d8b..735dad5a1d 100644
--- a/lib/spack/spack/main.py
+++ b/lib/spack/spack/main.py
@@ -100,14 +100,14 @@ def set_working_dir():
 
 def add_all_commands(parser):
     """Add all spack subcommands to the parser."""
-    for cmd in spack.cmd.commands:
+    for cmd in spack.cmd.all_commands:
         parser.add_command(cmd)
 
 
 def index_commands():
     """create an index of commands by section for this help level"""
     index = {}
-    for command in spack.cmd.commands:
+    for command in spack.cmd.all_commands:
         cmd_module = spack.cmd.get_module(command)
 
         # make sure command modules have required properties
@@ -166,7 +166,7 @@ class SpackArgumentParser(argparse.ArgumentParser):
             self.actions = self._subparsers._actions[-1]._get_subactions()
 
         # make a set of commands not yet added.
-        remaining = set(spack.cmd.commands)
+        remaining = set(spack.cmd.all_commands)
 
         def add_group(group):
             formatter.start_section(group.title)
diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py
new file mode 100644
index 0000000000..84f26e246d
--- /dev/null
+++ b/lib/spack/spack/test/cmd/commands.py
@@ -0,0 +1,70 @@
+##############################################################################
+# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/spack/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import re
+
+from llnl.util.argparsewriter import ArgparseWriter
+
+import spack.cmd
+import spack.main
+from spack.main import SpackCommand
+
+commands = SpackCommand('commands')
+
+parser = spack.main.make_argument_parser()
+spack.main.add_all_commands(parser)
+
+
+def test_commands_by_name():
+    """Test default output of spack commands."""
+    out = commands()
+    assert out.strip().split('\n') == sorted(spack.cmd.all_commands)
+
+
+def test_subcommands():
+    """Test subcommand traversal."""
+    out = commands('--format=subcommands')
+    assert 'spack mirror create' in out
+    assert 'spack buildcache list' in out
+    assert 'spack repo add' in out
+    assert 'spack pkg diff' in out
+    assert 'spack url parse' in out
+    assert 'spack view symlink' in out
+
+    class Subcommands(ArgparseWriter):
+        def begin_command(self, prog):
+            assert prog in out
+
+    Subcommands().write(parser)
+
+
+def test_rst():
+    """Do some simple sanity checks of the rst writer."""
+    out = commands('--format=rst')
+
+    class Subcommands(ArgparseWriter):
+        def begin_command(self, prog):
+            assert prog in out
+            assert re.sub(r' ', '-', prog) in out
+    Subcommands().write(parser)