commands: Add --header and --update options to `spack commands`

The Spack documentation currently hard-codes some functionality in
`conf.py`, which makes the doc build less "pluggable" for things like
localized doc builds.

In particular, we unconditionally generate an index of commands and a
package list as part of the docs, but those should really only be done if
things are not up to date.

This commit does the following:

- Add `--header` option to `spack commands` so that it can do the work of
  prepending text to its output.

- Add `--update FILE` option to `spack commands` that makes it generate a
  new command index *only* if FILE is out of date w.r.t. commands in the
  Spack source.

- Simplify code in `conf.py` to use these options and only update the
  command index when needed.

4 files changed, 143 insertions, 54 deletions

diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py
index b826299611..4af4e13035 100644
--- a/lib/spack/docs/conf.py
+++ b/lib/spack/docs/conf.py
@@ -51,36 +51,18 @@ os.environ['PATH'] += '%s%s/bin' % (os.pathsep, spack_root)
 # Set an environment variable so that colify will print output like it would to
 # a terminal.
 os.environ['COLIFY_SIZE'] = '25x120'
+os.environ['COLUMNS'] = '120'
 
-#
-# Generate package list using spack command
-#
-with open('package_list.html', 'w') as plist_file:
-    subprocess.Popen(
-        [spack_root + '/bin/spack', 'list', '--format=html'],
-        stdout=plist_file)
+# Generate full package list if needed
+subprocess.Popen(
+    ['spack', 'list', '--format=html', '--update=package_list.html'])
 
-#
-# Find all the `cmd-spack-*` references and add them to a command index
-#
-import spack
-import spack.cmd
-command_names = spack.cmd.all_commands()
-documented_commands = set()
-for filename in glob('*rst'):
-    with open(filename) as f:
-        for line in f:
-            match = re.match('.. _cmd-(spack-.*):', line)
-            if match:
-                documented_commands.add(match.group(1).strip())
-
-os.environ['COLUMNS'] = '120'
-shutil.copy('command_index.in', 'command_index.rst')
-with open('command_index.rst', 'a') as index:
-    subprocess.Popen(
-        [spack_root + '/bin/spack', 'commands', '--format=rst'] + list(
-            documented_commands),
-        stdout=index)
+# Generate a command index if an update is needed
+subprocess.call([
+    'spack', 'commands',
+    '--format=rst',
+    '--header=command_index.in',
+    '--update=command_index.rst'] + glob('*rst'))
 
 #
 # Run sphinx-apidoc
@@ -158,6 +140,7 @@ copyright = u'2013-2019, Lawrence Livermore National Laboratory.'
 # built documents.
 #
 # The short X.Y version.
+import spack
 version = '.'.join(str(s) for s in spack.spack_version_info[:2])
 # The full version, including alpha/beta/rc tags.
 release = spack.spack_version
diff --git a/lib/spack/llnl/util/argparsewriter.py b/lib/spack/llnl/util/argparsewriter.py
index 4b39e5683f..16bb570a77 100644
--- a/lib/spack/llnl/util/argparsewriter.py
+++ b/lib/spack/llnl/util/argparsewriter.py
@@ -12,8 +12,9 @@ import sys
 
 class ArgparseWriter(object):
     """Analyzes an argparse ArgumentParser for easy generation of help."""
-    def __init__(self):
+    def __init__(self, out=sys.stdout):
         self.level = 0
+        self.out = out
 
     def _write(self, parser, root=True, level=0):
         self.parser = parser
@@ -148,8 +149,7 @@ class ArgparseRstWriter(ArgparseWriter):
             strip_root_prog (bool): if ``True``, strip the base command name
                 from subcommands in output
         """
-        super(ArgparseWriter, self).__init__()
-        self.out = out
+        super(ArgparseRstWriter, self).__init__(out)
         self.rst_levels = rst_levels
         self.strip_root_prog = strip_root_prog
 
diff --git a/lib/spack/spack/cmd/commands.py b/lib/spack/spack/cmd/commands.py
index ad9bc657d7..8c1a6b1e77 100644
--- a/lib/spack/spack/cmd/commands.py
+++ b/lib/spack/spack/cmd/commands.py
@@ -6,11 +6,15 @@
 from __future__ import print_function
 
 import sys
+import os
 import re
 import argparse
 
+import llnl.util.tty as tty
 from llnl.util.argparsewriter import ArgparseWriter, ArgparseRstWriter
+from llnl.util.tty.colify import colify
 
+import spack.cmd
 import spack.main
 from spack.main import section_descriptions
 
@@ -35,8 +39,14 @@ def setup_parser(subparser):
         '--format', default='names', choices=formatters,
         help='format to be used to print the output (default: names)')
     subparser.add_argument(
-        'documented_commands', nargs=argparse.REMAINDER,
-        help='list of documented commands to cross-references')
+        '--header', metavar='FILE', default=None, action='store',
+        help='prepend contents of FILE to the output (useful for rst format)')
+    subparser.add_argument(
+        '--update', metavar='FILE', default=None, action='store',
+        help='write output to the specified file, if any command is newer')
+    subparser.add_argument(
+        'rst_files', nargs=argparse.REMAINDER,
+        help='list of rst files to search for `_cmd-spack-<cmd>` cross-refs')
 
 
 class SpackArgparseRstWriter(ArgparseRstWriter):
@@ -56,17 +66,18 @@ class SpackArgparseRstWriter(ArgparseRstWriter):
 
 class SubcommandWriter(ArgparseWriter):
     def begin_command(self, prog):
-        print('    ' * self.level + prog)
+        self.out.write('    ' * self.level + prog)
+        self.out.write('\n')
 
 
 @formatter
-def subcommands(args):
+def subcommands(args, out):
     parser = spack.main.make_argument_parser()
     spack.main.add_all_commands(parser)
-    SubcommandWriter().write(parser)
+    SubcommandWriter(out).write(parser)
 
 
-def rst_index(out=sys.stdout):
+def rst_index(out):
     out.write('\n')
 
     index = spack.main.index_commands()
@@ -94,30 +105,65 @@ def rst_index(out=sys.stdout):
 
 
 @formatter
-def rst(args):
-    # print an index to each command
-    rst_index()
-    print()
-
+def rst(args, out):
     # create a parser with all commands
     parser = spack.main.make_argument_parser()
     spack.main.add_all_commands(parser)
 
-    # get documented commands from the command line
-    documented_commands = set(args.documented_commands)
+    # extract cross-refs of the form `_cmd-spack-<cmd>:` from rst files
+    documented_commands = set()
+    for filename in args.rst_files:
+        with open(filename) as f:
+            for line in f:
+                match = re.match(r'\.\. _cmd-(spack-.*):', line)
+                if match:
+                    documented_commands.add(match.group(1).strip())
+
+    # print an index to each command
+    rst_index(out)
+    out.write('\n')
 
     # print sections for each command and subcommand
-    SpackArgparseRstWriter(documented_commands).write(parser, root=1)
+    SpackArgparseRstWriter(documented_commands, out).write(parser, root=1)
 
 
 @formatter
-def names(args):
-    for cmd in spack.cmd.all_commands():
-        print(cmd)
+def names(args, out):
+    colify(spack.cmd.all_commands(), output=out)
 
 
-def commands(parser, args):
+def prepend_header(args, out):
+    if not args.header:
+        return
+
+    with open(args.header) as header:
+        out.write(header.read())
 
-    # Print to stdout
-    formatters[args.format](args)
-    return
+
+def commands(parser, args):
+    formatter = formatters[args.format]
+
+    # check header first so we don't open out files unnecessarily
+    if args.header and not os.path.exists(args.header):
+        tty.die("No such file: '%s'" % args.header)
+
+    # if we're updating an existing file, only write output if a command
+    # is newer than the file.
+    if args.update:
+        if os.path.exists(args.update):
+            files = [
+                spack.cmd.get_module(command).__file__.rstrip('c')  # pyc -> py
+                for command in spack.cmd.all_commands()]
+            last_update = os.path.getmtime(args.update)
+            if not any(os.path.getmtime(f) > last_update for f in files):
+                tty.msg('File is up to date: %s' % args.update)
+                return
+
+        tty.msg('Updating file: %s' % args.update)
+        with open(args.update, 'w') as f:
+            prepend_header(args, f)
+            formatter(args, f)
+
+    else:
+        prepend_header(args, sys.stdout)
+        formatter(args, sys.stdout)
diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py
index e24bd9ac14..f0b80ea031 100644
--- a/lib/spack/spack/test/cmd/commands.py
+++ b/lib/spack/spack/test/cmd/commands.py
@@ -4,14 +4,14 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import re
+import pytest
 
 from llnl.util.argparsewriter import ArgparseWriter
 
 import spack.cmd
 import spack.main
-from spack.main import SpackCommand
 
-commands = SpackCommand('commands')
+commands = spack.main.SpackCommand('commands')
 
 parser = spack.main.make_argument_parser()
 spack.main.add_all_commands(parser)
@@ -49,3 +49,63 @@ def test_rst():
             assert prog in out
             assert re.sub(r' ', '-', prog) in out
     Subcommands().write(parser)
+
+
+def test_rst_with_input_files(tmpdir):
+    filename = tmpdir.join('file.rst')
+    with filename.open('w') as f:
+        f.write('''
+.. _cmd-spack-fetch:
+cmd-spack-list:
+.. _cmd-spack-stage:
+_cmd-spack-install:
+.. _cmd-spack-patch:
+''')
+
+    out = commands('--format=rst', str(filename))
+    for name in ['fetch', 'stage', 'patch']:
+        assert (':ref:`More documentation <cmd-spack-%s>`' % name) in out
+
+    for name in ['list', 'install']:
+        assert (':ref:`More documentation <cmd-spack-%s>`' % name) not in out
+
+
+def test_rst_with_header(tmpdir):
+    fake_header = 'this is a header!\n\n'
+
+    filename = tmpdir.join('header.txt')
+    with filename.open('w') as f:
+        f.write(fake_header)
+
+    out = commands('--format=rst', '--header', str(filename))
+    assert out.startswith(fake_header)
+
+    with pytest.raises(spack.main.SpackCommandError):
+        commands('--format=rst', '--header', 'asdfjhkf')
+
+
+def test_rst_update(tmpdir):
+    update_file = tmpdir.join('output')
+
+    # not yet created when commands is run
+    commands('--update', str(update_file))
+    assert update_file.exists()
+    with update_file.open() as f:
+        assert f.read()
+
+    # created but older than commands
+    with update_file.open('w') as f:
+        f.write('empty\n')
+    update_file.setmtime(0)
+    commands('--update', str(update_file))
+    assert update_file.exists()
+    with update_file.open() as f:
+        assert f.read() != 'empty\n'
+
+    # newer than commands
+    with update_file.open('w') as f:
+        f.write('empty\n')
+    commands('--update', str(update_file))
+    assert update_file.exists()
+    with update_file.open() as f:
+        assert f.read() == 'empty\n'