From 6380f1917a840bfe9fab6bc219c6806e30b7ce2a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 20 May 2019 12:44:36 -0700 Subject: commands: Add --header and --update options to `spack commands` The Spack documentation currently hard-codes some functionality in `conf.py`, which makes the doc build less "pluggable" for things like localized doc builds. In particular, we unconditionally generate an index of commands and a package list as part of the docs, but those should really only be done if things are not up to date. This commit does the following: - Add `--header` option to `spack commands` so that it can do the work of prepending text to its output. - Add `--update FILE` option to `spack commands` that makes it generate a new command index *only* if FILE is out of date w.r.t. commands in the Spack source. - Simplify code in `conf.py` to use these options and only update the command index when needed. --- lib/spack/docs/conf.py | 39 +++++----------- lib/spack/llnl/util/argparsewriter.py | 6 +-- lib/spack/spack/cmd/commands.py | 88 ++++++++++++++++++++++++++--------- lib/spack/spack/test/cmd/commands.py | 64 ++++++++++++++++++++++++- 4 files changed, 143 insertions(+), 54 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index b826299611..4af4e13035 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -51,36 +51,18 @@ os.environ['PATH'] += '%s%s/bin' % (os.pathsep, spack_root) # Set an environment variable so that colify will print output like it would to # a terminal. os.environ['COLIFY_SIZE'] = '25x120' +os.environ['COLUMNS'] = '120' -# -# Generate package list using spack command -# -with open('package_list.html', 'w') as plist_file: - subprocess.Popen( - [spack_root + '/bin/spack', 'list', '--format=html'], - stdout=plist_file) +# Generate full package list if needed +subprocess.Popen( + ['spack', 'list', '--format=html', '--update=package_list.html']) -# -# Find all the `cmd-spack-*` references and add them to a command index -# -import spack -import spack.cmd -command_names = spack.cmd.all_commands() -documented_commands = set() -for filename in glob('*rst'): - with open(filename) as f: - for line in f: - match = re.match('.. _cmd-(spack-.*):', line) - if match: - documented_commands.add(match.group(1).strip()) - -os.environ['COLUMNS'] = '120' -shutil.copy('command_index.in', 'command_index.rst') -with open('command_index.rst', 'a') as index: - subprocess.Popen( - [spack_root + '/bin/spack', 'commands', '--format=rst'] + list( - documented_commands), - stdout=index) +# Generate a command index if an update is needed +subprocess.call([ + 'spack', 'commands', + '--format=rst', + '--header=command_index.in', + '--update=command_index.rst'] + glob('*rst')) # # Run sphinx-apidoc @@ -158,6 +140,7 @@ copyright = u'2013-2019, Lawrence Livermore National Laboratory.' # built documents. # # The short X.Y version. +import spack version = '.'.join(str(s) for s in spack.spack_version_info[:2]) # The full version, including alpha/beta/rc tags. release = spack.spack_version diff --git a/lib/spack/llnl/util/argparsewriter.py b/lib/spack/llnl/util/argparsewriter.py index 4b39e5683f..16bb570a77 100644 --- a/lib/spack/llnl/util/argparsewriter.py +++ b/lib/spack/llnl/util/argparsewriter.py @@ -12,8 +12,9 @@ import sys class ArgparseWriter(object): """Analyzes an argparse ArgumentParser for easy generation of help.""" - def __init__(self): + def __init__(self, out=sys.stdout): self.level = 0 + self.out = out def _write(self, parser, root=True, level=0): self.parser = parser @@ -148,8 +149,7 @@ class ArgparseRstWriter(ArgparseWriter): strip_root_prog (bool): if ``True``, strip the base command name from subcommands in output """ - super(ArgparseWriter, self).__init__() - self.out = out + super(ArgparseRstWriter, self).__init__(out) self.rst_levels = rst_levels self.strip_root_prog = strip_root_prog diff --git a/lib/spack/spack/cmd/commands.py b/lib/spack/spack/cmd/commands.py index ad9bc657d7..8c1a6b1e77 100644 --- a/lib/spack/spack/cmd/commands.py +++ b/lib/spack/spack/cmd/commands.py @@ -6,11 +6,15 @@ from __future__ import print_function import sys +import os import re import argparse +import llnl.util.tty as tty from llnl.util.argparsewriter import ArgparseWriter, ArgparseRstWriter +from llnl.util.tty.colify import colify +import spack.cmd import spack.main from spack.main import section_descriptions @@ -35,8 +39,14 @@ def setup_parser(subparser): '--format', default='names', choices=formatters, help='format to be used to print the output (default: names)') subparser.add_argument( - 'documented_commands', nargs=argparse.REMAINDER, - help='list of documented commands to cross-references') + '--header', metavar='FILE', default=None, action='store', + help='prepend contents of FILE to the output (useful for rst format)') + subparser.add_argument( + '--update', metavar='FILE', default=None, action='store', + help='write output to the specified file, if any command is newer') + subparser.add_argument( + 'rst_files', nargs=argparse.REMAINDER, + help='list of rst files to search for `_cmd-spack-` cross-refs') class SpackArgparseRstWriter(ArgparseRstWriter): @@ -56,17 +66,18 @@ class SpackArgparseRstWriter(ArgparseRstWriter): class SubcommandWriter(ArgparseWriter): def begin_command(self, prog): - print(' ' * self.level + prog) + self.out.write(' ' * self.level + prog) + self.out.write('\n') @formatter -def subcommands(args): +def subcommands(args, out): parser = spack.main.make_argument_parser() spack.main.add_all_commands(parser) - SubcommandWriter().write(parser) + SubcommandWriter(out).write(parser) -def rst_index(out=sys.stdout): +def rst_index(out): out.write('\n') index = spack.main.index_commands() @@ -94,30 +105,65 @@ def rst_index(out=sys.stdout): @formatter -def rst(args): - # print an index to each command - rst_index() - print() - +def rst(args, out): # create a parser with all commands parser = spack.main.make_argument_parser() spack.main.add_all_commands(parser) - # get documented commands from the command line - documented_commands = set(args.documented_commands) + # extract cross-refs of the form `_cmd-spack-:` from rst files + documented_commands = set() + for filename in args.rst_files: + with open(filename) as f: + for line in f: + match = re.match(r'\.\. _cmd-(spack-.*):', line) + if match: + documented_commands.add(match.group(1).strip()) + + # print an index to each command + rst_index(out) + out.write('\n') # print sections for each command and subcommand - SpackArgparseRstWriter(documented_commands).write(parser, root=1) + SpackArgparseRstWriter(documented_commands, out).write(parser, root=1) @formatter -def names(args): - for cmd in spack.cmd.all_commands(): - print(cmd) +def names(args, out): + colify(spack.cmd.all_commands(), output=out) -def commands(parser, args): +def prepend_header(args, out): + if not args.header: + return + + with open(args.header) as header: + out.write(header.read()) - # Print to stdout - formatters[args.format](args) - return + +def commands(parser, args): + formatter = formatters[args.format] + + # check header first so we don't open out files unnecessarily + if args.header and not os.path.exists(args.header): + tty.die("No such file: '%s'" % args.header) + + # if we're updating an existing file, only write output if a command + # is newer than the file. + if args.update: + if os.path.exists(args.update): + files = [ + spack.cmd.get_module(command).__file__.rstrip('c') # pyc -> py + for command in spack.cmd.all_commands()] + last_update = os.path.getmtime(args.update) + if not any(os.path.getmtime(f) > last_update for f in files): + tty.msg('File is up to date: %s' % args.update) + return + + tty.msg('Updating file: %s' % args.update) + with open(args.update, 'w') as f: + prepend_header(args, f) + formatter(args, f) + + else: + prepend_header(args, sys.stdout) + formatter(args, sys.stdout) diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index e24bd9ac14..f0b80ea031 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -4,14 +4,14 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re +import pytest from llnl.util.argparsewriter import ArgparseWriter import spack.cmd import spack.main -from spack.main import SpackCommand -commands = SpackCommand('commands') +commands = spack.main.SpackCommand('commands') parser = spack.main.make_argument_parser() spack.main.add_all_commands(parser) @@ -49,3 +49,63 @@ def test_rst(): assert prog in out assert re.sub(r' ', '-', prog) in out Subcommands().write(parser) + + +def test_rst_with_input_files(tmpdir): + filename = tmpdir.join('file.rst') + with filename.open('w') as f: + f.write(''' +.. _cmd-spack-fetch: +cmd-spack-list: +.. _cmd-spack-stage: +_cmd-spack-install: +.. _cmd-spack-patch: +''') + + out = commands('--format=rst', str(filename)) + for name in ['fetch', 'stage', 'patch']: + assert (':ref:`More documentation `' % name) in out + + for name in ['list', 'install']: + assert (':ref:`More documentation `' % name) not in out + + +def test_rst_with_header(tmpdir): + fake_header = 'this is a header!\n\n' + + filename = tmpdir.join('header.txt') + with filename.open('w') as f: + f.write(fake_header) + + out = commands('--format=rst', '--header', str(filename)) + assert out.startswith(fake_header) + + with pytest.raises(spack.main.SpackCommandError): + commands('--format=rst', '--header', 'asdfjhkf') + + +def test_rst_update(tmpdir): + update_file = tmpdir.join('output') + + # not yet created when commands is run + commands('--update', str(update_file)) + assert update_file.exists() + with update_file.open() as f: + assert f.read() + + # created but older than commands + with update_file.open('w') as f: + f.write('empty\n') + update_file.setmtime(0) + commands('--update', str(update_file)) + assert update_file.exists() + with update_file.open() as f: + assert f.read() != 'empty\n' + + # newer than commands + with update_file.open('w') as f: + f.write('empty\n') + commands('--update', str(update_file)) + assert update_file.exists() + with update_file.open() as f: + assert f.read() == 'empty\n' -- cgit v1.2.3-70-g09d2