From 50ca4979e1cf9d02ab748ef9e0c599ba56ae089d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 16 Jan 2018 00:12:11 -0800 Subject: Speed up doc builds with spack list --format=html (#6945) - Generating the HTML from for >2300 packages from RST in Sphinx seems to take forever. - Add an option to `spack list` to generate straight HTML instead. - This reduces the doc build time to about a minute (from 5 minutes on a mac laptop). --- lib/spack/docs/.gitignore | 2 +- lib/spack/docs/Makefile | 2 +- lib/spack/docs/conf.py | 5 +- lib/spack/docs/package_list.rst | 12 ++++ lib/spack/spack/cmd/list.py | 138 ++++++++++++++++++++++++++++++++++++---- 5 files changed, 141 insertions(+), 18 deletions(-) create mode 100644 lib/spack/docs/package_list.rst diff --git a/lib/spack/docs/.gitignore b/lib/spack/docs/.gitignore index 9afb658706..0b6258af50 100644 --- a/lib/spack/docs/.gitignore +++ b/lib/spack/docs/.gitignore @@ -1,4 +1,4 @@ -package_list.rst +package_list.html command_index.rst spack*.rst llnl*.rst diff --git a/lib/spack/docs/Makefile b/lib/spack/docs/Makefile index 3503794021..9b07a39fbb 100644 --- a/lib/spack/docs/Makefile +++ b/lib/spack/docs/Makefile @@ -83,7 +83,7 @@ help: @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: - -rm -f package_list.rst command_index.rst + -rm -f command_index.rst -rm -rf $(BUILDDIR)/* $(APIDOC_FILES) html: diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index 94d301f6d9..2d38211b4c 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -68,9 +68,10 @@ os.environ['COLIFY_SIZE'] = '25x120' # # Generate package list using spack command # -with open('package_list.rst', 'w') as plist_file: +with open('package_list.html', 'w') as plist_file: subprocess.Popen( - [spack_root + '/bin/spack', 'list', '--format=rst'], stdout=plist_file) + [spack_root + '/bin/spack', 'list', '--format=html'], + stdout=plist_file) # # Find all the `cmd-spack-*` references and add them to a command index diff --git a/lib/spack/docs/package_list.rst b/lib/spack/docs/package_list.rst new file mode 100644 index 0000000000..2deb5bfc0f --- /dev/null +++ b/lib/spack/docs/package_list.rst @@ -0,0 +1,12 @@ +.. _package-list: + +============ +Package List +============ + +This is a list of things you can install using Spack. It is +automatically generated based on the packages in the latest Spack +release. + +.. raw:: html + :file: package_list.html diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index 3549de361a..85d94a3fce 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -23,12 +23,14 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from __future__ import print_function +from __future__ import division import argparse import cgi import fnmatch import re import sys +import math from six import StringIO @@ -113,23 +115,35 @@ def name_only(pkgs): colify(pkgs, indent=indent) -@formatter -def rst(pkgs): - """Print out information on all packages in restructured text.""" +def github_url(pkg): + """Link to a package file on github.""" + url = 'https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/{0}/package.py' + return url.format(pkg.name) + + +def rst_table(elts): + """Print out a RST-style table.""" + cols = StringIO() + ncol, widths = colify(elts, output=cols, tty=True) + header = ' '.join('=' * (w - 1) for w in widths) + return '%s\n%s%s' % (header, cols.getvalue(), header) - def github_url(pkg): - """Link to a package file on github.""" - url = 'https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/{0}/package.py' - return url.format(pkg.name) - def rst_table(elts): - """Print out a RST-style table.""" - cols = StringIO() - ncol, widths = colify(elts, output=cols, tty=True) - header = ' '.join('=' * (w - 1) for w in widths) - return '%s\n%s%s' % (header, cols.getvalue(), header) +def rows_for_ncols(elts, ncols): + """Print out rows in a table with ncols of elts laid out vertically.""" + clen = int(math.ceil(len(elts) / ncols)) + for r in range(clen): + row = [] + for c in range(ncols): + i = c * clen + r + row.append(elts[i] if i < len(elts) else None) + yield row + + +@formatter +def rst(pkg_names): + """Print out information on all packages in restructured text.""" - pkg_names = pkgs pkgs = [spack.repo.get(name) for name in pkg_names] print('.. _package-list:') @@ -183,6 +197,102 @@ def rst(pkgs): print() +@formatter +def html(pkg_names): + """Print out information on all packages in Sphinx HTML. + + This is intended to be inlined directly into Sphinx documentation. + We write HTML instead of RST for speed; generating RST from *all* + packages causes the Sphinx build to take forever. Including this as + raw HTML is much faster. + """ + + # Read in all packages + pkgs = [spack.repo.get(name) for name in pkg_names] + + # Start at 2 because the title of the page from Sphinx is id1. + span_id = 2 + + # HTML header with an increasing id span + def head(n, span_id, title, anchor=None): + if anchor is None: + anchor = title + print(('' + '

%s' + '

') % (span_id, title, anchor)) + + # Start with the number of packages, skipping the title and intro + # blurb, which we maintain in the RST file. + print('

') + print('Spack currently has %d mainline packages:' % len(pkgs)) + print('

') + + # Table of links to all packages + print('') + print('') + for i, row in enumerate(rows_for_ncols(pkg_names, 3)): + print('' if i % 2 == 0 else + '') + for name in row: + print('' + % (name, name)) + print('') + print('') + print('') + print('
') + print('%s
') + print('
') + + # Output some text for each package. + for pkg in pkgs: + print('
' % pkg.name) + head(2, span_id, pkg.name) + span_id += 1 + + print('
') + + print('
Homepage:
') + print('
    ') + print(('
  • ' + '%s' + '
    • ') % (pkg.homepage, cgi.escape(pkg.homepage))) + print('
') + + print('
Spack package:
') + print('
    ') + print(('
  • ' + '%s/package.py' + '
    • ') % (github_url(pkg), pkg.name)) + print('
') + + if pkg.versions: + print('
Versions:
') + print('
') + print(', '.join(str(v) for v in reversed(sorted(pkg.versions)))) + print('
') + + for deptype in spack.all_deptypes: + deps = pkg.dependencies_of_type(deptype) + if deps: + print('
%s Dependencies:
' % deptype.capitalize()) + print('
') + print(', '.join( + d if d not in pkg_names else + '%s' % (d, d) + for d in deps)) + print('
') + + print('
Description:
') + print('
') + print(cgi.escape(pkg.format_doc(indent=2))) + print('
') + print('
') + + print('
') + print('
') + + def list(parser, args): # Retrieve the names of all the packages pkgs = set(spack.repo.all_package_names()) -- cgit v1.2.3-70-g09d2