From 3340d586c4743f6ee95367c2ffb0056f799c0987 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 20 May 2019 14:25:03 -0700 Subject: commands: add --update option to `spack list` - Add a `--update FILE` option to `spack list` - Output is written to the file only if any package is newer than the file - Simplify the code in docs/conf.py using this new option --- lib/spack/docs/conf.py | 4 +- lib/spack/spack/cmd/list.py | 216 ++++++++++++++++++++++----------------- lib/spack/spack/repo.py | 12 +++ lib/spack/spack/test/cmd/list.py | 27 +++++ lib/spack/spack/test/repo.py | 8 ++ 5 files changed, 171 insertions(+), 96 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index 4af4e13035..bb460e40aa 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -54,8 +54,8 @@ os.environ['COLIFY_SIZE'] = '25x120' os.environ['COLUMNS'] = '120' # Generate full package list if needed -subprocess.Popen( - ['spack', 'list', '--format=html', '--update=package_list.html']) +subprocess.call([ + 'spack', 'list', '--format=html', '--update=package_list.html']) # Generate a command index if an update is needed subprocess.call([ diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index c93e39ad16..bba1fc5ceb 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -9,6 +9,7 @@ from __future__ import division import argparse import cgi import fnmatch +import os import re import sys import math @@ -46,6 +47,9 @@ def setup_parser(subparser): subparser.add_argument( '--format', default='name_only', choices=formatters, help='format to be used to print the output [default: name_only]') + subparser.add_argument( + '--update', metavar='FILE', default=None, action='store', + help='write output to the specified file, if any package is newer') arguments.add_common_arguments(subparser, ['tags']) @@ -90,11 +94,11 @@ def filter_by_name(pkgs, args): @formatter -def name_only(pkgs): +def name_only(pkgs, out): indent = 0 - if sys.stdout.isatty(): + if out.isatty(): tty.msg("%d packages." % len(pkgs)) - colify(pkgs, indent=indent) + colify(pkgs, indent=indent, output=out) def github_url(pkg): @@ -123,64 +127,69 @@ def rows_for_ncols(elts, ncols): @formatter -def rst(pkg_names): +def rst(pkg_names, out): """Print out information on all packages in restructured text.""" pkgs = [spack.repo.get(name) for name in pkg_names] - print('.. _package-list:') - print() - print('============') - print('Package List') - print('============') - print() - print('This is a list of things you can install using Spack. It is') - print('automatically generated based on the packages in the latest Spack') - print('release.') - print() - print('Spack currently has %d mainline packages:' % len(pkgs)) - print() - print(rst_table('`%s`_' % p for p in pkg_names)) - print() + out.write('.. _package-list:\n') + out.write('\n') + out.write('============\n') + out.write('Package List\n') + out.write('============\n') + out.write('\n') + out.write('This is a list of things you can install using Spack. It is\n') + out.write( + 'automatically generated based on the packages in the latest Spack\n') + out.write('release.\n') + out.write('\n') + out.write('Spack currently has %d mainline packages:\n' % len(pkgs)) + out.write('\n') + out.write(rst_table('`%s`_' % p for p in pkg_names)) + out.write('\n') + out.write('\n') # Output some text for each package. for pkg in pkgs: - print('-----') - print() - print('.. _%s:' % pkg.name) - print() + out.write('-----\n') + out.write('\n') + out.write('.. _%s:\n' % pkg.name) + out.write('\n') # Must be at least 2 long, breaks for single letter packages like R. - print('-' * max(len(pkg.name), 2)) - print(pkg.name) - print('-' * max(len(pkg.name), 2)) - print() - print('Homepage:') - print(' * `%s <%s>`__' % (cgi.escape(pkg.homepage), pkg.homepage)) - print() - print('Spack package:') - print(' * `%s/package.py <%s>`__' % (pkg.name, github_url(pkg))) - print() + out.write('-' * max(len(pkg.name), 2)) + out.write('\n') + out.write(pkg.name) + out.write('\n') + out.write('-' * max(len(pkg.name), 2)) + out.write('\n\n') + out.write('Homepage:\n') + out.write( + ' * `%s <%s>`__\n' % (cgi.escape(pkg.homepage), pkg.homepage)) + out.write('\n') + out.write('Spack package:\n') + out.write(' * `%s/package.py <%s>`__\n' % (pkg.name, github_url(pkg))) + out.write('\n') if pkg.versions: - print('Versions:') - print(' ' + ', '.join(str(v) for v in - reversed(sorted(pkg.versions)))) - print() + out.write('Versions:\n') + out.write(' ' + ', '.join(str(v) for v in + reversed(sorted(pkg.versions)))) + out.write('\n\n') for deptype in spack.dependency.all_deptypes: deps = pkg.dependencies_of_type(deptype) if deps: - print('%s Dependencies' % deptype.capitalize()) - print(' ' + ', '.join('%s_' % d if d in pkg_names - else d for d in deps)) - print() + out.write('%s Dependencies\n' % deptype.capitalize()) + out.write(' ' + ', '.join('%s_' % d if d in pkg_names + else d for d in deps)) + out.write('\n\n') - print('Description:') - print(pkg.format_doc(indent=2)) - print() + out.write('Description:\n') + out.write(pkg.format_doc(indent=2)) + out.write('\n\n') @formatter -def html(pkg_names): +def html(pkg_names, out): """Print out information on all packages in Sphinx HTML. This is intended to be inlined directly into Sphinx documentation. @@ -199,83 +208,90 @@ def html(pkg_names): def head(n, span_id, title, anchor=None): if anchor is None: anchor = title - print(('' - '

%s' - '

') % (span_id, title, anchor)) + out.write(('' + '

%s' + '

\n') % (span_id, title, anchor)) # Start with the number of packages, skipping the title and intro # blurb, which we maintain in the RST file. - print('

') - print('Spack currently has %d mainline packages:' % len(pkgs)) - print('

') + out.write('

\n') + out.write('Spack currently has %d mainline packages:\n' % len(pkgs)) + out.write('

\n') # Table of links to all packages - print('') - print('') + out.write('
\n') + out.write('\n') for i, row in enumerate(rows_for_ncols(pkg_names, 3)): - print('' if i % 2 == 0 else - '') + out.write('\n' if i % 2 == 0 else + '\n') for name in row: - print('' - % (name, name)) - print('') - print('') - print('') - print('
') - print('%s
') - print('
') + out.write('\n') + out.write('%s\n' + % (name, name)) + out.write('\n') + out.write('\n') + out.write('\n') + out.write('\n') + out.write('
\n') # Output some text for each package. for pkg in pkgs: - print('
' % pkg.name) + out.write('
\n' % pkg.name) head(2, span_id, pkg.name) span_id += 1 - print('
') + out.write('
\n') - print('
Homepage:
') - print('
    ') - print(('
  • ' - '%s' - '
    • ') % (pkg.homepage, cgi.escape(pkg.homepage))) - print('
') + out.write('
Homepage:
\n') + out.write('
    \n') + out.write(('
  • ' + '%s' + '
    • \n') % (pkg.homepage, cgi.escape(pkg.homepage))) + out.write('
\n') - print('
Spack package:
') - print('
    ') - print(('
  • ' - '%s/package.py' - '
    • ') % (github_url(pkg), pkg.name)) - print('
') + out.write('
Spack package:
\n') + out.write('
    \n') + out.write(('
  • ' + '%s/package.py' + '
    • \n') % (github_url(pkg), pkg.name)) + out.write('
\n') if pkg.versions: - print('
Versions:
') - print('
') - print(', '.join(str(v) for v in reversed(sorted(pkg.versions)))) - print('
') + out.write('
Versions:
\n') + out.write('
\n') + out.write(', '.join( + str(v) for v in reversed(sorted(pkg.versions)))) + out.write('\n') + out.write('
\n') for deptype in spack.dependency.all_deptypes: deps = pkg.dependencies_of_type(deptype) if deps: - print('
%s Dependencies:
' % deptype.capitalize()) - print('
') - print(', '.join( + out.write('
%s Dependencies:
\n' % deptype.capitalize()) + out.write('
\n') + out.write(', '.join( d if d not in pkg_names else '%s' % (d, d) for d in deps)) - print('
') + out.write('\n') + out.write('\n') - print('
Description:
') - print('
') - print(cgi.escape(pkg.format_doc(indent=2))) - print('
') - print('
') + out.write('
Description:
\n') + out.write('
\n') + out.write(cgi.escape(pkg.format_doc(indent=2))) + out.write('\n') + out.write('
\n') + out.write('
\n') - print('
') - print('
') + out.write('
\n') + out.write('
\n') def list(parser, args): + # retrieve the formatter to use from args + formatter = formatters[args.format] + # Retrieve the names of all the packages pkgs = set(spack.repo.all_package_names()) # Filter the set appropriately @@ -288,5 +304,17 @@ def list(parser, args): sorted_packages = set(sorted_packages) & packages_with_tags sorted_packages = sorted(sorted_packages) - # Print to stdout - formatters[args.format](sorted_packages) + if args.update: + # change output stream if user asked for update + if os.path.exists(args.update): + if os.path.getmtime(args.update) > spack.repo.path.last_mtime(): + tty.msg('File is up to date: %s' % args.update) + return + + tty.msg('Updating file: %s' % args.update) + with open(args.update, 'w') as f: + formatter(sorted_packages, f) + + else: + # Print to stdout + formatter(sorted_packages, sys.stdout) diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py index c90f0f9ad4..7a2e235a4e 100644 --- a/lib/spack/spack/repo.py +++ b/lib/spack/spack/repo.py @@ -183,6 +183,10 @@ class FastPackageChecker(Mapping): return cache + def last_mtime(self): + return max( + sinfo.st_mtime for sinfo in self._packages_to_stats.values()) + def __getitem__(self, item): return self._packages_to_stats[item] @@ -607,6 +611,10 @@ class RepoPath(object): sys.modules[fullname] = module return module + def last_mtime(self): + """Time a package file in this repo was last updated.""" + return max(repo.last_mtime() for repo in self.repos) + def repo_for_pkg(self, spec): """Given a spec, get the repository for its package.""" # We don't @_autospec this function b/c it's called very frequently @@ -1018,6 +1026,10 @@ class Repo(object): """Whether a package with the supplied name exists.""" return pkg_name in self._pkg_checker + def last_mtime(self): + """Time a package file in this repo was last updated.""" + return self._pkg_checker.last_mtime() + def is_virtual(self, pkg_name): """True if the package with this name is virtual, False otherwise.""" return self.provider_index.contains(pkg_name) diff --git a/lib/spack/spack/test/cmd/list.py b/lib/spack/spack/test/cmd/list.py index 6e2f18809a..2becd2e148 100644 --- a/lib/spack/spack/test/cmd/list.py +++ b/lib/spack/spack/test/cmd/list.py @@ -59,3 +59,30 @@ def test_list_format_html(): assert '
' in output assert '

hdf5' in output + + +def test_list_update(tmpdir): + update_file = tmpdir.join('output') + + # not yet created when list is run + list('--update', str(update_file)) + assert update_file.exists() + with update_file.open() as f: + assert f.read() + + # created but older than any package + with update_file.open('w') as f: + f.write('empty\n') + update_file.setmtime(0) + list('--update', str(update_file)) + assert update_file.exists() + with update_file.open() as f: + assert f.read() != 'empty\n' + + # newer than any packages + with update_file.open('w') as f: + f.write('empty\n') + list('--update', str(update_file)) + assert update_file.exists() + with update_file.open() as f: + assert f.read() == 'empty\n' diff --git a/lib/spack/spack/test/repo.py b/lib/spack/spack/test/repo.py index 8752caa88c..8f37bc6754 100644 --- a/lib/spack/spack/test/repo.py +++ b/lib/spack/spack/test/repo.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os import pytest import spack.repo @@ -56,3 +57,10 @@ def test_repo_pkg_with_unknown_namespace(repo_for_test): def test_repo_unknown_pkg(repo_for_test): with pytest.raises(spack.repo.UnknownPackageError): repo_for_test.get('builtin.mock.nonexistentpackage') + + +@pytest.mark.maybeslow +def test_repo_last_mtime(): + latest_mtime = max(os.path.getmtime(p.module.__file__) + for p in spack.repo.path.all_packages()) + assert spack.repo.path.last_mtime() == latest_mtime -- cgit v1.2.3-60-g2f50