From a18a0e7a47ca20c9745e2dfc12f7dbc1f47253b1 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 23 Dec 2021 13:45:06 -0800 Subject: commands: add `spack pkg source` and `spack pkg hash` To make it easier to see how package hashes change and how they are computed, add two commands: * `spack pkg source `: dumps source code for a package to the terminal * `spack pkg source --canonical `: dumps canonicalized source code for a package to the terminal. It strips comments, directives, and known-unused multimethods from the package. It is used to generate package hashes. * `spack pkg hash `: This gives the package hash for a particular spec. It is generated from the canonical source code for the spec. - [x] `add spack pkg source` and `spack pkg hash` - [x] add tests - [x] fix bug in multimethod resolution with boolean `@when` values Co-authored-by: Greg Becker --- lib/spack/spack/cmd/pkg.py | 58 ++++++++++++++++++++++++++++++---- lib/spack/spack/test/cmd/pkg.py | 60 ++++++++++++++++++++++++++++++++++++ lib/spack/spack/util/package_hash.py | 17 +++++++++- 3 files changed, 128 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index 8440cbd82c..eda32d98fc 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -7,6 +7,7 @@ from __future__ import print_function import os import re +import sys import llnl.util.tty as tty from llnl.util.filesystem import working_dir @@ -16,6 +17,7 @@ import spack.cmd import spack.cmd.common.arguments as arguments import spack.paths import spack.repo +import spack.util.package_hash as ph from spack.util.executable import which description = "query packages associated with particular git revisions" @@ -70,6 +72,15 @@ def setup_parser(subparser): 'rev2', nargs='?', default='HEAD', help="revision to compare to rev1 (default is HEAD)") + source_parser = sp.add_parser('source', help=pkg_source.__doc__) + source_parser.add_argument( + '-c', '--canonical', action='store_true', default=False, + help="dump canonical source as used by package hash.") + arguments.add_common_arguments(source_parser, ['spec']) + + hash_parser = sp.add_parser('hash', help=pkg_hash.__doc__) + arguments.add_common_arguments(hash_parser, ['spec']) + def packages_path(): """Get the test repo if it is active, otherwise the builtin repo.""" @@ -201,14 +212,49 @@ def pkg_changed(args): colify(sorted(packages)) +def pkg_source(args): + """dump source code for a package""" + specs = spack.cmd.parse_specs(args.spec, concretize=False) + if len(specs) != 1: + tty.die("spack pkg source requires exactly one spec") + + spec = specs[0] + filename = spack.repo.path.filename_for_package_name(spec.name) + + # regular source dump -- just get the package and print its contents + if args.canonical: + message = "Canonical source for %s:" % filename + content = ph.canonical_source(spec) + else: + message = "Source for %s:" % filename + with open(filename) as f: + content = f.read() + + if sys.stdout.isatty(): + tty.msg(message) + sys.stdout.write(content) + + +def pkg_hash(args): + """dump canonical source code hash for a package spec""" + specs = spack.cmd.parse_specs(args.spec, concretize=False) + + for spec in specs: + print(ph.package_hash(spec)) + + def pkg(parser, args): if not spack.cmd.spack_is_git_repo(): tty.die("This spack is not a git clone. Can't use 'spack pkg'") - action = {'add': pkg_add, - 'diff': pkg_diff, - 'list': pkg_list, - 'removed': pkg_removed, - 'added': pkg_added, - 'changed': pkg_changed} + action = { + 'add': pkg_add, + 'diff': pkg_diff, + 'list': pkg_list, + 'removed': pkg_removed, + 'added': pkg_added, + 'changed': pkg_changed, + 'source': pkg_source, + 'hash': pkg_hash, + } action[args.pkg_command](args) diff --git a/lib/spack/spack/test/cmd/pkg.py b/lib/spack/spack/test/cmd/pkg.py index 4036a33f95..54a6821677 100644 --- a/lib/spack/spack/test/cmd/pkg.py +++ b/lib/spack/spack/test/cmd/pkg.py @@ -236,3 +236,63 @@ def test_pkg_fails_when_not_git_repo(monkeypatch): monkeypatch.setattr(spack.cmd, 'spack_is_git_repo', lambda: False) with pytest.raises(spack.main.SpackCommandError): pkg('added') + + +def test_pkg_source_requires_one_arg(mock_packages): + with pytest.raises(spack.main.SpackCommandError): + pkg("source", "a", "b") + + with pytest.raises(spack.main.SpackCommandError): + pkg("source", "--canonical", "a", "b") + + +def test_pkg_source(mock_packages): + fake_source = pkg("source", "fake") + + fake_file = spack.repo.path.filename_for_package_name("fake") + with open(fake_file) as f: + contents = f.read() + assert fake_source == contents + + +def test_pkg_canonical_source(mock_packages): + source = pkg("source", "multimethod") + assert "@when('@2.0')" in source + assert "Check that multimethods work with boolean values" in source + + canonical_1 = pkg("source", "--canonical", "multimethod@1.0") + assert "@when" not in canonical_1 + assert "should_not_be_reached by diamond inheritance test" not in canonical_1 + assert "return 'base@1.0'" in canonical_1 + assert "return 'base@2.0'" not in canonical_1 + assert "return 'first_parent'" not in canonical_1 + assert "'should_not_be_reached by diamond inheritance test'" not in canonical_1 + + canonical_2 = pkg("source", "--canonical", "multimethod@2.0") + assert "@when" not in canonical_2 + assert "return 'base@1.0'" not in canonical_2 + assert "return 'base@2.0'" in canonical_2 + assert "return 'first_parent'" in canonical_2 + assert "'should_not_be_reached by diamond inheritance test'" not in canonical_2 + + canonical_3 = pkg("source", "--canonical", "multimethod@3.0") + assert "@when" not in canonical_3 + assert "return 'base@1.0'" not in canonical_3 + assert "return 'base@2.0'" not in canonical_3 + assert "return 'first_parent'" not in canonical_3 + assert "'should_not_be_reached by diamond inheritance test'" not in canonical_3 + + canonical_4 = pkg("source", "--canonical", "multimethod@4.0") + assert "@when" not in canonical_4 + assert "return 'base@1.0'" not in canonical_4 + assert "return 'base@2.0'" not in canonical_4 + assert "return 'first_parent'" not in canonical_4 + assert "'should_not_be_reached by diamond inheritance test'" in canonical_4 + + +def test_pkg_hash(mock_packages): + output = pkg("hash", "a", "b").strip().split() + assert len(output) == 2 and all(len(elt) == 32 for elt in output) + + output = pkg("hash", "multimethod").strip().split() + assert len(output) == 1 and all(len(elt) == 32 for elt in output) diff --git a/lib/spack/spack/util/package_hash.py b/lib/spack/spack/util/package_hash.py index 877dd7e458..d2b9055579 100644 --- a/lib/spack/spack/util/package_hash.py +++ b/lib/spack/spack/util/package_hash.py @@ -96,7 +96,22 @@ class TagMultiMethods(ast.NodeVisitor): try: # evaluate spec condition for any when's cond = dec.args[0].s - conditions.append(self.spec.satisfies(cond, strict=True)) + + # Boolean literals come through like this + if isinstance(cond, bool): + conditions.append(cond) + continue + + # otherwise try to make a spec + try: + cond_spec = spack.spec.Spec(cond) + except Exception: + # Spec parsing failed -- we don't know what this is. + conditions.append(None) + else: + # Check statically whether spec satisfies the condition + conditions.append(self.spec.satisfies(cond_spec, strict=True)) + except AttributeError: # In this case the condition for the 'when' decorator is # not a string literal (for example it may be a Python -- cgit v1.2.3-70-g09d2