diff options
| -rw-r--r-- | lib/spack/docs/basic_usage.rst | 130 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/diff.py | 225 | ||||
| -rw-r--r-- | lib/spack/spack/solver/asp.py | 9 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/diff.py | 79 | ||||
| -rwxr-xr-x | share/spack/spack-completion.bash | 11 |
5 files changed, 448 insertions, 6 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index cf1fc28614..3503704319 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -695,6 +695,136 @@ structured the way you want: } +^^^^^^^^^^^^^^ +``spack diff`` +^^^^^^^^^^^^^^ + +It's often the case that you have two versions of a spec that you need to +disambiguate. Let's say that we've installed two variants of zlib, one with +and one without the optimize variant: + +.. code-block:: console + + $ spack install zlib + $ spack install zlib -optimize + +When we do ``spack find`` we see the two versions. + +.. code-block:: console + + $ spack find zlib + ==> 2 installed packages + -- linux-ubuntu20.04-skylake / gcc@9.3.0 ------------------------ + zlib@1.2.11 zlib@1.2.11 + + +Let's now say that we want to uninstall zlib. We run the command, and hit a problem +real quickly since we have two! + +.. code-block:: console + + $ spack uninstall zlib + ==> Error: zlib matches multiple packages: + + -- linux-ubuntu20.04-skylake / gcc@9.3.0 ------------------------ + efzjziy zlib@1.2.11 sl7m27m zlib@1.2.11 + + ==> Error: You can either: + a) use a more specific spec, or + b) specify the spec by its hash (e.g. `spack uninstall /hash`), or + c) use `spack uninstall --all` to uninstall ALL matching specs. + +Oh no! We can see from the above that we have two different versions of zlib installed, +and the only difference between the two is the hash. This is a good use case for +``spack diff``, which can easily show us the "diff" or set difference +between properties for two packages. Let's try it out. +Since the only difference we see in the ``spack find`` view is the hash, let's use +``spack diff`` to look for more detail. We will provide the two hashes: + +.. code-block::console + + $ spack diff /efzjziy /sl7m27m + ==> Warning: This interface is subject to change. + + --- zlib@1.2.11efzjziyc3dmb5h5u5azsthgbgog5mj7g + +++ zlib@1.2.11sl7m27mzkbejtkrajigj3a3m37ygv4u2 + @@ variant_value @@ + - zlib optimize bool(False) + + zlib optimize bool(True) + + +The output is colored, and written in the style of a git diff. This means that you +can copy paste it into a GitHub markdown as a code block with language "diff" and it +will render nicely! Here is an example: + +.. code-block::markdown + + ```diff + --- zlib@1.2.11/efzjziyc3dmb5h5u5azsthgbgog5mj7g + +++ zlib@1.2.11/sl7m27mzkbejtkrajigj3a3m37ygv4u2 + @@ variant_value @@ + - zlib optimize bool(False) + + zlib optimize bool(True) + ``` + +Awesome! Now let's read the diff. It tells us that our first zlib was built without optimize (False) +and the second was built with optimize (True). You can't see it in the docs here, but +the output above is also colored based on the content being an addition (+) or subtraction (-). + +This is a small example, but there are actually several kinds of differences that you can view, a variant value +being just one of them. The first package that you provide (A) +being diffed against B means that we see what is added to B but not in A (green) and what is present in A that is +removed in B (red). Here is another example with an additional difference type, ``VERSION``: + +.. code-block::console + + $ spack diff python@2.7.8 python@3.8.11 + ==> Warning: This interface is subject to change. + + --- python@2.7.8/tsxdi6gl4lihp25qrm4d6nys3nypufbf + +++ python@3.8.11/yjtseru4nbpllbaxb46q7wfkyxbuvzxx + @@ variant_value @@ + - python patches a8c52415a8b03c0e5f28b5d52ae498f7a7e602007db2b9554df28cd5685839b8 + + python patches 0d98e93189bc278fbc37a50ed7f183bd8aaf249a8e1670a465f0db6bb4f8cf87 + @@ version @@ + - openssl Version(1.0.2u) + + openssl Version(1.1.1k) + - python Version(2.7.8) + + python Version(3.8.11) + +Let's say that we were only interested in one kind of attribute above, versions! +We can ask the command to only output this attribute. To do this, you'd add +the ``-a`` for attribute parameter, which defaults to all. +Here is how you would filter to show just versions: + + +.. code-block:: console + + $ spack diff -a version python@2.7.8 python@3.8.11 + ==> Warning: This interface is subject to change. + + --- python@2.7.8/tsxdi6gl4lihp25qrm4d6nys3nypufbf + +++ python@3.8.11/yjtseru4nbpllbaxb46q7wfkyxbuvzxx + @@ version @@ + - openssl Version(1.0.2u) + + openssl Version(1.1.1k) + - python Version(2.7.8) + + python Version(3.8.11) + +And you can add as many attributes as you'd like with multiple `-a`. +Finally, if you want to view the data as json (and possibly pipe into an output file) +just add ``--json``: + + +.. code-block:: console + + $ spack diff --json python@2.7.8 python@3.8.11 + + +This data will be much longer because along with the differences for A vs. B and +B vs. A, we also capture the intersection. + + ------------------------ Using installed packages ------------------------ diff --git a/lib/spack/spack/cmd/diff.py b/lib/spack/spack/cmd/diff.py new file mode 100644 index 0000000000..ff1f9a41d6 --- /dev/null +++ b/lib/spack/spack/cmd/diff.py @@ -0,0 +1,225 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +import sys + +import llnl.util.tty as tty +import llnl.util.tty.color as color + +import spack.cmd +import spack.cmd.common.arguments as arguments +import spack.environment as ev +import spack.solver.asp as asp +import spack.util.environment +import spack.util.spack_json as sjson + +description = "compare two specs" +section = "basic" +level = "long" + + +def setup_parser(subparser): + arguments.add_common_arguments( + subparser, ['specs']) + + subparser.add_argument( + '--json', + action='store_true', + default=False, + dest='dump_json', + help="Dump json output instead of pretty printing." + ) + subparser.add_argument( + '--first', + action='store_true', + default=False, + dest='load_first', + help="load the first match if multiple packages match the spec" + ) + subparser.add_argument( + '-a', '--attribute', + action='append', + help="select the attributes to show (defaults to all)" + ) + + +def boldblue(string): + """ + Make a header string bold and blue we can easily see it + """ + return color.colorize("@*b{%s}" % string) + + +def green(string): + return color.colorize("@G{%s}" % string) + + +def red(string): + return color.colorize("@R{%s}" % string) + + +def compare_specs(a, b, to_string=False, colorful=True): + """ + Generate a comparison, including diffs (for each side) and an intersection. + + We can either print the result to the console, or parse + into a json object for the user to save. We return an object that shows + the differences, intersection, and names for a pair of specs a and b. + + Arguments: + a (spack.spec.Spec): the first spec to compare + b (spack.spec.Spec): the second spec to compare + a_name (str): the name of spec a + b_name (str): the name of spec b + to_string (bool): return an object that can be json dumped + colorful (bool): do not format the names for the console + """ + # Prepare a solver setup to parse differences + setup = asp.SpackSolverSetup() + + a_facts = set(to_tuple(t) for t in setup.spec_clauses(a, body=True)) + b_facts = set(to_tuple(t) for t in setup.spec_clauses(b, body=True)) + + # We want to present them to the user as simple key: values + intersect = list(a_facts.intersection(b_facts)) + spec1_not_spec2 = list(a_facts.difference(b_facts)) + spec2_not_spec1 = list(b_facts.difference(a_facts)) + + # Format the spec names to be colored + fmt = "{name}{@version}{/hash}" + a_name = a.format(fmt, color=color.get_color_when()) + b_name = b.format(fmt, color=color.get_color_when()) + + # We want to show what is the same, and then difference for each + return { + "intersect": flatten(intersect) if to_string else intersect, + "a_not_b": flatten(spec1_not_spec2) if to_string else spec1_not_spec2, + "b_not_a": flatten(spec2_not_spec1) if to_string else spec2_not_spec1, + "a_name": a_name if colorful else a.format("{name}{@version}{/hash}"), + "b_name": b_name if colorful else b.format("{name}{@version}{/hash}") + } + + +def to_tuple(asp_function): + """ + Prepare tuples of objects. + + If we need to save to json, convert to strings + See https://gist.github.com/tgamblin/83eba3c6d27f90d9fa3afebfc049ceaf + """ + args = [] + for arg in asp_function.args: + if isinstance(arg, str): + args.append(arg) + continue + args.append("%s(%s)" % (type(arg).__name__, str(arg))) + return tuple([asp_function.name] + args) + + +def flatten(tuple_list): + """ + Given a list of tuples, convert into a list of key: value tuples. + + We are squashing whatever is after the first index into one string for + easier parsing in the interface + """ + updated = [] + for item in tuple_list: + updated.append([item[0], " ".join(item[1:])]) + return updated + + +def print_difference(c, attributes="all", out=None): + """ + Print the difference. + + Given a diffset for A and a diffset for B, print red/green diffs to show + the differences. + """ + # Default to standard out unless another stream is provided + out = out or sys.stdout + + A = c['b_not_a'] + B = c['a_not_b'] + + out.write(red("--- %s\n" % c["a_name"])) + out.write(green("+++ %s\n" % c["b_name"])) + + # Cut out early if we don't have any differences! + if not A and not B: + print("No differences\n") + return + + def group_by_type(diffset): + grouped = {} + for entry in diffset: + if entry[0] not in grouped: + grouped[entry[0]] = [] + grouped[entry[0]].append(entry[1]) + + # Sort by second value to make comparison slightly closer + for key, values in grouped.items(): + values.sort() + return grouped + + A = group_by_type(A) + B = group_by_type(B) + + # print a directionally relevant diff + keys = list(A) + list(B) + + category = None + for key in keys: + if "all" not in attributes and key not in attributes: + continue + + # Write the attribute, B is subtraction A is addition + subtraction = [] if key not in B else B[key] + addition = [] if key not in A else A[key] + + # Bail out early if we don't have any entries + if not subtraction and not addition: + continue + + # If we have a new category, create a new section + if category != key: + category = key + + # print category in bold, colorized + out.write(boldblue("@@ %s @@\n" % category)) + + # Print subtractions first + while subtraction: + out.write(red("- %s\n" % subtraction.pop(0))) + if addition: + out.write(green("+ %s\n" % addition.pop(0))) + + # Any additions left? + while addition: + out.write(green("+ %s\n" % addition.pop(0))) + + +def diff(parser, args): + env = ev.get_env(args, 'diff') + + if len(args.specs) != 2: + tty.die("You must provide two specs to diff.") + + specs = [spack.cmd.disambiguate_spec(spec, env, first=args.load_first) + for spec in spack.cmd.parse_specs(args.specs)] + + # Calculate the comparison (c) + c = compare_specs(specs[0], specs[1], to_string=True, + colorful=not args.dump_json) + + # Default to all attributes + attributes = args.attribute or ["all"] + + if args.dump_json: + print(sjson.dump(c)) + else: + tty.warn("This interface is subject to change.\n") + print_difference(c, attributes) diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index 2fb20d914a..2251183c55 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -96,7 +96,10 @@ def _id(thing): class AspFunction(AspObject): def __init__(self, name, args=None): self.name = name - self.args = [] if args is None else args + self.args = () if args is None else args + + def _cmp_key(self): + return (self.name, self.args) def __call__(self, *args): return AspFunction(self.name, args) @@ -112,10 +115,6 @@ class AspFunction(AspObject): return clingo.Function( self.name, [argify(arg) for arg in self.args], positive=positive) - def __getitem___(self, *args): - self.args[:] = args - return self - def __str__(self): return "%s(%s)" % ( self.name, ', '.join(str(_id(arg)) for arg in self.args)) diff --git a/lib/spack/spack/test/cmd/diff.py b/lib/spack/spack/test/cmd/diff.py new file mode 100644 index 0000000000..76ab8455b4 --- /dev/null +++ b/lib/spack/spack/test/cmd/diff.py @@ -0,0 +1,79 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import pytest + +import spack.cmd.diff +import spack.config +import spack.main +import spack.store +import spack.util.spack_json as sjson + +install = spack.main.SpackCommand('install') +diff = spack.main.SpackCommand('diff') + + +def test_diff(install_mockery, mock_fetch, mock_archive, mock_packages): + """Test that we can install two packages and diff them""" + + specA = spack.spec.Spec('mpileaks').concretized() + specB = spack.spec.Spec('mpileaks+debug').concretized() + + # Specs should be the same as themselves + c = spack.cmd.diff.compare_specs(specA, specA, to_string=True) + assert len(c['a_not_b']) == 0 + assert len(c['b_not_a']) == 0 + + # Calculate the comparison (c) + c = spack.cmd.diff.compare_specs(specA, specB, to_string=True) + assert len(c['a_not_b']) == 1 + assert len(c['b_not_a']) == 1 + assert c['a_not_b'][0] == ['variant_value', 'mpileaks debug bool(False)'] + assert c['b_not_a'][0] == ['variant_value', 'mpileaks debug bool(True)'] + + +def test_load_first(install_mockery, mock_fetch, mock_archive, mock_packages): + """Test with and without the --first option""" + install('mpileaks') + + # Only one version of mpileaks will work + diff('mpileaks', 'mpileaks') + + # 2 specs are required for a diff + with pytest.raises(spack.main.SpackCommandError): + diff('mpileaks') + with pytest.raises(spack.main.SpackCommandError): + diff('mpileaks', 'mpileaks', 'mpileaks') + + # Ensure they are the same + assert "No differences" in diff('mpileaks', 'mpileaks') + output = diff('--json', 'mpileaks', 'mpileaks') + result = sjson.load(output) + + assert len(result['a_not_b']) == 0 + assert len(result['b_not_a']) == 0 + + assert 'mpileaks' in result['a_name'] + assert 'mpileaks' in result['b_name'] + assert "intersect" in result and len(result['intersect']) > 50 + + # After we install another version, it should ask us to disambiguate + install('mpileaks+debug') + + # There are two versions of mpileaks + with pytest.raises(spack.main.SpackCommandError): + diff('mpileaks', 'mpileaks+debug') + + # But if we tell it to use the first, it won't try to disambiguate + assert "variant" in diff('--first', 'mpileaks', 'mpileaks+debug') + + # This matches them exactly + output = diff("--json", "mpileaks@2.3/ysubb76", "mpileaks@2.3/ft5qff3") + result = sjson.load(output) + + assert len(result['a_not_b']) == 1 + assert len(result['b_not_a']) == 1 + assert result['a_not_b'][0] == ['variant_value', 'mpileaks debug bool(False)'] + assert result['b_not_a'][0] == ['variant_value', 'mpileaks debug bool(True)'] diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index bf2a488a85..03e9770c37 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -333,7 +333,7 @@ _spack() { then SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars" else - SPACK_COMPREPLY="activate add analyze arch audit blame bootstrap build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config containerize create deactivate debug dependencies dependents deprecate dev-build develop docs edit env extensions external fetch find flake8 gc gpg graph help info install license list load location log-parse maintainers mark mirror module monitor patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style test test-env tutorial undevelop uninstall unit-test unload url verify versions view" + SPACK_COMPREPLY="activate add analyze arch audit blame bootstrap build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config containerize create deactivate debug dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find flake8 gc gpg graph help info install license list load location log-parse maintainers mark mirror module monitor patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style test test-env tutorial undevelop uninstall unit-test unload url verify versions view" fi } @@ -842,6 +842,15 @@ _spack_develop() { fi } +_spack_diff() { + if $list_options + then + SPACK_COMPREPLY="-h --help --json --first -a --attribute" + else + _all_packages + fi +} + _spack_docs() { SPACK_COMPREPLY="-h --help" } |