From 20367e472d780d4090c6343a9af2000d01997f8a Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Wed, 18 Nov 2020 12:20:56 +0100 Subject: cmd: add `spack mark` command (#16662) This adds a new `mark` command that can be used to mark packages as either explicitly or implicitly installed. Apart from fixing the package database after installing a dependency manually, it can be used to implement upgrade workflows as outlined in #13385. The following commands demonstrate how the `mark` and `gc` commands can be used to only keep the current version of a package installed: ```console $ spack install pkgA $ spack install pkgB $ git pull # Imagine new versions for pkgA and/or pkgB are introduced $ spack mark -i -a $ spack install pkgA $ spack install pkgB $ spack gc ``` If there is no new version for a package, `install` will simply mark it as explicitly installed and `gc` will not remove it. Co-authored-by: Greg Becker --- lib/spack/docs/basic_usage.rst | 96 ++++++++++++++++++++++++++++++ lib/spack/spack/cmd/__init__.py | 24 +++++--- lib/spack/spack/cmd/mark.py | 122 +++++++++++++++++++++++++++++++++++++++ lib/spack/spack/cmd/uninstall.py | 8 ++- lib/spack/spack/database.py | 18 ++++++ lib/spack/spack/installer.py | 27 ++------- lib/spack/spack/test/cmd/mark.py | 67 +++++++++++++++++++++ 7 files changed, 329 insertions(+), 33 deletions(-) create mode 100644 lib/spack/spack/cmd/mark.py create mode 100644 lib/spack/spack/test/cmd/mark.py (limited to 'lib') diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 1a73bd0339..f7def9d439 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -280,6 +280,102 @@ and removed everything that is not either: You can check :ref:`cmd-spack-find-metadata` to see how to query for explicitly installed packages or :ref:`dependency-types` for a more thorough treatment of dependency types. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Marking packages explicit or implicit +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, Spack will mark packages a user installs as explicitly installed, +while all of its dependencies will be marked as implicitly installed. Packages +can be marked manually as explicitly or implicitly installed by using +``spack mark``. This can be used in combination with ``spack gc`` to clean up +packages that are no longer required. + +.. code-block:: console + + $ spack install m4 + ==> 29005: Installing libsigsegv + [...] + ==> 29005: Installing m4 + [...] + + $ spack install m4 ^libsigsegv@2.11 + ==> 39798: Installing libsigsegv + [...] + ==> 39798: Installing m4 + [...] + + $ spack find -d + ==> 4 installed packages + -- linux-fedora32-haswell / gcc@10.1.1 -------------------------- + libsigsegv@2.11 + + libsigsegv@2.12 + + m4@1.4.18 + libsigsegv@2.12 + + m4@1.4.18 + libsigsegv@2.11 + + $ spack gc + ==> There are no unused specs. Spack's store is clean. + + $ spack mark -i m4 ^libsigsegv@2.11 + ==> m4@1.4.18 : marking the package implicit + + $ spack gc + ==> The following packages will be uninstalled: + + -- linux-fedora32-haswell / gcc@10.1.1 -------------------------- + 5fj7p2o libsigsegv@2.11 c6ensc6 m4@1.4.18 + + ==> Do you want to proceed? [y/N] + +In the example above, we ended up with two versions of ``m4`` since they depend +on different versions of ``libsigsegv``. ``spack gc`` will not remove any of +the packages since both versions of ``m4`` have been installed explicitly +and both versions of ``libsigsegv`` are required by the ``m4`` packages. + +``spack mark`` can also be used to implement upgrade workflows. The following +example demonstrates how the ``spack mark`` and ``spack gc`` can be used to +only keep the current version of a package installed. + +When updating Spack via ``git pull``, new versions for either ``libsigsegv`` +or ``m4`` might be introduced. This will cause Spack to install duplicates. +Since we only want to keep one version, we mark everything as implicitly +installed before updating Spack. If there is no new version for either of the +packages, ``spack install`` will simply mark them as explicitly installed and +``spack gc`` will not remove them. + +.. code-block:: console + + $ spack install m4 + ==> 62843: Installing libsigsegv + [...] + ==> 62843: Installing m4 + [...] + + $ spack mark -i -a + ==> m4@1.4.18 : marking the package implicit + + $ git pull + [...] + + $ spack install m4 + [...] + ==> m4@1.4.18 : marking the package explicit + [...] + + $ spack gc + ==> There are no unused specs. Spack's store is clean. + +When using this workflow for installations that contain more packages, care +has to be taken to either only mark selected packages or issue ``spack install`` +for all packages that should be kept. + +You can check :ref:`cmd-spack-find-metadata` to see how to query for explicitly +or implicitly installed packages. + ^^^^^^^^^^^^^^^^^^^^^^^^^ Non-Downloadable Tarballs ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 5172bdee07..d48301f5de 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -338,6 +338,7 @@ def display_specs(specs, args=None, **kwargs): decorators (dict): dictionary mappng specs to decorators header_callback (function): called at start of arch/compiler groups all_headers (bool): show headers even when arch/compiler aren't defined + output (stream): A file object to write to. Default is ``sys.stdout`` """ def get_arg(name, default=None): @@ -358,6 +359,7 @@ def display_specs(specs, args=None, **kwargs): variants = get_arg('variants', False) groups = get_arg('groups', True) all_headers = get_arg('all_headers', False) + output = get_arg('output', sys.stdout) decorator = get_arg('decorator', None) if decorator is None: @@ -406,31 +408,39 @@ def display_specs(specs, args=None, **kwargs): # unless any of these are set, we can just colify and be done. if not any((deps, paths)): - colify((f[0] for f in formatted), indent=indent) - return + colify((f[0] for f in formatted), indent=indent, output=output) + return '' # otherwise, we'll print specs one by one max_width = max(len(f[0]) for f in formatted) path_fmt = "%%-%ds%%s" % (max_width + 2) + out = '' # getting lots of prefixes requires DB lookups. Ensure # all spec.prefix calls are in one transaction. with spack.store.db.read_transaction(): for string, spec in formatted: if not string: - print() # print newline from above + # print newline from above + out += '\n' continue if paths: - print(path_fmt % (string, spec.prefix)) + out += path_fmt % (string, spec.prefix) + '\n' else: - print(string) + out += string + '\n' + return out + + out = '' if groups: for specs in iter_groups(specs, indent, all_headers): - format_list(specs) + out += format_list(specs) else: - format_list(sorted(specs)) + out = format_list(sorted(specs)) + + output.write(out) + output.flush() def spack_is_git_repo(): diff --git a/lib/spack/spack/cmd/mark.py b/lib/spack/spack/cmd/mark.py new file mode 100644 index 0000000000..85a22b9741 --- /dev/null +++ b/lib/spack/spack/cmd/mark.py @@ -0,0 +1,122 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from __future__ import print_function + +import sys + +import spack.cmd +import spack.error +import spack.package +import spack.cmd.common.arguments as arguments +import spack.repo +import spack.store +from spack.database import InstallStatuses + +from llnl.util import tty + +description = "mark packages as explicitly or implicitly installed" +section = "admin" +level = "long" + +error_message = """You can either: + a) use a more specific spec, or + b) use `spack mark --all` to mark ALL matching specs. +""" + +# Arguments for display_specs when we find ambiguity +display_args = { + 'long': True, + 'show_flags': False, + 'variants': False, + 'indent': 4, +} + + +def setup_parser(subparser): + arguments.add_common_arguments( + subparser, ['installed_specs']) + subparser.add_argument( + '-a', '--all', action='store_true', dest='all', + help="Mark ALL installed packages that match each " + "supplied spec. If you `mark --all libelf`," + " ALL versions of `libelf` are marked. If no spec is " + "supplied, all installed packages will be marked.") + exim = subparser.add_mutually_exclusive_group(required=True) + exim.add_argument( + '-e', '--explicit', action='store_true', dest='explicit', + help="Mark packages as explicitly installed.") + exim.add_argument( + '-i', '--implicit', action='store_true', dest='implicit', + help="Mark packages as implicitly installed.") + + +def find_matching_specs(specs, allow_multiple_matches=False): + """Returns a list of specs matching the not necessarily + concretized specs given from cli + + Args: + specs (list): list of specs to be matched against installed packages + allow_multiple_matches (bool): if True multiple matches are admitted + + Return: + list of specs + """ + # List of specs that match expressions given via command line + specs_from_cli = [] + has_errors = False + + for spec in specs: + install_query = [InstallStatuses.INSTALLED] + matching = spack.store.db.query_local(spec, installed=install_query) + # For each spec provided, make sure it refers to only one package. + # Fail and ask user to be unambiguous if it doesn't + if not allow_multiple_matches and len(matching) > 1: + tty.error('{0} matches multiple packages:'.format(spec)) + sys.stderr.write('\n') + spack.cmd.display_specs(matching, output=sys.stderr, + **display_args) + sys.stderr.write('\n') + sys.stderr.flush() + has_errors = True + + # No installed package matches the query + if len(matching) == 0 and spec is not any: + tty.die('{0} does not match any installed packages.'.format(spec)) + + specs_from_cli.extend(matching) + + if has_errors: + tty.die(error_message) + + return specs_from_cli + + +def do_mark(specs, explicit): + """Marks all the specs in a list. + + Args: + specs (list): list of specs to be marked + explicit (bool): whether to mark specs as explicitly installed + """ + for spec in specs: + spack.store.db.update_explicit(spec, explicit) + + +def mark_specs(args, specs): + mark_list = find_matching_specs(specs, args.all) + + # Mark everything on the list + do_mark(mark_list, args.explicit) + + +def mark(parser, args): + if not args.specs and not args.all: + tty.die('mark requires at least one package argument.', + ' Use `spack mark --all` to mark ALL packages.') + + # [any] here handles the --all case by forcing all specs to be returned + specs = spack.cmd.parse_specs(args.specs) if args.specs else [any] + mark_specs(args, specs) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index cec71c6749..e541eaf91b 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -90,9 +90,11 @@ def find_matching_specs(env, specs, allow_multiple_matches=False, force=False): # Fail and ask user to be unambiguous if it doesn't if not allow_multiple_matches and len(matching) > 1: tty.error('{0} matches multiple packages:'.format(spec)) - print() - spack.cmd.display_specs(matching, **display_args) - print() + sys.stderr.write('\n') + spack.cmd.display_specs(matching, output=sys.stderr, + **display_args) + sys.stderr.write('\n') + sys.stderr.flush() has_errors = True # No installed package matches the query diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index f673e40f40..db1e6a636b 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -1532,6 +1532,24 @@ class Database(object): return unused + def update_explicit(self, spec, explicit): + """ + Update the spec's explicit state in the database. + + Args: + spec (Spec): the spec whose install record is being updated + explicit (bool): ``True`` if the package was requested explicitly + by the user, ``False`` if it was pulled in as a dependency of + an explicit package. + """ + rec = self.get_record(spec) + if explicit != rec.explicit: + with self.write_transaction(): + message = '{s.name}@{s.version} : marking the package {0}' + status = 'explicit' if explicit else 'implicit' + tty.debug(message.format(status, s=spec)) + rec.explicit = explicit + class UpstreamDatabaseLockingError(SpackError): """Raised when an operation would need to lock an upstream database""" diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index dd35db0839..b70528dea2 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -315,11 +315,11 @@ def _process_external_package(pkg, explicit): try: # Check if the package was already registered in the DB. # If this is the case, then just exit. - rec = spack.store.db.get_record(spec) tty.debug('{0} already registered in DB'.format(pre)) - # Update the value of rec.explicit if it is necessary - _update_explicit_entry_in_db(pkg, rec, explicit) + # Update the explicit state if it is necessary + if explicit: + spack.store.db.update_explicit(spec, explicit) except KeyError: # If not, register it and generate the module file. @@ -395,25 +395,6 @@ def _try_install_from_binary_cache(pkg, explicit, unsigned=False, preferred_mirrors=preferred_mirrors) -def _update_explicit_entry_in_db(pkg, rec, explicit): - """ - Ensure the spec is marked explicit in the database. - - Args: - pkg (Package): the package whose install record is being updated - rec (InstallRecord): the external package - explicit (bool): if the package was requested explicitly by the user, - ``False`` if it was pulled in as a dependency of an explicit - package. - """ - if explicit and not rec.explicit: - with spack.store.db.write_transaction(): - rec = spack.store.db.get_record(pkg.spec) - message = '{s.name}@{s.version} : marking the package explicit' - tty.debug(message.format(s=pkg.spec)) - rec.explicit = True - - def clear_failures(): """ Remove all failure tracking markers for the Spack instance. @@ -816,7 +797,7 @@ class PackageInstaller(object): # Only update the explicit entry once for the explicit package if task.explicit: - _update_explicit_entry_in_db(task.pkg, rec, True) + spack.store.db.update_explicit(task.pkg.spec, True) # In case the stage directory has already been created, this # check ensures it is removed after we checked that the spec is diff --git a/lib/spack/spack/test/cmd/mark.py b/lib/spack/spack/test/cmd/mark.py new file mode 100644 index 0000000000..5b488bbf3e --- /dev/null +++ b/lib/spack/spack/test/cmd/mark.py @@ -0,0 +1,67 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import pytest +import spack.store +from spack.main import SpackCommand, SpackCommandError + +gc = SpackCommand('gc') +mark = SpackCommand('mark') +install = SpackCommand('install') +uninstall = SpackCommand('uninstall') + + +@pytest.mark.db +def test_mark_mode_required(mutable_database): + with pytest.raises(SystemExit): + mark('-a') + + +@pytest.mark.db +def test_mark_spec_required(mutable_database): + with pytest.raises(SpackCommandError): + mark('-i') + + +@pytest.mark.db +def test_mark_all_explicit(mutable_database): + mark('-e', '-a') + gc('-y') + all_specs = spack.store.layout.all_specs() + assert len(all_specs) == 14 + + +@pytest.mark.db +def test_mark_all_implicit(mutable_database): + mark('-i', '-a') + gc('-y') + all_specs = spack.store.layout.all_specs() + assert len(all_specs) == 0 + + +@pytest.mark.db +def test_mark_one_explicit(mutable_database): + mark('-e', 'libelf') + uninstall('-y', '-a', 'mpileaks') + gc('-y') + all_specs = spack.store.layout.all_specs() + assert len(all_specs) == 2 + + +@pytest.mark.db +def test_mark_one_implicit(mutable_database): + mark('-i', 'externaltest') + gc('-y') + all_specs = spack.store.layout.all_specs() + assert len(all_specs) == 13 + + +@pytest.mark.db +def test_mark_all_implicit_then_explicit(mutable_database): + mark('-i', '-a') + mark('-e', '-a') + gc('-y') + all_specs = spack.store.layout.all_specs() + assert len(all_specs) == 14 -- cgit v1.2.3-70-g09d2