From efad7ac81b0ad759f03d5f154188b98efd9a8b75 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 2 Nov 2018 12:43:02 -0700 Subject: env: consolidate most of `spack env status` into `spack find` - `spack env status` used to show install status; consolidate that into `spack find`. - `spack env status` will still print out whether there is an active environment --- lib/spack/spack/cmd/__init__.py | 76 +++++++++++++++++++------------ lib/spack/spack/cmd/env.py | 23 ++++------ lib/spack/spack/cmd/find.py | 70 +++++++++++++++++++++++++---- lib/spack/spack/environment.py | 94 +++++++++++++++++++++------------------ lib/spack/spack/modules/common.py | 2 +- lib/spack/spack/spec.py | 63 ++++++++++++++------------ lib/spack/spack/test/cmd/env.py | 79 ++++++++++++++++++++++---------- 7 files changed, 259 insertions(+), 148 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index e207617009..2963c053e4 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -175,7 +175,8 @@ def disambiguate_spec(spec): def gray_hash(spec, length): - return colorize('@K{%s}' % spec.dag_hash(length)) + h = spec.dag_hash(length) if spec.concrete else '-' * length + return colorize('@K{%s}' % h) def display_specs(specs, args=None, **kwargs): @@ -209,7 +210,9 @@ def display_specs(specs, args=None, **kwargs): show_flags (bool): Show compiler flags with specs variants (bool): Show variants with specs indent (int): indent each line this much - + decorators (dict): dictionary mappng specs to decorators + header_callback (function): called at start of arch/compiler sections + all_headers (bool): show headers even when arch/compiler aren't defined """ def get_arg(name, default=None): """Prefer kwargs, then args, then default.""" @@ -220,12 +223,17 @@ def display_specs(specs, args=None, **kwargs): else: return default - mode = get_arg('mode', 'short') - hashes = get_arg('long', False) - namespace = get_arg('namespace', False) - flags = get_arg('show_flags', False) + mode = get_arg('mode', 'short') + hashes = get_arg('long', False) + namespace = get_arg('namespace', False) + flags = get_arg('show_flags', False) full_compiler = get_arg('show_full_compiler', False) - variants = get_arg('variants', False) + variants = get_arg('variants', False) + all_headers = get_arg('all_headers', False) + + decorator = get_arg('decorator', None) + if decorator is None: + decorator = lambda s, f: f indent = get_arg('indent', 0) ispace = indent * ' ' @@ -235,7 +243,7 @@ def display_specs(specs, args=None, **kwargs): hashes = True hlen = None - nfmt = '.' if namespace else '_' + nfmt = '{fullpackage}' if namespace else '{package}' ffmt = '' if full_compiler or flags: ffmt += '$%' @@ -247,35 +255,46 @@ def display_specs(specs, args=None, **kwargs): # Make a dict with specs keyed by architecture and compiler. index = index_by(specs, ('architecture', 'compiler')) + transform = {'package': decorator, 'fullpackage': decorator} # Traverse the index and print out each package for i, (architecture, compiler) in enumerate(sorted(index)): if i > 0: print() - header = "%s{%s} / %s{%s}" % (spack.spec.architecture_color, - architecture, spack.spec.compiler_color, - compiler) + header = "%s{%s} / %s{%s}" % ( + spack.spec.architecture_color, + architecture if architecture else 'no arch', + spack.spec.compiler_color, + compiler if compiler else 'no compiler') + # Sometimes we want to display specs that are not yet concretized. # If they don't have a compiler / architecture attached to them, # then skip the header - if architecture is not None or compiler is not None: + if all_headers or (architecture is not None or compiler is not None): sys.stdout.write(ispace) tty.hline(colorize(header), char='-') specs = index[(architecture, compiler)] specs.sort() - abbreviated = [s.cformat(format_string) for s in specs] if mode == 'paths': # Print one spec per line along with prefix path + abbreviated = [s.cformat(format_string, transform=transform) + for s in specs] width = max(len(s) for s in abbreviated) width += 2 - format = " %%-%ds%%s" % width for abbrv, spec in zip(abbreviated, specs): - prefix = gray_hash(spec, hlen) if hashes else '' - print(ispace + prefix + (format % (abbrv, spec.prefix))) + # optional hash prefix for paths + h = gray_hash(spec, hlen) if hashes else '' + + # only show prefix for concrete specs + prefix = spec.prefix if spec.concrete else '' + + # print it all out at once + fmt = "%%s%%s %%-%ds%%s" % width + print(fmt % (ispace, h, abbrv, prefix)) elif mode == 'deps': for spec in specs: @@ -285,24 +304,25 @@ def display_specs(specs, args=None, **kwargs): prefix=(lambda s: gray_hash(s, hlen)) if hashes else None)) elif mode == 'short': - # Print columns of output if not printing flags - if not flags and not full_compiler: - - def fmt(s): - string = "" - if hashes: - string += gray_hash(s, hlen) + ' ' - string += s.cformat('$-%s$@%s' % (nfmt, vfmt)) - - return string + def fmt(s): + string = "" + if hashes: + string += gray_hash(s, hlen) + ' ' + string += s.cformat( + '$%s$@%s' % (nfmt, vfmt), transform=transform) + return string + if not flags and not full_compiler: + # Print columns of output if not printing flags colify((fmt(s) for s in specs), indent=indent) - # Print one entry per line if including flags + else: + # Print one entry per line if including flags for spec in specs: # Print the hash if necessary hsh = gray_hash(spec, hlen) + ' ' if hashes else '' - print(ispace + hsh + spec.cformat(format_string)) + print(ispace + hsh + spec.cformat( + format_string, transform=transform)) else: raise ValueError( diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py index 2d5f95f640..3403194659 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/env.py @@ -271,26 +271,19 @@ def env_list(args): # env status # def env_status_setup_parser(subparser): - """get install status of specs in an environment""" - subparser.add_argument( - 'env', nargs='?', help='name of environment to show status for') - arguments.add_common_arguments( - subparser, - ['recurse_dependencies', 'long', 'very_long']) + """print whether there is an active environment""" def env_status(args): env = ev.get_env(args, 'env status', required=False) - if not env: + if env: + if env.path == os.getcwd(): + tty.msg('Using %s in current directory: %s' + % (ev.manifest_name, env.path)) + else: + tty.msg('In environment %s' % env.name) + else: tty.msg('No active environment') - return - - # TODO: option to show packages w/ multiple instances? - env.status( - sys.stdout, recurse_dependencies=args.recurse_dependencies, - hashes=args.long or args.very_long, - hashlen=None if args.very_long else 7, - install_status=True) # diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 680e44af28..358e3c6f01 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -2,15 +2,17 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import sys +from __future__ import print_function import llnl.util.tty as tty +import llnl.util.tty.color as color import llnl.util.lang +import spack.environment as ev import spack.repo import spack.cmd.common.arguments as arguments from spack.cmd import display_specs +from spack.util.string import plural description = "list and search installed packages" section = "basic" @@ -40,6 +42,9 @@ def setup_parser(subparser): arguments.add_common_arguments( subparser, ['long', 'very_long', 'tags']) + subparser.add_argument('-c', '--show-concretized', + action='store_true', + help='show concretized specs in an environment') subparser.add_argument('-f', '--show-flags', action='store_true', dest='show_flags', @@ -116,12 +121,42 @@ def query_arguments(args): return q_args +def setup_env(env): + """Create a function for decorating specs when in an environment.""" + + def strip_build(seq): + return set(s.copy(deps=('link', 'run')) for s in seq) + + added = set(strip_build(env.added_specs())) + roots = set(strip_build(env.roots())) + removed = set(strip_build(env.removed_specs())) + + def decorator(spec, fmt): + # add +/-/* to show added/removed/root specs + if spec in roots: + return color.colorize('@*{%s}' % fmt) + elif spec in removed: + return color.colorize('@K{%s}' % fmt) + else: + return '%s' % fmt + + return decorator, added, roots, removed + + def find(parser, args): q_args = query_arguments(args) - query_specs = args.specs(**q_args) + results = args.specs(**q_args) + + decorator = lambda s, f: f + added = set() + removed = set() + + env = ev.get_env(args, 'find', required=False) + if env: + decorator, added, roots, removed = setup_env(env) # Exit early if no package matches the constraint - if not query_specs and args.constraint: + if not results and args.constraint: msg = "No package matches the query: {0}" msg = msg.format(' '.join(args.constraint)) tty.msg(msg) @@ -130,10 +165,29 @@ def find(parser, args): # If tags have been specified on the command line, filter by tags if args.tags: packages_with_tags = spack.repo.path.packages_with_tags(*args.tags) - query_specs = [x for x in query_specs if x.name in packages_with_tags] + results = [x for x in results if x.name in packages_with_tags] # Display the result - if sys.stdout.isatty(): - tty.msg("%d installed packages." % len(query_specs)) + if env: + tty.msg('In environment %s' % env.name) + + print() + tty.msg('Root specs') + + if not env.user_specs: + print('none') + else: + display_specs( + env.user_specs, args, + decorator=lambda s, f: color.colorize('@*{%s}' % f)) + print() + + if args.show_concretized: + tty.msg('Concretized roots') + display_specs( + env.specs_by_hash.values(), args, decorator=decorator) + print() + + tty.msg("%s" % plural(len(results), 'installed package')) - display_specs(query_specs, args) + display_specs(results, args, decorator=decorator, all_headers=True) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index c221aab855..70bba192d7 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -13,7 +13,6 @@ import ruamel.yaml import llnl.util.filesystem as fs import llnl.util.tty as tty -import llnl.util.tty.color as color import spack.error import spack.repo @@ -152,7 +151,7 @@ def get_env(args, cmd_name, required=True): """ # try arguments - env = args.env + env = getattr(args, 'env', None) # try a manifest file in the current directory if not env: @@ -356,6 +355,11 @@ class Environment(object): self._repo = None # RepoPath for this env (memoized) self._previous_active = None # previously active environment + @property + def internal(self): + """Whether this environment is managed by Spack.""" + return self.path.startswith(env_path) + @property def name(self): """Human-readable representation of the environment. @@ -363,7 +367,7 @@ class Environment(object): This is the path for directory environments, and just the name for named environments. """ - if self.path.startswith(env_path): + if self.internal: return os.path.basename(self.path) else: return self.path @@ -625,46 +629,6 @@ class Environment(object): os.remove(build_log_link) os.symlink(spec.package.build_log_path, build_log_link) - def status(self, stream, **kwargs): - """List the specs in an environment.""" - tty.msg('In environment %s' % self.name) - - concretized = [(spec, self.specs_by_hash[h]) - for spec, h in zip(self.concretized_user_specs, - self.concretized_order)] - - added = [s for s in self.user_specs - if s not in self.concretized_user_specs] - removed = [(s, c) for s, c in concretized if s not in self.user_specs] - current = [(s, c) for s, c in concretized if s in self.user_specs] - - def write_kind(s): - color.cwrite('@c{%s}\n' % color.cescape(s), stream) - - def write_user_spec(s, c): - color.cwrite('@%s{----} %s\n' % (c, color.cescape(s)), stream) - - if added: - write_kind('added:') - for s in added: - write_user_spec(s, 'g') - - if current: - if added: - stream.write('\n') - write_kind('concrete:') - for s, c in current: - write_user_spec(s, 'K') - stream.write(c.tree(**kwargs)) - - if removed: - if added or current: - stream.write('\n') - write_kind('removed:') - for s, c in removed: - write_user_spec(s, 'r') - stream.write(c.tree(**kwargs)) - def all_specs_by_hash(self): """Map of hashes to spec for all specs in this environment.""" hashes = {} @@ -682,6 +646,50 @@ class Environment(object): """Return all specs, even those a user spec would shadow.""" return list(self.all_specs_by_hash().keys()) + def roots(self): + """Specs explicitly requested by the user *in this environment*. + + Yields both added and installed specs that have user specs in + `spack.yaml`. + """ + concretized = dict(self.concretized_specs()) + for spec in self.user_specs: + concrete = concretized.get(spec) + yield concrete if concrete else spec + + def added_specs(self): + """Specs that are not yet installed. + + Yields the user spec for non-concretized specs, and the concrete + spec for already concretized but not yet installed specs. + """ + concretized = dict(self.concretized_specs()) + for spec in self.user_specs: + concrete = concretized.get(spec) + if not concrete: + yield spec + elif not concrete.package.installed: + yield concrete + + def concretized_specs(self): + """Tuples of (user spec, concrete spec) for all concrete specs.""" + for s, h in zip(self.concretized_user_specs, self.concretized_order): + yield (s, self.specs_by_hash[h]) + + def removed_specs(self): + """Tuples of (user spec, concrete spec) for all specs that will be + removed on nexg concretize.""" + needed = set() + for s, c in self.concretized_specs(): + if s in self.user_specs: + for d in c.traverse(): + needed.add(d) + + for s, c in self.concretized_specs(): + for d in c.traverse(): + if d not in needed: + yield d + def _get_environment_specs(self, recurse_dependencies=True): """Returns the specs of all the packages in an environment. diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py index 3acc2b0dcf..87ad261121 100644 --- a/lib/spack/spack/modules/common.py +++ b/lib/spack/spack/modules/common.py @@ -546,7 +546,7 @@ class BaseContext(tengine.Context): # tokens uppercase. transform = {} for token in _valid_tokens: - transform[token] = str.upper + transform[token] = lambda spec, string: str.upper(string) for x in env: # Ensure all the tokens are valid in this context diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index bebb1ca967..8b48da52e5 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -2906,7 +2906,7 @@ class Spec(object): """Comparison key for just *this node* and not its deps.""" return (self.name, self.namespace, - self.versions, + tuple(self.versions), self.variants, self.architecture, self.compiler, @@ -2964,6 +2964,7 @@ class Spec(object): You can also use full-string versions, which elide the prefixes:: ${PACKAGE} Package name + ${FULLPACKAGE} Full package name (with namespace) ${VERSION} Version ${COMPILER} Full compiler string ${COMPILERNAME} Compiler name @@ -2995,11 +2996,9 @@ class Spec(object): Args: format_string (str): string containing the format to be expanded - **kwargs (dict): the following list of keywords is supported - - - color (bool): True if returned string is colored - - - transform (dict): maps full-string formats to a callable \ + Keyword Args: + color (bool): True if returned string is colored + transform (dict): maps full-string formats to a callable \ that accepts a string and returns another one Examples: @@ -3019,16 +3018,18 @@ class Spec(object): color = kwargs.get('color', False) # Dictionary of transformations for named tokens - token_transforms = {} - token_transforms.update(kwargs.get('transform', {})) + token_transforms = dict( + (k.upper(), v) for k, v in kwargs.get('transform', {}).items()) length = len(format_string) out = StringIO() named = escape = compiler = False named_str = fmt = '' - def write(s, c): - f = color_formats[c] + cescape(s) + '@.' + def write(s, c=None): + f = cescape(s) + if c is not None: + f = color_formats[c] + f + '@.' cwrite(f, stream=out, color=color) iterator = enumerate(format_string) @@ -3048,7 +3049,8 @@ class Spec(object): name = self.name if self.name else '' out.write(fmt % name) elif c == '.': - out.write(fmt % self.fullname) + name = self.fullname if self.fullname else '' + out.write(fmt % name) elif c == '@': if self.versions and self.versions != _any_version: write(fmt % (c + str(self.versions)), c) @@ -3103,60 +3105,63 @@ class Spec(object): # # The default behavior is to leave the string unchanged # (`lambda x: x` is the identity function) - token_transform = token_transforms.get(named_str, lambda x: x) + transform = token_transforms.get(named_str, lambda s, x: x) if named_str == 'PACKAGE': name = self.name if self.name else '' - write(fmt % token_transform(name), '@') - if named_str == 'VERSION': + write(fmt % transform(self, name)) + elif named_str == 'FULLPACKAGE': + name = self.fullname if self.fullname else '' + write(fmt % transform(self, name)) + elif named_str == 'VERSION': if self.versions and self.versions != _any_version: - write(fmt % token_transform(str(self.versions)), '@') + write(fmt % transform(self, str(self.versions)), '@') elif named_str == 'COMPILER': if self.compiler: - write(fmt % token_transform(self.compiler), '%') + write(fmt % transform(self, self.compiler), '%') elif named_str == 'COMPILERNAME': if self.compiler: - write(fmt % token_transform(self.compiler.name), '%') + write(fmt % transform(self, self.compiler.name), '%') elif named_str in ['COMPILERVER', 'COMPILERVERSION']: if self.compiler: write( - fmt % token_transform(self.compiler.versions), + fmt % transform(self, self.compiler.versions), '%' ) elif named_str == 'COMPILERFLAGS': if self.compiler: write( - fmt % token_transform(str(self.compiler_flags)), + fmt % transform(self, str(self.compiler_flags)), '%' ) elif named_str == 'OPTIONS': if self.variants: - write(fmt % token_transform(str(self.variants)), '+') + write(fmt % transform(self, str(self.variants)), '+') elif named_str in ["ARCHITECTURE", "PLATFORM", "TARGET", "OS"]: if self.architecture and str(self.architecture): if named_str == "ARCHITECTURE": write( - fmt % token_transform(str(self.architecture)), + fmt % transform(self, str(self.architecture)), '=' ) elif named_str == "PLATFORM": platform = str(self.architecture.platform) - write(fmt % token_transform(platform), '=') + write(fmt % transform(self, platform), '=') elif named_str == "OS": operating_sys = str(self.architecture.platform_os) - write(fmt % token_transform(operating_sys), '=') + write(fmt % transform(self, operating_sys), '=') elif named_str == "TARGET": target = str(self.architecture.target) - write(fmt % token_transform(target), '=') + write(fmt % transform(self, target), '=') elif named_str == 'SHA1': if self.dependencies: - out.write(fmt % token_transform(str(self.dag_hash(7)))) + out.write(fmt % transform(self, str(self.dag_hash(7)))) elif named_str == 'SPACK_ROOT': - out.write(fmt % token_transform(spack.paths.prefix)) + out.write(fmt % transform(self, spack.paths.prefix)) elif named_str == 'SPACK_INSTALL': - out.write(fmt % token_transform(spack.store.root)) + out.write(fmt % transform(self, spack.store.root)) elif named_str == 'PREFIX': - out.write(fmt % token_transform(self.prefix)) + out.write(fmt % transform(self, self.prefix)) elif named_str.startswith('HASH'): if named_str.startswith('HASH:'): _, hashlen = named_str.split(':') @@ -3165,7 +3170,7 @@ class Spec(object): hashlen = None out.write(fmt % (self.dag_hash(hashlen))) elif named_str == 'NAMESPACE': - out.write(fmt % token_transform(self.namespace)) + out.write(fmt % transform(self.namespace)) named = False diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index 2828f5806c..843ea7d2e9 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -21,13 +21,14 @@ from spack.main import SpackCommand pytestmark = pytest.mark.usefixtures( 'mutable_mock_env_path', 'config', 'mutable_mock_packages') -env = SpackCommand('env') -install = SpackCommand('install') -add = SpackCommand('add') -remove = SpackCommand('remove') -concretize = SpackCommand('concretize') -stage = SpackCommand('stage') -uninstall = SpackCommand('uninstall') +env = SpackCommand('env') +install = SpackCommand('install') +add = SpackCommand('add') +remove = SpackCommand('remove') +concretize = SpackCommand('concretize') +stage = SpackCommand('stage') +uninstall = SpackCommand('uninstall') +find = SpackCommand('find') def test_add(): @@ -156,32 +157,62 @@ def test_remove_after_concretize(): def test_remove_command(): env('create', 'test') + assert 'test' in env('list') with ev.read('test'): add('mpileaks') - assert 'mpileaks' in env('status', 'test') + assert 'mpileaks' in find() + assert 'mpileaks@' not in find() + assert 'mpileaks@' not in find('--show-concretized') with ev.read('test'): remove('mpileaks') - assert 'mpileaks' not in env('status', 'test') + assert 'mpileaks' not in find() + assert 'mpileaks@' not in find() + assert 'mpileaks@' not in find('--show-concretized') with ev.read('test'): add('mpileaks') - assert 'mpileaks' in env('status', 'test') + assert 'mpileaks' in find() + assert 'mpileaks@' not in find() + assert 'mpileaks@' not in find('--show-concretized') + with ev.read('test'): + concretize() + assert 'mpileaks' in find() + assert 'mpileaks@' not in find() + assert 'mpileaks@' in find('--show-concretized') -def test_environment_status(): - e = ev.create('test') - e.add('mpileaks') - e.concretize() - e.add('python') - mock_stream = StringIO() - e.status(mock_stream) - list_content = mock_stream.getvalue() - assert 'mpileaks' in list_content - assert 'python' in list_content - mpileaks_spec = e.specs_by_hash[e.concretized_order[0]] - assert mpileaks_spec.format() in list_content + with ev.read('test'): + remove('mpileaks') + assert 'mpileaks' not in find() + # removed but still in last concretized specs + assert 'mpileaks@' in find('--show-concretized') + + with ev.read('test'): + concretize() + assert 'mpileaks' not in find() + assert 'mpileaks@' not in find() + # now the lockfile is regenerated and it's gone. + assert 'mpileaks@' not in find('--show-concretized') + + +def test_environment_status(capfd, tmpdir): + with capfd.disabled(): + with tmpdir.as_cwd(): + assert 'No active environment' in env('status') + + with ev.create('test'): + assert 'In environment test' in env('status') + + with ev.Environment('local_dir'): + assert os.path.join(os.getcwd(), 'local_dir') in env('status') + + e = ev.Environment('myproject') + e.write() + with tmpdir.join('myproject').as_cwd(): + with e: + assert 'in current directory' in env('status') def test_to_lockfile_dict(): @@ -309,8 +340,8 @@ env: out = env('list') assert 'test' in out - out = env('status', 'test') - assert 'mpileaks' in out + with ev.read('test'): + assert 'mpileaks' in find() env('remove', '-y', 'test') -- cgit v1.2.3-60-g2f50