diff options
Diffstat (limited to 'lib')
59 files changed, 4625 insertions, 553 deletions
diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 9e71d25caf..266e41cb48 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -1,12 +1,11 @@ #!/usr/bin/env python import sys -if not sys.version_info[:2] >= (2,7): - sys.exit("Spack requires Python 2.7. Version was %s." % sys.version_info) +if not sys.version_info[:2] >= (2,6): + sys.exit("Spack requires Python 2.6. Version was %s." % sys.version_info) import os import re import subprocess -import argparse from contextlib import closing # Import spack parameters through the build environment. @@ -18,6 +17,7 @@ if not spack_lib: # Grab a minimal set of spack packages sys.path.append(spack_lib) from spack.compilation import * +from external import argparse import llnl.util.tty as tty spack_prefix = get_env_var("SPACK_PREFIX") diff --git a/lib/spack/external/__init__.py b/lib/spack/external/__init__.py new file mode 100644 index 0000000000..1cc981930a --- /dev/null +++ b/lib/spack/external/__init__.py @@ -0,0 +1,33 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +""" +This module contains external, potentially separately licensed, +packages that are included in spack. + +So far: + argparse: We include our own version to be Python 2.6 compatible. + pyqver2: External script to query required python version of python source code. + Used for ensuring 2.6 compatibility. +""" diff --git a/lib/spack/external/argparse.py b/lib/spack/external/argparse.py new file mode 100644 index 0000000000..c8dfdd3bed --- /dev/null +++ b/lib/spack/external/argparse.py @@ -0,0 +1,2382 @@ +# argparse is (c) 2006-2009 Steven J. Bethard <steven.bethard@gmail.com>. +# +# The argparse module was contributed to Python as of Python 2.7 and thus +# was licensed under the Python license. Same license applies to all files in +# the argparse package project. +# +# For details about the Python License, please see doc/Python-License.txt. +# +# History +# ------- +# +# Before (and including) argparse 1.1, the argparse package was licensed under +# Apache License v2.0. +# +# After argparse 1.1, all project files from the argparse project were deleted +# due to license compatibility issues between Apache License 2.0 and GNU GPL v2. +# +# The project repository then had a clean start with some files taken from +# Python 2.7.1, so definitely all files are under Python License now. +# +# Author: Steven J. Bethard <steven.bethard@gmail.com>. +# +"""Command-line parsing library + +This module is an optparse-inspired command-line parsing library that: + + - handles both optional and positional arguments + - produces highly informative usage messages + - supports parsers that dispatch to sub-parsers + +The following is a simple usage example that sums integers from the +command-line and writes the result to a file:: + + parser = argparse.ArgumentParser( + description='sum the integers at the command line') + parser.add_argument( + 'integers', metavar='int', nargs='+', type=int, + help='an integer to be summed') + parser.add_argument( + '--log', default=sys.stdout, type=argparse.FileType('w'), + help='the file where the sum should be written') + args = parser.parse_args() + args.log.write('%s' % sum(args.integers)) + args.log.close() + +The module contains the following public classes: + + - ArgumentParser -- The main entry point for command-line parsing. As the + example above shows, the add_argument() method is used to populate + the parser with actions for optional and positional arguments. Then + the parse_args() method is invoked to convert the args at the + command-line into an object with attributes. + + - ArgumentError -- The exception raised by ArgumentParser objects when + there are errors with the parser's actions. Errors raised while + parsing the command-line are caught by ArgumentParser and emitted + as command-line messages. + + - FileType -- A factory for defining types of files to be created. As the + example above shows, instances of FileType are typically passed as + the type= argument of add_argument() calls. + + - Action -- The base class for parser actions. Typically actions are + selected by passing strings like 'store_true' or 'append_const' to + the action= argument of add_argument(). However, for greater + customization of ArgumentParser actions, subclasses of Action may + be defined and passed as the action= argument. + + - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter, + ArgumentDefaultsHelpFormatter -- Formatter classes which + may be passed as the formatter_class= argument to the + ArgumentParser constructor. HelpFormatter is the default, + RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser + not to change the formatting for help text, and + ArgumentDefaultsHelpFormatter adds information about argument defaults + to the help. + +All other classes in this module are considered implementation details. +(Also note that HelpFormatter and RawDescriptionHelpFormatter are only +considered public as object names -- the API of the formatter objects is +still considered an implementation detail.) +""" + +__version__ = '1.2.1' +__all__ = [ + 'ArgumentParser', + 'ArgumentError', + 'ArgumentTypeError', + 'FileType', + 'HelpFormatter', + 'ArgumentDefaultsHelpFormatter', + 'RawDescriptionHelpFormatter', + 'RawTextHelpFormatter', + 'Namespace', + 'Action', + 'ONE_OR_MORE', + 'OPTIONAL', + 'PARSER', + 'REMAINDER', + 'SUPPRESS', + 'ZERO_OR_MORE', +] + + +import copy as _copy +import os as _os +import re as _re +import sys as _sys +import textwrap as _textwrap + +from gettext import gettext as _ + +try: + set +except NameError: + # for python < 2.4 compatibility (sets module is there since 2.3): + from sets import Set as set + +try: + basestring +except NameError: + basestring = str + +try: + sorted +except NameError: + # for python < 2.4 compatibility: + def sorted(iterable, reverse=False): + result = list(iterable) + result.sort() + if reverse: + result.reverse() + return result + + +def _callable(obj): + return hasattr(obj, '__call__') or hasattr(obj, '__bases__') + + +SUPPRESS = '==SUPPRESS==' + +OPTIONAL = '?' +ZERO_OR_MORE = '*' +ONE_OR_MORE = '+' +PARSER = 'A...' +REMAINDER = '...' +_UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args' + +# ============================= +# Utility functions and classes +# ============================= + +class _AttributeHolder(object): + """Abstract base class that provides __repr__. + + The __repr__ method returns a string in the format:: + ClassName(attr=name, attr=name, ...) + The attributes are determined either by a class-level attribute, + '_kwarg_names', or by inspecting the instance __dict__. + """ + + def __repr__(self): + type_name = type(self).__name__ + arg_strings = [] + for arg in self._get_args(): + arg_strings.append(repr(arg)) + for name, value in self._get_kwargs(): + arg_strings.append('%s=%r' % (name, value)) + return '%s(%s)' % (type_name, ', '.join(arg_strings)) + + def _get_kwargs(self): + return sorted(self.__dict__.items()) + + def _get_args(self): + return [] + + +def _ensure_value(namespace, name, value): + if getattr(namespace, name, None) is None: + setattr(namespace, name, value) + return getattr(namespace, name) + + +# =============== +# Formatting Help +# =============== + +class HelpFormatter(object): + """Formatter for generating usage messages and argument help strings. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def __init__(self, + prog, + indent_increment=2, + max_help_position=24, + width=None): + + # default setting for width + if width is None: + try: + width = int(_os.environ['COLUMNS']) + except (KeyError, ValueError): + width = 80 + width -= 2 + + self._prog = prog + self._indent_increment = indent_increment + self._max_help_position = max_help_position + self._width = width + + self._current_indent = 0 + self._level = 0 + self._action_max_length = 0 + + self._root_section = self._Section(self, None) + self._current_section = self._root_section + + self._whitespace_matcher = _re.compile(r'\s+') + self._long_break_matcher = _re.compile(r'\n\n\n+') + + # =============================== + # Section and indentation methods + # =============================== + def _indent(self): + self._current_indent += self._indent_increment + self._level += 1 + + def _dedent(self): + self._current_indent -= self._indent_increment + assert self._current_indent >= 0, 'Indent decreased below 0.' + self._level -= 1 + + class _Section(object): + + def __init__(self, formatter, parent, heading=None): + self.formatter = formatter + self.parent = parent + self.heading = heading + self.items = [] + + def format_help(self): + # format the indented section + if self.parent is not None: + self.formatter._indent() + join = self.formatter._join_parts + for func, args in self.items: + func(*args) + item_help = join([func(*args) for func, args in self.items]) + if self.parent is not None: + self.formatter._dedent() + + # return nothing if the section was empty + if not item_help: + return '' + + # add the heading if the section was non-empty + if self.heading is not SUPPRESS and self.heading is not None: + current_indent = self.formatter._current_indent + heading = '%*s%s:\n' % (current_indent, '', self.heading) + else: + heading = '' + + # join the section-initial newline, the heading and the help + return join(['\n', heading, item_help, '\n']) + + def _add_item(self, func, args): + self._current_section.items.append((func, args)) + + # ======================== + # Message building methods + # ======================== + def start_section(self, heading): + self._indent() + section = self._Section(self, self._current_section, heading) + self._add_item(section.format_help, []) + self._current_section = section + + def end_section(self): + self._current_section = self._current_section.parent + self._dedent() + + def add_text(self, text): + if text is not SUPPRESS and text is not None: + self._add_item(self._format_text, [text]) + + def add_usage(self, usage, actions, groups, prefix=None): + if usage is not SUPPRESS: + args = usage, actions, groups, prefix + self._add_item(self._format_usage, args) + + def add_argument(self, action): + if action.help is not SUPPRESS: + + # find all invocations + get_invocation = self._format_action_invocation + invocations = [get_invocation(action)] + for subaction in self._iter_indented_subactions(action): + invocations.append(get_invocation(subaction)) + + # update the maximum item length + invocation_length = max([len(s) for s in invocations]) + action_length = invocation_length + self._current_indent + self._action_max_length = max(self._action_max_length, + action_length) + + # add the item to the list + self._add_item(self._format_action, [action]) + + def add_arguments(self, actions): + for action in actions: + self.add_argument(action) + + # ======================= + # Help-formatting methods + # ======================= + def format_help(self): + help = self._root_section.format_help() + if help: + help = self._long_break_matcher.sub('\n\n', help) + help = help.strip('\n') + '\n' + return help + + def _join_parts(self, part_strings): + return ''.join([part + for part in part_strings + if part and part is not SUPPRESS]) + + def _format_usage(self, usage, actions, groups, prefix): + if prefix is None: + prefix = _('usage: ') + + # if usage is specified, use that + if usage is not None: + usage = usage % dict(prog=self._prog) + + # if no optionals or positionals are available, usage is just prog + elif usage is None and not actions: + usage = '%(prog)s' % dict(prog=self._prog) + + # if optionals and positionals are available, calculate usage + elif usage is None: + prog = '%(prog)s' % dict(prog=self._prog) + + # split optionals from positionals + optionals = [] + positionals = [] + for action in actions: + if action.option_strings: + optionals.append(action) + else: + positionals.append(action) + + # build full usage string + format = self._format_actions_usage + action_usage = format(optionals + positionals, groups) + usage = ' '.join([s for s in [prog, action_usage] if s]) + + # wrap the usage parts if it's too long + text_width = self._width - self._current_indent + if len(prefix) + len(usage) > text_width: + + # break usage into wrappable parts + part_regexp = r'\(.*?\)+|\[.*?\]+|\S+' + opt_usage = format(optionals, groups) + pos_usage = format(positionals, groups) + opt_parts = _re.findall(part_regexp, opt_usage) + pos_parts = _re.findall(part_regexp, pos_usage) + assert ' '.join(opt_parts) == opt_usage + assert ' '.join(pos_parts) == pos_usage + + # helper for wrapping lines + def get_lines(parts, indent, prefix=None): + lines = [] + line = [] + if prefix is not None: + line_len = len(prefix) - 1 + else: + line_len = len(indent) - 1 + for part in parts: + if line_len + 1 + len(part) > text_width: + lines.append(indent + ' '.join(line)) + line = [] + line_len = len(indent) - 1 + line.append(part) + line_len += len(part) + 1 + if line: + lines.append(indent + ' '.join(line)) + if prefix is not None: + lines[0] = lines[0][len(indent):] + return lines + + # if prog is short, follow it with optionals or positionals + if len(prefix) + len(prog) <= 0.75 * text_width: + indent = ' ' * (len(prefix) + len(prog) + 1) + if opt_parts: + lines = get_lines([prog] + opt_parts, indent, prefix) + lines.extend(get_lines(pos_parts, indent)) + elif pos_parts: + lines = get_lines([prog] + pos_parts, indent, prefix) + else: + lines = [prog] + + # if prog is long, put it on its own line + else: + indent = ' ' * len(prefix) + parts = opt_parts + pos_parts + lines = get_lines(parts, indent) + if len(lines) > 1: + lines = [] + lines.extend(get_lines(opt_parts, indent)) + lines.extend(get_lines(pos_parts, indent)) + lines = [prog] + lines + + # join lines into usage + usage = '\n'.join(lines) + + # prefix with 'usage:' + return '%s%s\n\n' % (prefix, usage) + + def _format_actions_usage(self, actions, groups): + # find group indices and identify actions in groups + group_actions = set() + inserts = {} + for group in groups: + try: + start = actions.index(group._group_actions[0]) + except ValueError: + continue + else: + end = start + len(group._group_actions) + if actions[start:end] == group._group_actions: + for action in group._group_actions: + group_actions.add(action) + if not group.required: + if start in inserts: + inserts[start] += ' [' + else: + inserts[start] = '[' + inserts[end] = ']' + else: + if start in inserts: + inserts[start] += ' (' + else: + inserts[start] = '(' + inserts[end] = ')' + for i in range(start + 1, end): + inserts[i] = '|' + + # collect all actions format strings + parts = [] + for i, action in enumerate(actions): + + # suppressed arguments are marked with None + # remove | separators for suppressed arguments + if action.help is SUPPRESS: + parts.append(None) + if inserts.get(i) == '|': + inserts.pop(i) + elif inserts.get(i + 1) == '|': + inserts.pop(i + 1) + + # produce all arg strings + elif not action.option_strings: + part = self._format_args(action, action.dest) + + # if it's in a group, strip the outer [] + if action in group_actions: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + + # add the action string to the list + parts.append(part) + + # produce the first way to invoke the option in brackets + else: + option_string = action.option_strings[0] + + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = '%s' % option_string + + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = action.dest.upper() + args_string = self._format_args(action, default) + part = '%s %s' % (option_string, args_string) + + # make it look optional if it's not required or in a group + if not action.required and action not in group_actions: + part = '[%s]' % part + + # add the action string to the list + parts.append(part) + + # insert things at the necessary indices + for i in sorted(inserts, reverse=True): + parts[i:i] = [inserts[i]] + + # join all the action items with spaces + text = ' '.join([item for item in parts if item is not None]) + + # clean up separators for mutually exclusive groups + open = r'[\[(]' + close = r'[\])]' + text = _re.sub(r'(%s) ' % open, r'\1', text) + text = _re.sub(r' (%s)' % close, r'\1', text) + text = _re.sub(r'%s *%s' % (open, close), r'', text) + text = _re.sub(r'\(([^|]*)\)', r'\1', text) + text = text.strip() + + # return the text + return text + + def _format_text(self, text): + if '%(prog)' in text: + text = text % dict(prog=self._prog) + text_width = self._width - self._current_indent + indent = ' ' * self._current_indent + return self._fill_text(text, text_width, indent) + '\n\n' + + def _format_action(self, action): + # determine the required width and the entry label + help_position = min(self._action_max_length + 2, + self._max_help_position) + help_width = self._width - help_position + action_width = help_position - self._current_indent - 2 + action_header = self._format_action_invocation(action) + + # ho nelp; start on same line and add a final newline + if not action.help: + tup = self._current_indent, '', action_header + action_header = '%*s%s\n' % tup + + # short action name; start on the same line and pad two spaces + elif len(action_header) <= action_width: + tup = self._current_indent, '', action_width, action_header + action_header = '%*s%-*s ' % tup + indent_first = 0 + + # long action name; start on the next line + else: + tup = self._current_indent, '', action_header + action_header = '%*s%s\n' % tup + indent_first = help_position + + # collect the pieces of the action help + parts = [action_header] + + # if there was help for the action, add lines of help text + if action.help: + help_text = self._expand_help(action) + help_lines = self._split_lines(help_text, help_width) + parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) + for line in help_lines[1:]: + parts.append('%*s%s\n' % (help_position, '', line)) + + # or add a newline if the description doesn't end with one + elif not action_header.endswith('\n'): + parts.append('\n') + + # if there are any sub-actions, add their help as well + for subaction in self._iter_indented_subactions(action): + parts.append(self._format_action(subaction)) + + # return a single string + return self._join_parts(parts) + + def _format_action_invocation(self, action): + if not action.option_strings: + metavar, = self._metavar_formatter(action, action.dest)(1) + return metavar + + else: + parts = [] + + # if the Optional doesn't take a value, format is: + # -s, --long + if action.nargs == 0: + parts.extend(action.option_strings) + + # if the Optional takes a value, format is: + # -s ARGS, --long ARGS + else: + default = action.dest.upper() + args_string = self._format_args(action, default) + for option_string in action.option_strings: + parts.append('%s %s' % (option_string, args_string)) + + return ', '.join(parts) + + def _metavar_formatter(self, action, default_metavar): + if action.metavar is not None: + result = action.metavar + elif action.choices is not None: + choice_strs = [str(choice) for choice in action.choices] + result = '{%s}' % ','.join(choice_strs) + else: + result = default_metavar + + def format(tuple_size): + if isinstance(result, tuple): + return result + else: + return (result, ) * tuple_size + return format + + def _format_args(self, action, default_metavar): + get_metavar = self._metavar_formatter(action, default_metavar) + if action.nargs is None: + result = '%s' % get_metavar(1) + elif action.nargs == OPTIONAL: + result = '[%s]' % get_metavar(1) + elif action.nargs == ZERO_OR_MORE: + result = '[%s [%s ...]]' % get_metavar(2) + elif action.nargs == ONE_OR_MORE: + result = '%s [%s ...]' % get_metavar(2) + elif action.nargs == REMAINDER: + result = '...' + elif action.nargs == PARSER: + result = '%s ...' % get_metavar(1) + else: + formats = ['%s' for _ in range(action.nargs)] + result = ' '.join(formats) % get_metavar(action.nargs) + return result + + def _expand_help(self, action): + params = dict(vars(action), prog=self._prog) + for name in list(params): + if params[name] is SUPPRESS: + del params[name] + for name in list(params): + if hasattr(params[name], '__name__'): + params[name] = params[name].__name__ + if params.get('choices') is not None: + choices_str = ', '.join([str(c) for c in params['choices']]) + params['choices'] = choices_str + return self._get_help_string(action) % params + + def _iter_indented_subactions(self, action): + try: + get_subactions = action._get_subactions + except AttributeError: + pass + else: + self._indent() + for subaction in get_subactions(): + yield subaction + self._dedent() + + def _split_lines(self, text, width): + text = self._whitespace_matcher.sub(' ', text).strip() + return _textwrap.wrap(text, width) + + def _fill_text(self, text, width, indent): + text = self._whitespace_matcher.sub(' ', text).strip() + return _textwrap.fill(text, width, initial_indent=indent, + subsequent_indent=indent) + + def _get_help_string(self, action): + return action.help + + +class RawDescriptionHelpFormatter(HelpFormatter): + """Help message formatter which retains any formatting in descriptions. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _fill_text(self, text, width, indent): + return ''.join([indent + line for line in text.splitlines(True)]) + + +class RawTextHelpFormatter(RawDescriptionHelpFormatter): + """Help message formatter which retains formatting of all help text. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _split_lines(self, text, width): + return text.splitlines() + + +class ArgumentDefaultsHelpFormatter(HelpFormatter): + """Help message formatter which adds default values to argument help. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _get_help_string(self, action): + help = action.help + if '%(default)' not in action.help: + if action.default is not SUPPRESS: + defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] + if action.option_strings or action.nargs in defaulting_nargs: + help += ' (default: %(default)s)' + return help + + +# ===================== +# Options and Arguments +# ===================== + +def _get_action_name(argument): + if argument is None: + return None + elif argument.option_strings: + return '/'.join(argument.option_strings) + elif argument.metavar not in (None, SUPPRESS): + return argument.metavar + elif argument.dest not in (None, SUPPRESS): + return argument.dest + else: + return None + + +class ArgumentError(Exception): + """An error from creating or using an argument (optional or positional). + + The string value of this exception is the message, augmented with + information about the argument that caused it. + """ + + def __init__(self, argument, message): + self.argument_name = _get_action_name(argument) + self.message = message + + def __str__(self): + if self.argument_name is None: + format = '%(message)s' + else: + format = 'argument %(argument_name)s: %(message)s' + return format % dict(message=self.message, + argument_name=self.argument_name) + + +class ArgumentTypeError(Exception): + """An error from trying to convert a command line string to a type.""" + pass + + +# ============== +# Action classes +# ============== + +class Action(_AttributeHolder): + """Information about how to convert command line strings to Python objects. + + Action objects are used by an ArgumentParser to represent the information + needed to parse a single argument from one or more strings from the + command line. The keyword arguments to the Action constructor are also + all attributes of Action instances. + + Keyword Arguments: + + - option_strings -- A list of command-line option strings which + should be associated with this action. + + - dest -- The name of the attribute to hold the created object(s) + + - nargs -- The number of command-line arguments that should be + consumed. By default, one argument will be consumed and a single + value will be produced. Other values include: + - N (an integer) consumes N arguments (and produces a list) + - '?' consumes zero or one arguments + - '*' consumes zero or more arguments (and produces a list) + - '+' consumes one or more arguments (and produces a list) + Note that the difference between the default and nargs=1 is that + with the default, a single value will be produced, while with + nargs=1, a list containing a single value will be produced. + + - const -- The value to be produced if the option is specified and the + option uses an action that takes no values. + + - default -- The value to be produced if the option is not specified. + + - type -- The type which the command-line arguments should be converted + to, should be one of 'string', 'int', 'float', 'complex' or a + callable object that accepts a single string argument. If None, + 'string' is assumed. + + - choices -- A container of values that should be allowed. If not None, + after a command-line argument has been converted to the appropriate + type, an exception will be raised if it is not a member of this + collection. + + - required -- True if the action must always be specified at the + command line. This is only meaningful for optional command-line + arguments. + + - help -- The help string describing the argument. + + - metavar -- The name to be used for the option's argument with the + help string. If None, the 'dest' value will be used as the name. + """ + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + self.option_strings = option_strings + self.dest = dest + self.nargs = nargs + self.const = const + self.default = default + self.type = type + self.choices = choices + self.required = required + self.help = help + self.metavar = metavar + + def _get_kwargs(self): + names = [ + 'option_strings', + 'dest', + 'nargs', + 'const', + 'default', + 'type', + 'choices', + 'help', + 'metavar', + ] + return [(name, getattr(self, name)) for name in names] + + def __call__(self, parser, namespace, values, option_string=None): + raise NotImplementedError(_('.__call__() not defined')) + + +class _StoreAction(Action): + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + if nargs == 0: + raise ValueError('nargs for store actions must be > 0; if you ' + 'have nothing to store, actions such as store ' + 'true or store const may be more appropriate') + if const is not None and nargs != OPTIONAL: + raise ValueError('nargs must be %r to supply const' % OPTIONAL) + super(_StoreAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + const=const, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values) + + +class _StoreConstAction(Action): + + def __init__(self, + option_strings, + dest, + const, + default=None, + required=False, + help=None, + metavar=None): + super(_StoreConstAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + const=const, + default=default, + required=required, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, self.const) + + +class _StoreTrueAction(_StoreConstAction): + + def __init__(self, + option_strings, + dest, + default=False, + required=False, + help=None): + super(_StoreTrueAction, self).__init__( + option_strings=option_strings, + dest=dest, + const=True, + default=default, + required=required, + help=help) + + +class _StoreFalseAction(_StoreConstAction): + + def __init__(self, + option_strings, + dest, + default=True, + required=False, + help=None): + super(_StoreFalseAction, self).__init__( + option_strings=option_strings, + dest=dest, + const=False, + default=default, + required=required, + help=help) + + +class _AppendAction(Action): + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + if nargs == 0: + raise ValueError('nargs for append actions must be > 0; if arg ' + 'strings are not supplying the value to append, ' + 'the append const action may be more appropriate') + if const is not None and nargs != OPTIONAL: + raise ValueError('nargs must be %r to supply const' % OPTIONAL) + super(_AppendAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + const=const, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + items = _copy.copy(_ensure_value(namespace, self.dest, [])) + items.append(values) + setattr(namespace, self.dest, items) + + +class _AppendConstAction(Action): + + def __init__(self, + option_strings, + dest, + const, + default=None, + required=False, + help=None, + metavar=None): + super(_AppendConstAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + const=const, + default=default, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + items = _copy.copy(_ensure_value(namespace, self.dest, [])) + items.append(self.const) + setattr(namespace, self.dest, items) + + +class _CountAction(Action): + + def __init__(self, + option_strings, + dest, + default=None, + required=False, + help=None): + super(_CountAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + default=default, + required=required, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + new_count = _ensure_value(namespace, self.dest, 0) + 1 + setattr(namespace, self.dest, new_count) + + +class _HelpAction(Action): + + def __init__(self, + option_strings, + dest=SUPPRESS, + default=SUPPRESS, + help=None): + super(_HelpAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + parser.print_help() + parser.exit() + + +class _VersionAction(Action): + + def __init__(self, + option_strings, + version=None, + dest=SUPPRESS, + default=SUPPRESS, + help="show program's version number and exit"): + super(_VersionAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + self.version = version + + def __call__(self, parser, namespace, values, option_string=None): + version = self.version + if version is None: + version = parser.version + formatter = parser._get_formatter() + formatter.add_text(version) + parser.exit(message=formatter.format_help()) + + +class _SubParsersAction(Action): + + class _ChoicesPseudoAction(Action): + + def __init__(self, name, help): + sup = super(_SubParsersAction._ChoicesPseudoAction, self) + sup.__init__(option_strings=[], dest=name, help=help) + + def __init__(self, + option_strings, + prog, + parser_class, + dest=SUPPRESS, + help=None, + metavar=None): + + self._prog_prefix = prog + self._parser_class = parser_class + self._name_parser_map = {} + self._choices_actions = [] + + super(_SubParsersAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=PARSER, + choices=self._name_parser_map, + help=help, + metavar=metavar) + + def add_parser(self, name, **kwargs): + # set prog from the existing prefix + if kwargs.get('prog') is None: + kwargs['prog'] = '%s %s' % (self._prog_prefix, name) + + # create a pseudo-action to hold the choice help + if 'help' in kwargs: + help = kwargs.pop('help') + choice_action = self._ChoicesPseudoAction(name, help) + self._choices_actions.append(choice_action) + + # create the parser and add it to the map + parser = self._parser_class(**kwargs) + self._name_parser_map[name] = parser + return parser + + def _get_subactions(self): + return self._choices_actions + + def __call__(self, parser, namespace, values, option_string=None): + parser_name = values[0] + arg_strings = values[1:] + + # set the parser name if requested + if self.dest is not SUPPRESS: + setattr(namespace, self.dest, parser_name) + + # select the parser + try: + parser = self._name_parser_map[parser_name] + except KeyError: + tup = parser_name, ', '.join(self._name_parser_map) + msg = _('unknown parser %r (choices: %s)' % tup) + raise ArgumentError(self, msg) + + # parse all the remaining options into the namespace + # store any unrecognized options on the object, so that the top + # level parser can decide what to do with them + namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) + if arg_strings: + vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) + getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) + + +# ============== +# Type classes +# ============== + +class FileType(object): + """Factory for creating file object types + + Instances of FileType are typically passed as type= arguments to the + ArgumentParser add_argument() method. + + Keyword Arguments: + - mode -- A string indicating how the file is to be opened. Accepts the + same values as the builtin open() function. + - bufsize -- The file's desired buffer size. Accepts the same values as + the builtin open() function. + """ + + def __init__(self, mode='r', bufsize=None): + self._mode = mode + self._bufsize = bufsize + + def __call__(self, string): + # the special argument "-" means sys.std{in,out} + if string == '-': + if 'r' in self._mode: + return _sys.stdin + elif 'w' in self._mode: + return _sys.stdout + else: + msg = _('argument "-" with mode %r' % self._mode) + raise ValueError(msg) + + # all other arguments are used as file names + if self._bufsize: + return open(string, self._mode, self._bufsize) + else: + return open(string, self._mode) + + def __repr__(self): + args = [self._mode, self._bufsize] + args_str = ', '.join([repr(arg) for arg in args if arg is not None]) + return '%s(%s)' % (type(self).__name__, args_str) + +# =========================== +# Optional and Positional Parsing +# =========================== + +class Namespace(_AttributeHolder): + """Simple object for storing attributes. + + Implements equality by attribute names and values, and provides a simple + string representation. + """ + + def __init__(self, **kwargs): + for name in kwargs: + setattr(self, name, kwargs[name]) + + __hash__ = None + + def __eq__(self, other): + return vars(self) == vars(other) + + def __ne__(self, other): + return not (self == other) + + def __contains__(self, key): + return key in self.__dict__ + + +class _ActionsContainer(object): + + def __init__(self, + description, + prefix_chars, + argument_default, + conflict_handler): + super(_ActionsContainer, self).__init__() + + self.description = description + self.argument_default = argument_default + self.prefix_chars = prefix_chars + self.conflict_handler = conflict_handler + + # set up registries + self._registries = {} + + # register actions + self.register('action', None, _StoreAction) + self.register('action', 'store', _StoreAction) + self.register('action', 'store_const', _StoreConstAction) + self.register('action', 'store_true', _StoreTrueAction) + self.register('action', 'store_false', _StoreFalseAction) + self.register('action', 'append', _AppendAction) + self.register('action', 'append_const', _AppendConstAction) + self.register('action', 'count', _CountAction) + self.register('action', 'help', _HelpAction) + self.register('action', 'version', _VersionAction) + self.register('action', 'parsers', _SubParsersAction) + + # raise an exception if the conflict handler is invalid + self._get_handler() + + # action storage + self._actions = [] + self._option_string_actions = {} + + # groups + self._action_groups = [] + self._mutually_exclusive_groups = [] + + # defaults storage + self._defaults = {} + + # determines whether an "option" looks like a negative number + self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') + + # whether or not there are any optionals that look like negative + # numbers -- uses a list so it can be shared and edited + self._has_negative_number_optionals = [] + + # ==================== + # Registration methods + # ==================== + def register(self, registry_name, value, object): + registry = self._registries.setdefault(registry_name, {}) + registry[value] = object + + def _registry_get(self, registry_name, value, default=None): + return self._registries[registry_name].get(value, default) + + # ================================== + # Namespace default accessor methods + # ================================== + def set_defaults(self, **kwargs): + self._defaults.update(kwargs) + + # if these defaults match any existing arguments, replace + # the previous default on the object with the new one + for action in self._actions: + if action.dest in kwargs: + action.default = kwargs[action.dest] + + def get_default(self, dest): + for action in self._actions: + if action.dest == dest and action.default is not None: + return action.default + return self._defaults.get(dest, None) + + + # ======================= + # Adding argument actions + # ======================= + def add_argument(self, *args, **kwargs): + """ + add_argument(dest, ..., name=value, ...) + add_argument(option_string, option_string, ..., name=value, ...) + """ + + # if no positional args are supplied or only one is supplied and + # it doesn't look like an option string, parse a positional + # argument + chars = self.prefix_chars + if not args or len(args) == 1 and args[0][0] not in chars: + if args and 'dest' in kwargs: + raise ValueError('dest supplied twice for positional argument') + kwargs = self._get_positional_kwargs(*args, **kwargs) + + # otherwise, we're adding an optional argument + else: + kwargs = self._get_optional_kwargs(*args, **kwargs) + + # if no default was supplied, use the parser-level default + if 'default' not in kwargs: + dest = kwargs['dest'] + if dest in self._defaults: + kwargs['default'] = self._defaults[dest] + elif self.argument_default is not None: + kwargs['default'] = self.argument_default + + # create the action object, and add it to the parser + action_class = self._pop_action_class(kwargs) + if not _callable(action_class): + raise ValueError('unknown action "%s"' % action_class) + action = action_class(**kwargs) + + # raise an error if the action type is not callable + type_func = self._registry_get('type', action.type, action.type) + if not _callable(type_func): + raise ValueError('%r is not callable' % type_func) + + return self._add_action(action) + + def add_argument_group(self, *args, **kwargs): + group = _ArgumentGroup(self, *args, **kwargs) + self._action_groups.append(group) + return group + + def add_mutually_exclusive_group(self, **kwargs): + group = _MutuallyExclusiveGroup(self, **kwargs) + self._mutually_exclusive_groups.append(group) + return group + + def _add_action(self, action): + # resolve any conflicts + self._check_conflict(action) + + # add to actions list + self._actions.append(action) + action.container = self + + # index the action by any option strings it has + for option_string in action.option_strings: + self._option_string_actions[option_string] = action + + # set the flag if any option strings look like negative numbers + for option_string in action.option_strings: + if self._negative_number_matcher.match(option_string): + if not self._has_negative_number_optionals: + self._has_negative_number_optionals.append(True) + + # return the created action + return action + + def _remove_action(self, action): + self._actions.remove(action) + + def _add_container_actions(self, container): + # collect groups by titles + title_group_map = {} + for group in self._action_groups: + if group.title in title_group_map: + msg = _('cannot merge actions - two groups are named %r') + raise ValueError(msg % (group.title)) + title_group_map[group.title] = group + + # map each action to its group + group_map = {} + for group in container._action_groups: + + # if a group with the title exists, use that, otherwise + # create a new group matching the container's group + if group.title not in title_group_map: + title_group_map[group.title] = self.add_argument_group( + title=group.title, + description=group.description, + conflict_handler=group.conflict_handler) + + # map the actions to their new group + for action in group._group_actions: + group_map[action] = title_group_map[group.title] + + # add container's mutually exclusive groups + # NOTE: if add_mutually_exclusive_group ever gains title= and + # description= then this code will need to be expanded as above + for group in container._mutually_exclusive_groups: + mutex_group = self.add_mutually_exclusive_group( + required=group.required) + + # map the actions to their new mutex group + for action in group._group_actions: + group_map[action] = mutex_group + + # add all actions to this container or their group + for action in container._actions: + group_map.get(action, self)._add_action(action) + + def _get_positional_kwargs(self, dest, **kwargs): + # make sure required is not specified + if 'required' in kwargs: + msg = _("'required' is an invalid argument for positionals") + raise TypeError(msg) + + # mark positional arguments as required if at least one is + # always required + if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: + kwargs['required'] = True + if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: + kwargs['required'] = True + + # return the keyword arguments with no option strings + return dict(kwargs, dest=dest, option_strings=[]) + + def _get_optional_kwargs(self, *args, **kwargs): + # determine short and long option strings + option_strings = [] + long_option_strings = [] + for option_string in args: + # error on strings that don't start with an appropriate prefix + if not option_string[0] in self.prefix_chars: + msg = _('invalid option string %r: ' + 'must start with a character %r') + tup = option_string, self.prefix_chars + raise ValueError(msg % tup) + + # strings starting with two prefix characters are long options + option_strings.append(option_string) + if option_string[0] in self.prefix_chars: + if len(option_string) > 1: + if option_string[1] in self.prefix_chars: + long_option_strings.append(option_string) + + # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' + dest = kwargs.pop('dest', None) + if dest is None: + if long_option_strings: + dest_option_string = long_option_strings[0] + else: + dest_option_string = option_strings[0] + dest = dest_option_string.lstrip(self.prefix_chars) + if not dest: + msg = _('dest= is required for options like %r') + raise ValueError(msg % option_string) + dest = dest.replace('-', '_') + + # return the updated keyword arguments + return dict(kwargs, dest=dest, option_strings=option_strings) + + def _pop_action_class(self, kwargs, default=None): + action = kwargs.pop('action', default) + return self._registry_get('action', action, action) + + def _get_handler(self): + # determine function from conflict handler string + handler_func_name = '_handle_conflict_%s' % self.conflict_handler + try: + return getattr(self, handler_func_name) + except AttributeError: + msg = _('invalid conflict_resolution value: %r') + raise ValueError(msg % self.conflict_handler) + + def _check_conflict(self, action): + + # find all options that conflict with this option + confl_optionals = [] + for option_string in action.option_strings: + if option_string in self._option_string_actions: + confl_optional = self._option_string_actions[option_string] + confl_optionals.append((option_string, confl_optional)) + + # resolve any conflicts + if confl_optionals: + conflict_handler = self._get_handler() + conflict_handler(action, confl_optionals) + + def _handle_conflict_error(self, action, conflicting_actions): + message = _('conflicting option string(s): %s') + conflict_string = ', '.join([option_string + for option_string, action + in conflicting_actions]) + raise ArgumentError(action, message % conflict_string) + + def _handle_conflict_resolve(self, action, conflicting_actions): + + # remove all conflicting options + for option_string, action in conflicting_actions: + + # remove the conflicting option + action.option_strings.remove(option_string) + self._option_string_actions.pop(option_string, None) + + # if the option now has no option string, remove it from the + # container holding it + if not action.option_strings: + action.container._remove_action(action) + + +class _ArgumentGroup(_ActionsContainer): + + def __init__(self, container, title=None, description=None, **kwargs): + # add any missing keyword arguments by checking the container + update = kwargs.setdefault + update('conflict_handler', container.conflict_handler) + update('prefix_chars', container.prefix_chars) + update('argument_default', container.argument_default) + super_init = super(_ArgumentGroup, self).__init__ + super_init(description=description, **kwargs) + + # group attributes + self.title = title + self._group_actions = [] + + # share most attributes with the container + self._registries = container._registries + self._actions = container._actions + self._option_string_actions = container._option_string_actions + self._defaults = container._defaults + self._has_negative_number_optionals = \ + container._has_negative_number_optionals + + def _add_action(self, action): + action = super(_ArgumentGroup, self)._add_action(action) + self._group_actions.append(action) + return action + + def _remove_action(self, action): + super(_ArgumentGroup, self)._remove_action(action) + self._group_actions.remove(action) + + +class _MutuallyExclusiveGroup(_ArgumentGroup): + + def __init__(self, container, required=False): + super(_MutuallyExclusiveGroup, self).__init__(container) + self.required = required + self._container = container + + def _add_action(self, action): + if action.required: + msg = _('mutually exclusive arguments must be optional') + raise ValueError(msg) + action = self._container._add_action(action) + self._group_actions.append(action) + return action + + def _remove_action(self, action): + self._container._remove_action(action) + self._group_actions.remove(action) + + +class ArgumentParser(_AttributeHolder, _ActionsContainer): + """Object for parsing command line strings into Python objects. + + Keyword Arguments: + - prog -- The name of the program (default: sys.argv[0]) + - usage -- A usage message (default: auto-generated from arguments) + - description -- A description of what the program does + - epilog -- Text following the argument descriptions + - parents -- Parsers whose arguments should be copied into this one + - formatter_class -- HelpFormatter class for printing help messages + - prefix_chars -- Characters that prefix optional arguments + - fromfile_prefix_chars -- Characters that prefix files containing + additional arguments + - argument_default -- The default value for all arguments + - conflict_handler -- String indicating how to handle conflicts + - add_help -- Add a -h/-help option + """ + + def __init__(self, + prog=None, + usage=None, + description=None, + epilog=None, + version=None, + parents=[], + formatter_class=HelpFormatter, + prefix_chars='-', + fromfile_prefix_chars=None, + argument_default=None, + conflict_handler='error', + add_help=True): + + if version is not None: + import warnings + warnings.warn( + """The "version" argument to ArgumentParser is deprecated. """ + """Please use """ + """"add_argument(..., action='version', version="N", ...)" """ + """instead""", DeprecationWarning) + + superinit = super(ArgumentParser, self).__init__ + superinit(description=description, + prefix_chars=prefix_chars, + argument_default=argument_default, + conflict_handler=conflict_handler) + + # default setting for prog + if prog is None: + prog = _os.path.basename(_sys.argv[0]) + + self.prog = prog + self.usage = usage + self.epilog = epilog + self.version = version + self.formatter_class = formatter_class + self.fromfile_prefix_chars = fromfile_prefix_chars + self.add_help = add_help + + add_group = self.add_argument_group + self._positionals = add_group(_('positional arguments')) + self._optionals = add_group(_('optional arguments')) + self._subparsers = None + + # register types + def identity(string): + return string + self.register('type', None, identity) + + # add help and version arguments if necessary + # (using explicit default to override global argument_default) + if '-' in prefix_chars: + default_prefix = '-' + else: + default_prefix = prefix_chars[0] + if self.add_help: + self.add_argument( + default_prefix+'h', default_prefix*2+'help', + action='help', default=SUPPRESS, + help=_('show this help message and exit')) + if self.version: + self.add_argument( + default_prefix+'v', default_prefix*2+'version', + action='version', default=SUPPRESS, + version=self.version, + help=_("show program's version number and exit")) + + # add parent arguments and defaults + for parent in parents: + self._add_container_actions(parent) + try: + defaults = parent._defaults + except AttributeError: + pass + else: + self._defaults.update(defaults) + + # ======================= + # Pretty __repr__ methods + # ======================= + def _get_kwargs(self): + names = [ + 'prog', + 'usage', + 'description', + 'version', + 'formatter_class', + 'conflict_handler', + 'add_help', + ] + return [(name, getattr(self, name)) for name in names] + + # ================================== + # Optional/Positional adding methods + # ================================== + def add_subparsers(self, **kwargs): + if self._subparsers is not None: + self.error(_('cannot have multiple subparser arguments')) + + # add the parser class to the arguments if it's not present + kwargs.setdefault('parser_class', type(self)) + + if 'title' in kwargs or 'description' in kwargs: + title = _(kwargs.pop('title', 'subcommands')) + description = _(kwargs.pop('description', None)) + self._subparsers = self.add_argument_group(title, description) + else: + self._subparsers = self._positionals + + # prog defaults to the usage message of this parser, skipping + # optional arguments and with no "usage:" prefix + if kwargs.get('prog') is None: + formatter = self._get_formatter() + positionals = self._get_positional_actions() + groups = self._mutually_exclusive_groups + formatter.add_usage(self.usage, positionals, groups, '') + kwargs['prog'] = formatter.format_help().strip() + + # create the parsers action and add it to the positionals list + parsers_class = self._pop_action_class(kwargs, 'parsers') + action = parsers_class(option_strings=[], **kwargs) + self._subparsers._add_action(action) + + # return the created parsers action + return action + + def _add_action(self, action): + if action.option_strings: + self._optionals._add_action(action) + else: + self._positionals._add_action(action) + return action + + def _get_optional_actions(self): + return [action + for action in self._actions + if action.option_strings] + + def _get_positional_actions(self): + return [action + for action in self._actions + if not action.option_strings] + + # ===================================== + # Command line argument parsing methods + # ===================================== + def parse_args(self, args=None, namespace=None): + args, argv = self.parse_known_args(args, namespace) + if argv: + msg = _('unrecognized arguments: %s') + self.error(msg % ' '.join(argv)) + return args + + def parse_known_args(self, args=None, namespace=None): + # args default to the system args + if args is None: + args = _sys.argv[1:] + + # default Namespace built from parser defaults + if namespace is None: + namespace = Namespace() + + # add any action defaults that aren't present + for action in self._actions: + if action.dest is not SUPPRESS: + if not hasattr(namespace, action.dest): + if action.default is not SUPPRESS: + default = action.default + if isinstance(action.default, basestring): + default = self._get_value(action, default) + setattr(namespace, action.dest, default) + + # add any parser defaults that aren't present + for dest in self._defaults: + if not hasattr(namespace, dest): + setattr(namespace, dest, self._defaults[dest]) + + # parse the arguments and exit if there are any errors + try: + namespace, args = self._parse_known_args(args, namespace) + if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): + args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR)) + delattr(namespace, _UNRECOGNIZED_ARGS_ATTR) + return namespace, args + except ArgumentError: + err = _sys.exc_info()[1] + self.error(str(err)) + + def _parse_known_args(self, arg_strings, namespace): + # replace arg strings that are file references + if self.fromfile_prefix_chars is not None: + arg_strings = self._read_args_from_files(arg_strings) + + # map all mutually exclusive arguments to the other arguments + # they can't occur with + action_conflicts = {} + for mutex_group in self._mutually_exclusive_groups: + group_actions = mutex_group._group_actions + for i, mutex_action in enumerate(mutex_group._group_actions): + conflicts = action_conflicts.setdefault(mutex_action, []) + conflicts.extend(group_actions[:i]) + conflicts.extend(group_actions[i + 1:]) + + # find all option indices, and determine the arg_string_pattern + # which has an 'O' if there is an option at an index, + # an 'A' if there is an argument, or a '-' if there is a '--' + option_string_indices = {} + arg_string_pattern_parts = [] + arg_strings_iter = iter(arg_strings) + for i, arg_string in enumerate(arg_strings_iter): + + # all args after -- are non-options + if arg_string == '--': + arg_string_pattern_parts.append('-') + for arg_string in arg_strings_iter: + arg_string_pattern_parts.append('A') + + # otherwise, add the arg to the arg strings + # and note the index if it was an option + else: + option_tuple = self._parse_optional(arg_string) + if option_tuple is None: + pattern = 'A' + else: + option_string_indices[i] = option_tuple + pattern = 'O' + arg_string_pattern_parts.append(pattern) + + # join the pieces together to form the pattern + arg_strings_pattern = ''.join(arg_string_pattern_parts) + + # converts arg strings to the appropriate and then takes the action + seen_actions = set() + seen_non_default_actions = set() + + def take_action(action, argument_strings, option_string=None): + seen_actions.add(action) + argument_values = self._get_values(action, argument_strings) + + # error if this argument is not allowed with other previously + # seen arguments, assuming that actions that use the default + # value don't really count as "present" + if argument_values is not action.default: + seen_non_default_actions.add(action) + for conflict_action in action_conflicts.get(action, []): + if conflict_action in seen_non_default_actions: + msg = _('not allowed with argument %s') + action_name = _get_action_name(conflict_action) + raise ArgumentError(action, msg % action_name) + + # take the action if we didn't receive a SUPPRESS value + # (e.g. from a default) + if argument_values is not SUPPRESS: + action(self, namespace, argument_values, option_string) + + # function to convert arg_strings into an optional action + def consume_optional(start_index): + + # get the optional identified at this index + option_tuple = option_string_indices[start_index] + action, option_string, explicit_arg = option_tuple + + # identify additional optionals in the same arg string + # (e.g. -xyz is the same as -x -y -z if no args are required) + match_argument = self._match_argument + action_tuples = [] + while True: + + # if we found no optional action, skip it + if action is None: + extras.append(arg_strings[start_index]) + return start_index + 1 + + # if there is an explicit argument, try to match the + # optional's string arguments to only this + if explicit_arg is not None: + arg_count = match_argument(action, 'A') + + # if the action is a single-dash option and takes no + # arguments, try to parse more single-dash options out + # of the tail of the option string + chars = self.prefix_chars + if arg_count == 0 and option_string[1] not in chars: + action_tuples.append((action, [], option_string)) + char = option_string[0] + option_string = char + explicit_arg[0] + new_explicit_arg = explicit_arg[1:] or None + optionals_map = self._option_string_actions + if option_string in optionals_map: + action = optionals_map[option_string] + explicit_arg = new_explicit_arg + else: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) + + # if the action expect exactly one argument, we've + # successfully matched the option; exit the loop + elif arg_count == 1: + stop = start_index + 1 + args = [explicit_arg] + action_tuples.append((action, args, option_string)) + break + + # error if a double-dash option did not use the + # explicit argument + else: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) + + # if there is no explicit argument, try to match the + # optional's string arguments with the following strings + # if successful, exit the loop + else: + start = start_index + 1 + selected_patterns = arg_strings_pattern[start:] + arg_count = match_argument(action, selected_patterns) + stop = start + arg_count + args = arg_strings[start:stop] + action_tuples.append((action, args, option_string)) + break + + # add the Optional to the list and return the index at which + # the Optional's string args stopped + assert action_tuples + for action, args, option_string in action_tuples: + take_action(action, args, option_string) + return stop + + # the list of Positionals left to be parsed; this is modified + # by consume_positionals() + positionals = self._get_positional_actions() + + # function to convert arg_strings into positional actions + def consume_positionals(start_index): + # match as many Positionals as possible + match_partial = self._match_arguments_partial + selected_pattern = arg_strings_pattern[start_index:] + arg_counts = match_partial(positionals, selected_pattern) + + # slice off the appropriate arg strings for each Positional + # and add the Positional and its args to the list + for action, arg_count in zip(positionals, arg_counts): + args = arg_strings[start_index: start_index + arg_count] + start_index += arg_count + take_action(action, args) + + # slice off the Positionals that we just parsed and return the + # index at which the Positionals' string args stopped + positionals[:] = positionals[len(arg_counts):] + return start_index + + # consume Positionals and Optionals alternately, until we have + # passed the last option string + extras = [] + start_index = 0 + if option_string_indices: + max_option_string_index = max(option_string_indices) + else: + max_option_string_index = -1 + while start_index <= max_option_string_index: + + # consume any Positionals preceding the next option + next_option_string_index = min([ + index + for index in option_string_indices + if index >= start_index]) + if start_index != next_option_string_index: + positionals_end_index = consume_positionals(start_index) + + # only try to parse the next optional if we didn't consume + # the option string during the positionals parsing + if positionals_end_index > start_index: + start_index = positionals_end_index + continue + else: + start_index = positionals_end_index + + # if we consumed all the positionals we could and we're not + # at the index of an option string, there were extra arguments + if start_index not in option_string_indices: + strings = arg_strings[start_index:next_option_string_index] + extras.extend(strings) + start_index = next_option_string_index + + # consume the next optional and any arguments for it + start_index = consume_optional(start_index) + + # consume any positionals following the last Optional + stop_index = consume_positionals(start_index) + + # if we didn't consume all the argument strings, there were extras + extras.extend(arg_strings[stop_index:]) + + # if we didn't use all the Positional objects, there were too few + # arg strings supplied. + if positionals: + self.error(_('too few arguments')) + + # make sure all required actions were present + for action in self._actions: + if action.required: + if action not in seen_actions: + name = _get_action_name(action) + self.error(_('argument %s is required') % name) + + # make sure all required groups had one option present + for group in self._mutually_exclusive_groups: + if group.required: + for action in group._group_actions: + if action in seen_non_default_actions: + break + + # if no actions were used, report the error + else: + names = [_get_action_name(action) + for action in group._group_actions + if action.help is not SUPPRESS] + msg = _('one of the arguments %s is required') + self.error(msg % ' '.join(names)) + + # return the updated namespace and the extra arguments + return namespace, extras + + def _read_args_from_files(self, arg_strings): + # expand arguments referencing files + new_arg_strings = [] + for arg_string in arg_strings: + + # for regular arguments, just add them back into the list + if arg_string[0] not in self.fromfile_prefix_chars: + new_arg_strings.append(arg_string) + + # replace arguments referencing files with the file content + else: + try: + args_file = open(arg_string[1:]) + try: + arg_strings = [] + for arg_line in args_file.read().splitlines(): + for arg in self.convert_arg_line_to_args(arg_line): + arg_strings.append(arg) + arg_strings = self._read_args_from_files(arg_strings) + new_arg_strings.extend(arg_strings) + finally: + args_file.close() + except IOError: + err = _sys.exc_info()[1] + self.error(str(err)) + + # return the modified argument list + return new_arg_strings + + def convert_arg_line_to_args(self, arg_line): + return [arg_line] + + def _match_argument(self, action, arg_strings_pattern): + # match the pattern for this action to the arg strings + nargs_pattern = self._get_nargs_pattern(action) + match = _re.match(nargs_pattern, arg_strings_pattern) + + # raise an exception if we weren't able to find a match + if match is None: + nargs_errors = { + None: _('expected one argument'), + OPTIONAL: _('expected at most one argument'), + ONE_OR_MORE: _('expected at least one argument'), + } + default = _('expected %s argument(s)') % action.nargs + msg = nargs_errors.get(action.nargs, default) + raise ArgumentError(action, msg) + + # return the number of arguments matched + return len(match.group(1)) + + def _match_arguments_partial(self, actions, arg_strings_pattern): + # progressively shorten the actions list by slicing off the + # final actions until we find a match + result = [] + for i in range(len(actions), 0, -1): + actions_slice = actions[:i] + pattern = ''.join([self._get_nargs_pattern(action) + for action in actions_slice]) + match = _re.match(pattern, arg_strings_pattern) + if match is not None: + result.extend([len(string) for string in match.groups()]) + break + + # return the list of arg string counts + return result + + def _parse_optional(self, arg_string): + # if it's an empty string, it was meant to be a positional + if not arg_string: + return None + + # if it doesn't start with a prefix, it was meant to be positional + if not arg_string[0] in self.prefix_chars: + return None + + # if the option string is present in the parser, return the action + if arg_string in self._option_string_actions: + action = self._option_string_actions[arg_string] + return action, arg_string, None + + # if it's just a single character, it was meant to be positional + if len(arg_string) == 1: + return None + + # if the option string before the "=" is present, return the action + if '=' in arg_string: + option_string, explicit_arg = arg_string.split('=', 1) + if option_string in self._option_string_actions: + action = self._option_string_actions[option_string] + return action, option_string, explicit_arg + + # search through all possible prefixes of the option string + # and all actions in the parser for possible interpretations + option_tuples = self._get_option_tuples(arg_string) + + # if multiple actions match, the option string was ambiguous + if len(option_tuples) > 1: + options = ', '.join([option_string + for action, option_string, explicit_arg in option_tuples]) + tup = arg_string, options + self.error(_('ambiguous option: %s could match %s') % tup) + + # if exactly one action matched, this segmentation is good, + # so return the parsed action + elif len(option_tuples) == 1: + option_tuple, = option_tuples + return option_tuple + + # if it was not found as an option, but it looks like a negative + # number, it was meant to be positional + # unless there are negative-number-like options + if self._negative_number_matcher.match(arg_string): + if not self._has_negative_number_optionals: + return None + + # if it contains a space, it was meant to be a positional + if ' ' in arg_string: + return None + + # it was meant to be an optional but there is no such option + # in this parser (though it might be a valid option in a subparser) + return None, arg_string, None + + def _get_option_tuples(self, option_string): + result = [] + + # option strings starting with two prefix characters are only + # split at the '=' + chars = self.prefix_chars + if option_string[0] in chars and option_string[1] in chars: + if '=' in option_string: + option_prefix, explicit_arg = option_string.split('=', 1) + else: + option_prefix = option_string + explicit_arg = None + for option_string in self._option_string_actions: + if option_string.startswith(option_prefix): + action = self._option_string_actions[option_string] + tup = action, option_string, explicit_arg + result.append(tup) + + # single character options can be concatenated with their arguments + # but multiple character options always have to have their argument + # separate + elif option_string[0] in chars and option_string[1] not in chars: + option_prefix = option_string + explicit_arg = None + short_option_prefix = option_string[:2] + short_explicit_arg = option_string[2:] + + for option_string in self._option_string_actions: + if option_string == short_option_prefix: + action = self._option_string_actions[option_string] + tup = action, option_string, short_explicit_arg + result.append(tup) + elif option_string.startswith(option_prefix): + action = self._option_string_actions[option_string] + tup = action, option_string, explicit_arg + result.append(tup) + + # shouldn't ever get here + else: + self.error(_('unexpected option string: %s') % option_string) + + # return the collected option tuples + return result + + def _get_nargs_pattern(self, action): + # in all examples below, we have to allow for '--' args + # which are represented as '-' in the pattern + nargs = action.nargs + + # the default (None) is assumed to be a single argument + if nargs is None: + nargs_pattern = '(-*A-*)' + + # allow zero or one arguments + elif nargs == OPTIONAL: + nargs_pattern = '(-*A?-*)' + + # allow zero or more arguments + elif nargs == ZERO_OR_MORE: + nargs_pattern = '(-*[A-]*)' + + # allow one or more arguments + elif nargs == ONE_OR_MORE: + nargs_pattern = '(-*A[A-]*)' + + # allow any number of options or arguments + elif nargs == REMAINDER: + nargs_pattern = '([-AO]*)' + + # allow one argument followed by any number of options or arguments + elif nargs == PARSER: + nargs_pattern = '(-*A[-AO]*)' + + # all others should be integers + else: + nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) + + # if this is an optional action, -- is not allowed + if action.option_strings: + nargs_pattern = nargs_pattern.replace('-*', '') + nargs_pattern = nargs_pattern.replace('-', '') + + # return the pattern + return nargs_pattern + + # ======================== + # Value conversion methods + # ======================== + def _get_values(self, action, arg_strings): + # for everything but PARSER args, strip out '--' + if action.nargs not in [PARSER, REMAINDER]: + arg_strings = [s for s in arg_strings if s != '--'] + + # optional argument produces a default when not present + if not arg_strings and action.nargs == OPTIONAL: + if action.option_strings: + value = action.const + else: + value = action.default + if isinstance(value, basestring): + value = self._get_value(action, value) + self._check_value(action, value) + + # when nargs='*' on a positional, if there were no command-line + # args, use the default if it is anything other than None + elif (not arg_strings and action.nargs == ZERO_OR_MORE and + not action.option_strings): + if action.default is not None: + value = action.default + else: + value = arg_strings + self._check_value(action, value) + + # single argument or optional argument produces a single value + elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: + arg_string, = arg_strings + value = self._get_value(action, arg_string) + self._check_value(action, value) + + # REMAINDER arguments convert all values, checking none + elif action.nargs == REMAINDER: + value = [self._get_value(action, v) for v in arg_strings] + + # PARSER arguments convert all values, but check only the first + elif action.nargs == PARSER: + value = [self._get_value(action, v) for v in arg_strings] + self._check_value(action, value[0]) + + # all other types of nargs produce a list + else: + value = [self._get_value(action, v) for v in arg_strings] + for v in value: + self._check_value(action, v) + + # return the converted value + return value + + def _get_value(self, action, arg_string): + type_func = self._registry_get('type', action.type, action.type) + if not _callable(type_func): + msg = _('%r is not callable') + raise ArgumentError(action, msg % type_func) + + # convert the value to the appropriate type + try: + result = type_func(arg_string) + + # ArgumentTypeErrors indicate errors + except ArgumentTypeError: + name = getattr(action.type, '__name__', repr(action.type)) + msg = str(_sys.exc_info()[1]) + raise ArgumentError(action, msg) + + # TypeErrors or ValueErrors also indicate errors + except (TypeError, ValueError): + name = getattr(action.type, '__name__', repr(action.type)) + msg = _('invalid %s value: %r') + raise ArgumentError(action, msg % (name, arg_string)) + + # return the converted value + return result + + def _check_value(self, action, value): + # converted value must be one of the choices (if specified) + if action.choices is not None and value not in action.choices: + tup = value, ', '.join(map(repr, action.choices)) + msg = _('invalid choice: %r (choose from %s)') % tup + raise ArgumentError(action, msg) + + # ======================= + # Help-formatting methods + # ======================= + def format_usage(self): + formatter = self._get_formatter() + formatter.add_usage(self.usage, self._actions, + self._mutually_exclusive_groups) + return formatter.format_help() + + def format_help(self): + formatter = self._get_formatter() + + # usage + formatter.add_usage(self.usage, self._actions, + self._mutually_exclusive_groups) + + # description + formatter.add_text(self.description) + + # positionals, optionals and user-defined groups + for action_group in self._action_groups: + formatter.start_section(action_group.title) + formatter.add_text(action_group.description) + formatter.add_arguments(action_group._group_actions) + formatter.end_section() + + # epilog + formatter.add_text(self.epilog) + + # determine help from format above + return formatter.format_help() + + def format_version(self): + import warnings + warnings.warn( + 'The format_version method is deprecated -- the "version" ' + 'argument to ArgumentParser is no longer supported.', + DeprecationWarning) + formatter = self._get_formatter() + formatter.add_text(self.version) + return formatter.format_help() + + def _get_formatter(self): + return self.formatter_class(prog=self.prog) + + # ===================== + # Help-printing methods + # ===================== + def print_usage(self, file=None): + if file is None: + file = _sys.stdout + self._print_message(self.format_usage(), file) + + def print_help(self, file=None): + if file is None: + file = _sys.stdout + self._print_message(self.format_help(), file) + + def print_version(self, file=None): + import warnings + warnings.warn( + 'The print_version method is deprecated -- the "version" ' + 'argument to ArgumentParser is no longer supported.', + DeprecationWarning) + self._print_message(self.format_version(), file) + + def _print_message(self, message, file=None): + if message: + if file is None: + file = _sys.stderr + file.write(message) + + # =============== + # Exiting methods + # =============== + def exit(self, status=0, message=None): + if message: + self._print_message(message, _sys.stderr) + _sys.exit(status) + + def error(self, message): + """error(message: string) + + Prints a usage message incorporating the message to stderr and + exits. + + If you override this in a subclass, it should not return -- it + should either exit or raise an exception. + """ + self.print_usage(_sys.stderr) + self.exit(2, _('%s: error: %s\n') % (self.prog, message)) diff --git a/lib/spack/external/functools.py b/lib/spack/external/functools.py new file mode 100644 index 0000000000..19f0903c82 --- /dev/null +++ b/lib/spack/external/functools.py @@ -0,0 +1,30 @@ +# +# Backport of Python 2.7's total_ordering. +# + +def total_ordering(cls): + """Class decorator that fills in missing ordering methods""" + convert = { + '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)), + ('__le__', lambda self, other: self < other or self == other), + ('__ge__', lambda self, other: not self < other)], + '__le__': [('__ge__', lambda self, other: not self <= other or self == other), + ('__lt__', lambda self, other: self <= other and not self == other), + ('__gt__', lambda self, other: not self <= other)], + '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)), + ('__ge__', lambda self, other: self > other or self == other), + ('__le__', lambda self, other: not self > other)], + '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other), + ('__gt__', lambda self, other: self >= other and not self == other), + ('__lt__', lambda self, other: not self >= other)] + } + roots = set(dir(cls)) & set(convert) + if not roots: + raise ValueError('must define at least one ordering operation: < > <= >=') + root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__ + for opname, opfunc in convert[root]: + if opname not in roots: + opfunc.__name__ = opname + opfunc.__doc__ = getattr(int, opname).__doc__ + setattr(cls, opname, opfunc) + return cls diff --git a/lib/spack/external/ordereddict.py b/lib/spack/external/ordereddict.py new file mode 100644 index 0000000000..8ddad1477e --- /dev/null +++ b/lib/spack/external/ordereddict.py @@ -0,0 +1,262 @@ +# +# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. +# Passes Python2.7's test suite and incorporates all the latest updates. +# +# From http://code.activestate.com/recipes/576693-ordered-dictionary-for-py24/ +# This file is in the public domain, and has no particular license. +# +try: + from thread import get_ident as _get_ident +except ImportError: + from dummy_thread import get_ident as _get_ident + +try: + from _abcoll import KeysView, ValuesView, ItemsView +except ImportError: + pass + + +class OrderedDict(dict): + 'Dictionary that remembers insertion order' + # An inherited dict maps keys to values. + # The inherited dict provides __getitem__, __len__, __contains__, and get. + # The remaining methods are order-aware. + # Big-O running times for all methods are the same as for regular dictionaries. + + # The internal self.__map dictionary maps keys to links in a doubly linked list. + # The circular doubly linked list starts and ends with a sentinel element. + # The sentinel element never gets deleted (this simplifies the algorithm). + # Each link is stored as a list of length three: [PREV, NEXT, KEY]. + + def __init__(self, *args, **kwds): + '''Initialize an ordered dictionary. Signature is the same as for + regular dictionaries, but keyword arguments are not recommended + because their insertion order is arbitrary. + + ''' + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + try: + self.__root + except AttributeError: + self.__root = root = [] # sentinel node + root[:] = [root, root, None] + self.__map = {} + self.__update(*args, **kwds) + + def __setitem__(self, key, value, dict_setitem=dict.__setitem__): + 'od.__setitem__(i, y) <==> od[i]=y' + # Setting a new item creates a new link which goes at the end of the linked + # list, and the inherited dictionary is updated with the new key/value pair. + if key not in self: + root = self.__root + last = root[0] + last[1] = root[0] = self.__map[key] = [last, root, key] + dict_setitem(self, key, value) + + def __delitem__(self, key, dict_delitem=dict.__delitem__): + 'od.__delitem__(y) <==> del od[y]' + # Deleting an existing item uses self.__map to find the link which is + # then removed by updating the links in the predecessor and successor nodes. + dict_delitem(self, key) + link_prev, link_next, key = self.__map.pop(key) + link_prev[1] = link_next + link_next[0] = link_prev + + def __iter__(self): + 'od.__iter__() <==> iter(od)' + root = self.__root + curr = root[1] + while curr is not root: + yield curr[2] + curr = curr[1] + + def __reversed__(self): + 'od.__reversed__() <==> reversed(od)' + root = self.__root + curr = root[0] + while curr is not root: + yield curr[2] + curr = curr[0] + + def clear(self): + 'od.clear() -> None. Remove all items from od.' + try: + for node in self.__map.itervalues(): + del node[:] + root = self.__root + root[:] = [root, root, None] + self.__map.clear() + except AttributeError: + pass + dict.clear(self) + + def popitem(self, last=True): + '''od.popitem() -> (k, v), return and remove a (key, value) pair. + Pairs are returned in LIFO order if last is true or FIFO order if false. + + ''' + if not self: + raise KeyError('dictionary is empty') + root = self.__root + if last: + link = root[0] + link_prev = link[0] + link_prev[1] = root + root[0] = link_prev + else: + link = root[1] + link_next = link[1] + root[1] = link_next + link_next[0] = root + key = link[2] + del self.__map[key] + value = dict.pop(self, key) + return key, value + + # -- the following methods do not depend on the internal structure -- + + def keys(self): + 'od.keys() -> list of keys in od' + return list(self) + + def values(self): + 'od.values() -> list of values in od' + return [self[key] for key in self] + + def items(self): + 'od.items() -> list of (key, value) pairs in od' + return [(key, self[key]) for key in self] + + def iterkeys(self): + 'od.iterkeys() -> an iterator over the keys in od' + return iter(self) + + def itervalues(self): + 'od.itervalues -> an iterator over the values in od' + for k in self: + yield self[k] + + def iteritems(self): + 'od.iteritems -> an iterator over the (key, value) items in od' + for k in self: + yield (k, self[k]) + + def update(*args, **kwds): + '''od.update(E, **F) -> None. Update od from dict/iterable E and F. + + If E is a dict instance, does: for k in E: od[k] = E[k] + If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] + Or if E is an iterable of items, does: for k, v in E: od[k] = v + In either case, this is followed by: for k, v in F.items(): od[k] = v + + ''' + if len(args) > 2: + raise TypeError('update() takes at most 2 positional ' + 'arguments (%d given)' % (len(args),)) + elif not args: + raise TypeError('update() takes at least 1 argument (0 given)') + self = args[0] + # Make progressively weaker assumptions about "other" + other = () + if len(args) == 2: + other = args[1] + if isinstance(other, dict): + for key in other: + self[key] = other[key] + elif hasattr(other, 'keys'): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + __update = update # let subclasses override update without breaking __init__ + + __marker = object() + + def pop(self, key, default=__marker): + '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. + If key is not found, d is returned if given, otherwise KeyError is raised. + + ''' + if key in self: + result = self[key] + del self[key] + return result + if default is self.__marker: + raise KeyError(key) + return default + + def setdefault(self, key, default=None): + 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' + if key in self: + return self[key] + self[key] = default + return default + + def __repr__(self, _repr_running={}): + 'od.__repr__() <==> repr(od)' + call_key = id(self), _get_ident() + if call_key in _repr_running: + return '...' + _repr_running[call_key] = 1 + try: + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + finally: + del _repr_running[call_key] + + def __reduce__(self): + 'Return state information for pickling' + items = [[k, self[k]] for k in self] + inst_dict = vars(self).copy() + for k in vars(OrderedDict()): + inst_dict.pop(k, None) + if inst_dict: + return (self.__class__, (items,), inst_dict) + return self.__class__, (items,) + + def copy(self): + 'od.copy() -> a shallow copy of od' + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S + and values equal to v (which defaults to None). + + ''' + d = cls() + for key in iterable: + d[key] = value + return d + + def __eq__(self, other): + '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive + while comparison to a regular mapping is order-insensitive. + + ''' + if isinstance(other, OrderedDict): + return len(self)==len(other) and self.items() == other.items() + return dict.__eq__(self, other) + + def __ne__(self, other): + return not self == other + + # -- the following methods are only used in Python 2.7 -- + + def viewkeys(self): + "od.viewkeys() -> a set-like object providing a view on od's keys" + return KeysView(self) + + def viewvalues(self): + "od.viewvalues() -> an object providing a view on od's values" + return ValuesView(self) + + def viewitems(self): + "od.viewitems() -> a set-like object providing a view on od's items" + return ItemsView(self) diff --git a/lib/spack/external/pyqver2.py b/lib/spack/external/pyqver2.py new file mode 100755 index 0000000000..cd45bf948f --- /dev/null +++ b/lib/spack/external/pyqver2.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python +# +# pyqver2.py +# by Greg Hewgill +# https://github.com/ghewgill/pyqver +# +# This software is provided 'as-is', without any express or implied +# warranty. In no event will the author be held liable for any damages +# arising from the use of this software. +# +# Permission is granted to anyone to use this software for any purpose, +# including commercial applications, and to alter it and redistribute it +# freely, subject to the following restrictions: +# +# 1. The origin of this software must not be misrepresented; you must not +# claim that you wrote the original software. If you use this software +# in a product, an acknowledgment in the product documentation would be +# appreciated but is not required. +# 2. Altered source versions must be plainly marked as such, and must not be +# misrepresented as being the original software. +# 3. This notice may not be removed or altered from any source distribution. +# +# Copyright (c) 2009-2013 Greg Hewgill http://hewgill.com +# + +import compiler +import platform +import sys + +StandardModules = { + "__future__": (2, 1), + "abc": (2, 6), + "argparse": (2, 7), + "ast": (2, 6), + "atexit": (2, 0), + "bz2": (2, 3), + "cgitb": (2, 2), + "collections": (2, 4), + "contextlib": (2, 5), + "cookielib": (2, 4), + "cProfile": (2, 5), + "csv": (2, 3), + "ctypes": (2, 5), + "datetime": (2, 3), + "decimal": (2, 4), + "difflib": (2, 1), + "DocXMLRPCServer": (2, 3), + "dummy_thread": (2, 3), + "dummy_threading": (2, 3), + "email": (2, 2), + "fractions": (2, 6), + "functools": (2, 5), + "future_builtins": (2, 6), + "hashlib": (2, 5), + "heapq": (2, 3), + "hmac": (2, 2), + "hotshot": (2, 2), + "HTMLParser": (2, 2), + "importlib": (2, 7), + "inspect": (2, 1), + "io": (2, 6), + "itertools": (2, 3), + "json": (2, 6), + "logging": (2, 3), + "modulefinder": (2, 3), + "msilib": (2, 5), + "multiprocessing": (2, 6), + "netrc": (1, 5, 2), + "numbers": (2, 6), + "optparse": (2, 3), + "ossaudiodev": (2, 3), + "pickletools": (2, 3), + "pkgutil": (2, 3), + "platform": (2, 3), + "pydoc": (2, 1), + "runpy": (2, 5), + "sets": (2, 3), + "shlex": (1, 5, 2), + "SimpleXMLRPCServer": (2, 2), + "spwd": (2, 5), + "sqlite3": (2, 5), + "ssl": (2, 6), + "stringprep": (2, 3), + "subprocess": (2, 4), + "sysconfig": (2, 7), + "tarfile": (2, 3), + "textwrap": (2, 3), + "timeit": (2, 3), + "unittest": (2, 1), + "uuid": (2, 5), + "warnings": (2, 1), + "weakref": (2, 1), + "winsound": (1, 5, 2), + "wsgiref": (2, 5), + "xml.dom": (2, 0), + "xml.dom.minidom": (2, 0), + "xml.dom.pulldom": (2, 0), + "xml.etree.ElementTree": (2, 5), + "xml.parsers.expat":(2, 0), + "xml.sax": (2, 0), + "xml.sax.handler": (2, 0), + "xml.sax.saxutils": (2, 0), + "xml.sax.xmlreader":(2, 0), + "xmlrpclib": (2, 2), + "zipfile": (1, 6), + "zipimport": (2, 3), + "_ast": (2, 5), + "_winreg": (2, 0), +} + +Functions = { + "all": (2, 5), + "any": (2, 5), + "collections.Counter": (2, 7), + "collections.defaultdict": (2, 5), + "collections.OrderedDict": (2, 7), + "functools.total_ordering": (2, 7), + "enumerate": (2, 3), + "frozenset": (2, 4), + "itertools.compress": (2, 7), + "math.erf": (2, 7), + "math.erfc": (2, 7), + "math.expm1": (2, 7), + "math.gamma": (2, 7), + "math.lgamma": (2, 7), + "memoryview": (2, 7), + "next": (2, 6), + "os.getresgid": (2, 7), + "os.getresuid": (2, 7), + "os.initgroups": (2, 7), + "os.setresgid": (2, 7), + "os.setresuid": (2, 7), + "reversed": (2, 4), + "set": (2, 4), + "subprocess.check_call": (2, 5), + "subprocess.check_output": (2, 7), + "sum": (2, 3), + "symtable.is_declared_global": (2, 7), + "weakref.WeakSet": (2, 7), +} + +Identifiers = { + "False": (2, 2), + "True": (2, 2), +} + +def uniq(a): + if len(a) == 0: + return [] + else: + return [a[0]] + uniq([x for x in a if x != a[0]]) + +class NodeChecker(object): + def __init__(self): + self.vers = dict() + self.vers[(2,0)] = [] + def add(self, node, ver, msg): + if ver not in self.vers: + self.vers[ver] = [] + self.vers[ver].append((node.lineno, msg)) + def default(self, node): + for child in node.getChildNodes(): + self.visit(child) + def visitCallFunc(self, node): + def rollup(n): + if isinstance(n, compiler.ast.Name): + return n.name + elif isinstance(n, compiler.ast.Getattr): + r = rollup(n.expr) + if r: + return r + "." + n.attrname + name = rollup(node.node) + if name: + v = Functions.get(name) + if v is not None: + self.add(node, v, name) + self.default(node) + def visitClass(self, node): + if node.bases: + self.add(node, (2,2), "new-style class") + if node.decorators: + self.add(node, (2,6), "class decorator") + self.default(node) + def visitDictComp(self, node): + self.add(node, (2,7), "dictionary comprehension") + self.default(node) + def visitFloorDiv(self, node): + self.add(node, (2,2), "// operator") + self.default(node) + def visitFrom(self, node): + v = StandardModules.get(node.modname) + if v is not None: + self.add(node, v, node.modname) + for n in node.names: + name = node.modname + "." + n[0] + v = Functions.get(name) + if v is not None: + self.add(node, v, name) + def visitFunction(self, node): + if node.decorators: + self.add(node, (2,4), "function decorator") + self.default(node) + def visitGenExpr(self, node): + self.add(node, (2,4), "generator expression") + self.default(node) + def visitGetattr(self, node): + if (isinstance(node.expr, compiler.ast.Const) + and isinstance(node.expr.value, str) + and node.attrname == "format"): + self.add(node, (2,6), "string literal .format()") + self.default(node) + def visitIfExp(self, node): + self.add(node, (2,5), "inline if expression") + self.default(node) + def visitImport(self, node): + for n in node.names: + v = StandardModules.get(n[0]) + if v is not None: + self.add(node, v, n[0]) + self.default(node) + def visitName(self, node): + v = Identifiers.get(node.name) + if v is not None: + self.add(node, v, node.name) + self.default(node) + def visitSet(self, node): + self.add(node, (2,7), "set literal") + self.default(node) + def visitSetComp(self, node): + self.add(node, (2,7), "set comprehension") + self.default(node) + def visitTryFinally(self, node): + # try/finally with a suite generates a Stmt node as the body, + # but try/except/finally generates a TryExcept as the body + if isinstance(node.body, compiler.ast.TryExcept): + self.add(node, (2,5), "try/except/finally") + self.default(node) + def visitWith(self, node): + if isinstance(node.body, compiler.ast.With): + self.add(node, (2,7), "with statement with multiple contexts") + else: + self.add(node, (2,5), "with statement") + self.default(node) + def visitYield(self, node): + self.add(node, (2,2), "yield expression") + self.default(node) + +def get_versions(source): + """Return information about the Python versions required for specific features. + + The return value is a dictionary with keys as a version number as a tuple + (for example Python 2.6 is (2,6)) and the value are a list of features that + require the indicated Python version. + """ + tree = compiler.parse(source) + checker = compiler.walk(tree, NodeChecker()) + return checker.vers + +def v27(source): + if sys.version_info >= (2, 7): + return qver(source) + else: + print >>sys.stderr, "Not all features tested, run --test with Python 2.7" + return (2, 7) + +def qver(source): + """Return the minimum Python version required to run a particular bit of code. + + >>> qver('print "hello world"') + (2, 0) + >>> qver('class test(object): pass') + (2, 2) + >>> qver('yield 1') + (2, 2) + >>> qver('a // b') + (2, 2) + >>> qver('True') + (2, 2) + >>> qver('enumerate(a)') + (2, 3) + >>> qver('total = sum') + (2, 0) + >>> qver('sum(a)') + (2, 3) + >>> qver('(x*x for x in range(5))') + (2, 4) + >>> qver('class C:\\n @classmethod\\n def m(): pass') + (2, 4) + >>> qver('y if x else z') + (2, 5) + >>> qver('import hashlib') + (2, 5) + >>> qver('from hashlib import md5') + (2, 5) + >>> qver('import xml.etree.ElementTree') + (2, 5) + >>> qver('try:\\n try: pass;\\n except: pass;\\nfinally: pass') + (2, 0) + >>> qver('try: pass;\\nexcept: pass;\\nfinally: pass') + (2, 5) + >>> qver('from __future__ import with_statement\\nwith x: pass') + (2, 5) + >>> qver('collections.defaultdict(list)') + (2, 5) + >>> qver('from collections import defaultdict') + (2, 5) + >>> qver('"{0}".format(0)') + (2, 6) + >>> qver('memoryview(x)') + (2, 7) + >>> v27('{1, 2, 3}') + (2, 7) + >>> v27('{x for x in s}') + (2, 7) + >>> v27('{x: y for x in s}') + (2, 7) + >>> qver('from __future__ import with_statement\\nwith x:\\n with y: pass') + (2, 5) + >>> v27('from __future__ import with_statement\\nwith x, y: pass') + (2, 7) + >>> qver('@decorator\\ndef f(): pass') + (2, 4) + >>> qver('@decorator\\nclass test:\\n pass') + (2, 6) + + #>>> qver('0o0') + #(2, 6) + #>>> qver('@foo\\nclass C: pass') + #(2, 6) + """ + return max(get_versions(source).keys()) + + +if __name__ == '__main__': + + Verbose = False + MinVersion = (2, 3) + Lint = False + + files = [] + i = 1 + while i < len(sys.argv): + a = sys.argv[i] + if a == "--test": + import doctest + doctest.testmod() + sys.exit(0) + if a == "-v" or a == "--verbose": + Verbose = True + elif a == "-l" or a == "--lint": + Lint = True + elif a == "-m" or a == "--min-version": + i += 1 + MinVersion = tuple(map(int, sys.argv[i].split("."))) + else: + files.append(a) + i += 1 + + if not files: + print >>sys.stderr, """Usage: %s [options] source ... + + Report minimum Python version required to run given source files. + + -m x.y or --min-version x.y (default 2.3) + report version triggers at or above version x.y in verbose mode + -v or --verbose + print more detailed report of version triggers for each version + """ % sys.argv[0] + sys.exit(1) + + for fn in files: + try: + f = open(fn) + source = f.read() + f.close() + ver = get_versions(source) + if Verbose: + print fn + for v in sorted([k for k in ver.keys() if k >= MinVersion], reverse=True): + reasons = [x for x in uniq(ver[v]) if x] + if reasons: + # each reason is (lineno, message) + print "\t%s\t%s" % (".".join(map(str, v)), ", ".join([x[1] for x in reasons])) + elif Lint: + for v in sorted([k for k in ver.keys() if k >= MinVersion], reverse=True): + reasons = [x for x in uniq(ver[v]) if x] + for r in reasons: + # each reason is (lineno, message) + print "%s:%s: %s %s" % (fn, r[0], ".".join(map(str, v)), r[1]) + else: + print "%s\t%s" % (".".join(map(str, max(ver.keys()))), fn) + except SyntaxError, x: + print "%s: syntax error compiling with Python %s: %s" % (fn, platform.python_version(), x) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 50fe453cfb..bf91a885ca 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -22,20 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## - -# -# When packages call 'from spack import *', this is what is brought in. -# -# Spack internal code calls 'import spack' and accesses other -# variables (spack.db, paths, etc.) directly. -# -# TODO: maybe this should be separated out and should go in build_environment.py? -# TODO: it's not clear where all the stuff that needs to be included in packages -# should live. This file is overloaded for spack core vs. for packages. -__all__ = ['Package', 'when', 'provides', 'depends_on', 'version', - 'patch', 'Version', 'working_dir', 'which', 'Executable', - 'filter_file', 'change_sed_delimiter'] - import os import tempfile from llnl.util.filesystem import * @@ -58,7 +44,6 @@ var_path = join_path(prefix, "var", "spack") stage_path = join_path(var_path, "stage") install_path = join_path(prefix, "opt") share_path = join_path(prefix, "share", "spack") -dotkit_path = join_path(share_path, "dotkit") # # Set up the packages database. @@ -81,7 +66,7 @@ mock_user_config = join_path(mock_config_path, "user_spackconfig") # stage directories. # from spack.directory_layout import SpecHashDirectoryLayout -install_layout = SpecHashDirectoryLayout(install_path, prefix_size=6) +install_layout = SpecHashDirectoryLayout(install_path) # # This controls how things are concretized in spack. @@ -141,11 +126,30 @@ do_checksum = True # sys_type = None + +# +# When packages call 'from spack import *', this extra stuff is brought in. +# +# Spack internal code should call 'import spack' and accesses other +# variables (spack.db, paths, etc.) directly. # -# Extra imports that should be generally usable from package.py files. +# TODO: maybe this should be separated out and should go in build_environment.py? +# TODO: it's not clear where all the stuff that needs to be included in packages +# should live. This file is overloaded for spack core vs. for packages. # -from llnl.util.filesystem import working_dir +__all__ = ['Package', 'Version', 'when'] from spack.package import Package -from spack.relations import * -from spack.multimethod import when from spack.version import Version +from spack.multimethod import when + +import llnl.util.filesystem +from llnl.util.filesystem import * +__all__ += llnl.util.filesystem.__all__ + +import spack.relations +from spack.relations import * +__all__ += spack.relations.__all__ + +import spack.util.executable +from spack.util.executable import * +__all__ += spack.util.executable.__all__ diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 38d5f70282..182a5629fa 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -122,7 +122,7 @@ def set_build_environment_variables(pkg): # Prefixes of all of the package's dependencies go in # SPACK_DEPENDENCIES - dep_prefixes = [d.package.prefix for d in pkg.spec.dependencies.values()] + dep_prefixes = [d.prefix for d in pkg.spec.traverse(root=False)] path_set(SPACK_DEPENDENCIES, dep_prefixes) # Install prefix @@ -143,6 +143,10 @@ def set_build_environment_variables(pkg): os.environ[SPACK_SPEC] = str(pkg.spec) os.environ[SPACK_DEBUG_LOG_DIR] = spack.spack_working_dir + # Add dependencies to CMAKE_PREFIX_PATH + dep_prefixes = [d.package.prefix for d in pkg.spec.dependencies.values()] + path_set("CMAKE_PREFIX_PATH", dep_prefixes) + def set_module_variables_for_package(pkg): """Populate the module scope of install() with some useful functions. diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index 31c908d42b..f75b68b00a 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -23,12 +23,13 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from subprocess import check_call, check_output +from subprocess import check_call import llnl.util.tty as tty -from llnl.util.filesystem import join_path +from llnl.util.filesystem import join_path, mkdirp import spack +from spack.util.executable import which description = "Create a new installation of spack in another prefix" @@ -38,8 +39,10 @@ def setup_parser(subparser): def get_origin_url(): git_dir = join_path(spack.prefix, '.git') - origin_url = check_output( - ['git', '--git-dir=%s' % git_dir, 'config', '--get', 'remote.origin.url']) + git = which('git', required=True) + origin_url = git( + '--git-dir=%s' % git_dir, 'config', '--get', 'remote.origin.url', + return_output=True) return origin_url.strip() @@ -49,6 +52,11 @@ def bootstrap(parser, args): tty.msg("Fetching spack from origin: %s" % origin_url) + if os.path.isfile(prefix): + tty.die("There is already a file at %s" % prefix) + + mkdirp(prefix) + if os.path.exists(join_path(prefix, '.git')): tty.die("There already seems to be a git repository in %s" % prefix) @@ -62,10 +70,11 @@ def bootstrap(parser, args): "%s/lib/spack/..." % prefix) os.chdir(prefix) - check_call(['git', 'init', '--shared', '-q']) - check_call(['git', 'remote', 'add', 'origin', origin_url]) - check_call(['git', 'fetch', 'origin', 'master:refs/remotes/origin/master', '-n', '-q']) - check_call(['git', 'reset', '--hard', 'origin/master', '-q']) + git = which('git', required=True) + git('init', '--shared', '-q') + git('remote', 'add', 'origin', origin_url) + git('fetch', 'origin', 'master:refs/remotes/origin/master', '-n', '-q') + git('reset', '--hard', 'origin/master', '-q') tty.msg("Successfully created a new spack in %s" % prefix, "Run %s/bin/spack to use this installation." % prefix) diff --git a/lib/spack/spack/cmd/cd.py b/lib/spack/spack/cmd/cd.py new file mode 100644 index 0000000000..24d56db7d0 --- /dev/null +++ b/lib/spack/spack/cmd/cd.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import spack.cmd.location +import spack.modules + +description="cd to spack directories in the shell." + +def setup_parser(subparser): + """This is for decoration -- spack cd is used through spack's + shell support. This allows spack cd to print a descriptive + help message when called with -h.""" + spack.cmd.location.setup_parser(subparser) + + +def cd(parser, args): + spack.modules.print_help() diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index f5cf0d0143..5a8109b70f 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -24,7 +24,7 @@ ############################################################################## import os import re -import argparse +from external import argparse import hashlib from pprint import pprint from subprocess import CalledProcessError diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 6091cae6c8..1df9d87ae2 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import argparse +from external import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index a4cd2df7e2..ac9c844a4c 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import argparse +from external import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index 85f9642019..283bfc19b9 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -import argparse +from external import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/dependents.py b/lib/spack/spack/cmd/dependents.py index db6be88d32..652f243b98 100644 --- a/lib/spack/spack/cmd/dependents.py +++ b/lib/spack/spack/cmd/dependents.py @@ -22,14 +22,14 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import argparse +from external import argparse import llnl.util.tty as tty import spack import spack.cmd -description = "Show dependent packages." +description = "Show installed packages that depend on another." def setup_parser(subparser): subparser.add_argument( @@ -42,5 +42,5 @@ def dependents(parser, args): tty.die("spack dependents takes only one spec.") fmt = '$_$@$%@$+$=$#' - deps = [d.format(fmt) for d in specs[0].package.installed_dependents] - tty.msg("Dependents of %s" % specs[0].format(fmt), *deps) + deps = [d.format(fmt, color=True) for d in specs[0].package.installed_dependents] + tty.msg("Dependents of %s" % specs[0].format(fmt, color=True), *deps) diff --git a/lib/spack/spack/cmd/dotkit.py b/lib/spack/spack/cmd/dotkit.py deleted file mode 100644 index 7a691ae5c0..0000000000 --- a/lib/spack/spack/cmd/dotkit.py +++ /dev/null @@ -1,99 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import sys -import os -import shutil -import argparse - -import llnl.util.tty as tty -from llnl.util.lang import partition_list -from llnl.util.filesystem import mkdirp - -import spack.cmd -import spack.hooks.dotkit -from spack.spec import Spec - - -description ="Find dotkits for packages if they exist." - -def setup_parser(subparser): - subparser.add_argument( - '--refresh', action='store_true', help='Regenerate all dotkits') - - subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, help='spec to find a dotkit for.') - - -def dotkit_find(parser, args): - if not args.spec: - parser.parse_args(['dotkit', '-h']) - - spec = spack.cmd.parse_specs(args.spec) - if len(spec) > 1: - tty.die("You can only pass one spec.") - spec = spec[0] - - if not spack.db.exists(spec.name): - tty.die("No such package: %s" % spec.name) - - specs = [s for s in spack.db.installed_package_specs() if s.satisfies(spec)] - - if len(specs) == 0: - tty.die("No installed packages match spec %s" % spec) - - if len(specs) > 1: - tty.error("Multiple matches for spec %s. Choose one:" % spec) - for s in specs: - sys.stderr.write(s.tree(color=True)) - sys.exit(1) - - match = specs[0] - if not os.path.isfile(spack.hooks.dotkit.dotkit_file(match.package)): - tty.die("No dotkit is installed for package %s." % spec) - - print match.format('$_$@$+$%@$=$#') - - -def dotkit_refresh(parser, args): - query_specs = spack.cmd.parse_specs(args.spec) - - specs = spack.db.installed_package_specs() - if query_specs: - specs = [s for s in specs - if any(s.satisfies(q) for q in query_specs)] - else: - shutil.rmtree(spack.dotkit_path, ignore_errors=False) - mkdirp(spack.dotkit_path) - - for spec in specs: - spack.hooks.dotkit.post_install(spec.package) - - - -def dotkit(parser, args): - if args.refresh: - dotkit_refresh(parser, args) - else: - dotkit_find(parser, args) diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 1dd8703daf..0ccebd9486 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import argparse +from external import argparse import spack import spack.cmd diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 7f2bce119e..2238484a21 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -24,7 +24,7 @@ ############################################################################## import sys import collections -import argparse +from external import argparse from StringIO import StringIO import llnl.util.tty as tty @@ -89,7 +89,7 @@ def find(parser, args): format = " %-{}s%s".format(width) for abbrv, spec in zip(abbreviated, specs): - print format % (abbrv, spec.package.prefix) + print format % (abbrv, spec.prefix) elif args.full_specs: for spec in specs: diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index 648dbf905a..bb147b30f5 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -37,15 +37,17 @@ def info(parser, args): package = spack.db.get(args.name) print "Package: ", package.name print "Homepage: ", package.homepage - print "Download: ", package.url print print "Safe versions: " - if package.versions: - colify(reversed(sorted(package.versions)), indent=4) + if not package.versions: + print("None.") else: - print "None. Use spack versions %s to get a list of downloadable versions." % package.name + maxlen = max(len(str(v)) for v in package.versions) + fmt = "%%-%ss" % maxlen + for v in reversed(sorted(package.versions)): + print " " + (fmt % v) + " " + package.url_for_version(v) print print "Dependencies:" diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 4570d6c40f..2374d02feb 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -import argparse +from external import argparse import spack import spack.cmd diff --git a/lib/spack/spack/cmd/load.py b/lib/spack/spack/cmd/load.py new file mode 100644 index 0000000000..06574d9725 --- /dev/null +++ b/lib/spack/spack/cmd/load.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by David Beckingsale, david@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from external import argparse +import spack.modules + +description ="Add package to environment using modules." + +def setup_parser(subparser): + """Parser is only constructed so that this prints a nice help + message with -h. """ + subparser.add_argument( + 'spec', nargs=argparse.REMAINDER, help='Spec of package to load with modules.') + + +def load(parser, args): + spack.modules.print_help() diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py new file mode 100644 index 0000000000..074d984ee6 --- /dev/null +++ b/lib/spack/spack/cmd/location.py @@ -0,0 +1,93 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +from external import argparse + +import llnl.util.tty as tty +from llnl.util.filesystem import join_path + +import spack +import spack.cmd + +description="Print out locations of various diectories used by Spack" + +def setup_parser(subparser): + global directories + directories = subparser.add_mutually_exclusive_group() + + directories.add_argument( + '-m', '--module-dir', action='store_true', help="Spack python module directory.") + directories.add_argument( + '-r', '--spack-root', action='store_true', help="Spack installation root.") + + directories.add_argument( + '-i', '--install-dir', action='store_true', + help="Install prefix for spec (spec need not be installed).") + directories.add_argument( + '-p', '--package-dir', action='store_true', + help="Directory enclosing a spec's package.py file.") + directories.add_argument( + '-s', '--stage-dir', action='store_true', help="Stage directory for a spec.") + directories.add_argument( + '-b', '--build-dir', action='store_true', + help="Expanded archive directory for a spec (requires it to be staged first).") + + subparser.add_argument( + 'spec', nargs=argparse.REMAINDER, help="spec of package to fetch directory for.") + + +def location(parser, args): + if args.module_dir: + print spack.module_path + + elif args.spack_root: + print spack.prefix + + else: + specs = spack.cmd.parse_specs(args.spec, concretize=True) + if not specs: + tty.die("You must supply a spec.") + if len(specs) != 1: + tty.die("Too many specs. Need only one.") + spec = specs[0] + + if args.install_dir: + print spec.prefix + + elif args.package_dir: + print join_path(spack.db.root, spec.name) + + else: + pkg = spack.db.get(spec) + + if args.stage_dir: + print pkg.stage.path + + else: # args.build_dir is the default. + if not os.listdir(pkg.stage.path): + tty.die("Build directory does not exist yet. Run this to create it:", + "spack stage " + " ".join(args.spec)) + print pkg.stage.expanded_archive_path + diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 129ac6bd45..b42b329085 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -24,10 +24,10 @@ ############################################################################## import os import shutil -import argparse from datetime import datetime from contextlib import closing +from external import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify from llnl.util.filesystem import mkdirp, join_path @@ -41,7 +41,7 @@ from spack.stage import Stage from spack.util.compression import extension -description = "Manage spack mirrors." +description = "Manage mirrors." def setup_parser(subparser): subparser.add_argument( diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py new file mode 100644 index 0000000000..34f0855a50 --- /dev/null +++ b/lib/spack/spack/cmd/module.py @@ -0,0 +1,107 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import sys +import os +import shutil +from external import argparse + +import llnl.util.tty as tty +from llnl.util.lang import partition_list +from llnl.util.filesystem import mkdirp + +import spack.cmd +from spack.modules import module_types +from spack.util.string import * + +from spack.spec import Spec + +description ="Manipulate modules and dotkits." + + +def setup_parser(subparser): + sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='module_command') + + refresh_parser = sp.add_parser('refresh', help='Regenerate all module files.') + + find_parser = sp.add_parser('find', help='Find module files for packages.') + find_parser.add_argument( + 'module_type', help="Type of module to find file for. [" + '|'.join(module_types) + "]") + find_parser.add_argument('spec', nargs='+', help='spec to find a module file for.') + + +def module_find(mtype, spec_array): + """Look at all installed packages and see if the spec provided + matches any. If it does, check whether there is a module file + of type <mtype> there, and print out the name that the user + should type to use that package's module. + """ + if mtype not in module_types: + tty.die("Invalid module type: '%s'. Options are %s." % (mtype, comma_or(module_types))) + + specs = spack.cmd.parse_specs(spec_array) + if len(specs) > 1: + tty.die("You can only pass one spec.") + spec = specs[0] + + specs = [s for s in spack.db.installed_package_specs() if s.satisfies(spec)] + if len(specs) == 0: + tty.die("No installed packages match spec %s" % spec) + + if len(specs) > 1: + tty.error("Multiple matches for spec %s. Choose one:" % spec) + for s in specs: + sys.stderr.write(s.tree(color=True)) + sys.exit(1) + + mt = module_types[mtype] + mod = mt(specs[0]) + if not os.path.isfile(mod.file_name): + tty.die("No %s module is installed for %s." % (mtype, spec)) + + print mod.use_name + + +def module_refresh(): + """Regenerate all module files for installed packages known to + spack (some packages may no longer exist).""" + specs = [s for s in spack.db.installed_known_package_specs()] + + for name, cls in module_types.items(): + tty.msg("Regenerating %s module files." % name) + if os.path.isdir(cls.path): + shutil.rmtree(cls.path, ignore_errors=False) + mkdirp(cls.path) + for spec in specs: + tty.debug(" Writing file for %s." % spec) + cls(spec).write() + + + +def module(parser, args): + if args.module_command == 'refresh': + module_refresh() + + elif args.module_command == 'find': + module_find(args.module_type, args.spec) diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index 2356583b07..a6556c4828 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import argparse +from external import argparse import spack.cmd import spack diff --git a/lib/spack/spack/cmd/providers.py b/lib/spack/spack/cmd/providers.py index 1a652c82d1..2bcdc9fba2 100644 --- a/lib/spack/spack/cmd/providers.py +++ b/lib/spack/spack/cmd/providers.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import argparse +from external import argparse from llnl.util.tty.colify import colify diff --git a/lib/spack/spack/cmd/python.py b/lib/spack/spack/cmd/python.py index 641394044c..86b8c827f8 100644 --- a/lib/spack/spack/cmd/python.py +++ b/lib/spack/spack/cmd/python.py @@ -25,7 +25,7 @@ import os import sys import code -import argparse +from external import argparse import platform from contextlib import closing diff --git a/lib/spack/spack/cmd/spec.py b/lib/spack/spack/cmd/spec.py index fb5a900c3f..5fcb0a9b5a 100644 --- a/lib/spack/spack/cmd/spec.py +++ b/lib/spack/spack/cmd/spec.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import argparse +from external import argparse import spack.cmd import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index 1bf1f93c2f..c8bc473c55 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -22,8 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import argparse +import os +from external import argparse +import llnl.util.tty as tty import spack import spack.cmd @@ -33,18 +35,21 @@ def setup_parser(subparser): subparser.add_argument( '-n', '--no-checksum', action='store_true', dest='no_checksum', help="Do not check downloaded packages against checksum") + + dir_parser = subparser.add_mutually_exclusive_group() subparser.add_argument( - 'packages', nargs=argparse.REMAINDER, help="specs of packages to stage") + 'specs', nargs=argparse.REMAINDER, help="specs of packages to stage") def stage(parser, args): - if not args.packages: + if not args.specs: tty.die("stage requires at least one package argument") if args.no_checksum: spack.do_checksum = False - specs = spack.cmd.parse_specs(args.packages, concretize=True) + specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: package = spack.db.get(spec) package.do_stage() + diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 73c98a203b..84eb4703a6 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import argparse +from external import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/unload.py b/lib/spack/spack/cmd/unload.py new file mode 100644 index 0000000000..6442c48cb1 --- /dev/null +++ b/lib/spack/spack/cmd/unload.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by David Beckingsale, david@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from external import argparse +import spack.modules + +description ="Remove package from environment using module." + +def setup_parser(subparser): + """Parser is only constructed so that this prints a nice help + message with -h. """ + subparser.add_argument( + 'spec', nargs=argparse.REMAINDER, help='Spec of package to unload with modules.') + + +def unload(parser, args): + spack.modules.print_help() diff --git a/lib/spack/spack/cmd/unuse.py b/lib/spack/spack/cmd/unuse.py index a31e16d11a..2a7229a3a0 100644 --- a/lib/spack/spack/cmd/unuse.py +++ b/lib/spack/spack/cmd/unuse.py @@ -22,15 +22,17 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import argparse -import spack.cmd.use +from external import argparse +import spack.modules description ="Remove package from environment using dotkit." def setup_parser(subparser): + """Parser is only constructed so that this prints a nice help + message with -h. """ subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, help='Spec of package to remove.') + 'spec', nargs=argparse.REMAINDER, help='Spec of package to unuse with dotkit.') def unuse(parser, args): - spack.cmd.use.print_help() + spack.modules.print_help() diff --git a/lib/spack/spack/cmd/use.py b/lib/spack/spack/cmd/use.py index 10a0644df8..e34c194739 100644 --- a/lib/spack/spack/cmd/use.py +++ b/lib/spack/spack/cmd/use.py @@ -22,29 +22,17 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import argparse -import llnl.util.tty as tty -import spack +from external import argparse +import spack.modules description ="Add package to environment using dotkit." def setup_parser(subparser): + """Parser is only constructed so that this prints a nice help + message with -h. """ subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, help='Spec of package to add.') - - -def print_help(): - tty.msg("Spack dotkit support is not initialized.", - "", - "To use dotkit with Spack, you must first run the command", - "below, which you can copy and paste:", - "", - "For bash:", - " . %s/setup-env.bash" % spack.share_path, - "", - "ksh/csh/tcsh shells are currently unsupported", - "") + 'spec', nargs=argparse.REMAINDER, help='Spec of package to use with dotkit.') def use(parser, args): - print_help() + spack.modules.print_help() diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 716356bdd2..582f49eaf2 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -189,7 +189,7 @@ class Compiler(object): return None successful = [key for key in parmap(check, checks) if key is not None] - return { (v, p, s) : path for v, p, s, path in successful } + return dict(((v, p, s), path) for v, p, s, path in successful) @classmethod def find(cls, *path): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index c5bfe21ed4..467472cced 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -176,7 +176,7 @@ def compilers_for_spec(compiler_spec): config = _get_config() def get_compiler(cspec): - items = { k:v for k,v in config.items('compiler "%s"' % cspec) } + items = dict((k,v) for k,v in config.items('compiler "%s"' % cspec)) if not all(n in items for n in _required_instance_vars): raise InvalidCompilerConfigurationError(cspec) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index eb497711b7..e6d1bb87d4 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -118,7 +118,7 @@ class DefaultConcretizer(object): return try: - nearest = next(p for p in spec.preorder_traversal(direction='parents') + nearest = next(p for p in spec.traverse(direction='parents') if p.compiler is not None).compiler if not nearest in all_compilers: diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 5494adc324..85ee16a1c2 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -84,10 +84,9 @@ import os import re import inspect import ConfigParser as cp -from collections import OrderedDict +from external.ordereddict import OrderedDict from llnl.util.lang import memoized - import spack.error __all__ = [ @@ -222,7 +221,6 @@ class SpackConfigParser(cp.RawConfigParser): """ # Slightly modify Python option expressions to allow leading whitespace OPTCRE = re.compile(r'\s*' + cp.RawConfigParser.OPTCRE.pattern) - OPTCRE_NV = re.compile(r'\s*' + cp.RawConfigParser.OPTCRE_NV.pattern) def __init__(self, file_or_files): cp.RawConfigParser.__init__(self, dict_type=OrderedDict) @@ -341,14 +339,13 @@ class SpackConfigParser(cp.RawConfigParser): def _read(self, fp, fpname): - """This is a copy of Python 2.7's _read() method, with support for - continuation lines removed. - """ - cursect = None # None, or a dictionary + """This is a copy of Python 2.6's _read() method, with support for + continuation lines removed.""" + cursect = None # None, or a dictionary optname = None - lineno = 0 comment = 0 - e = None # None, or an exception + lineno = 0 + e = None # None, or an exception while True: line = fp.readline() if not line: @@ -359,7 +356,6 @@ class SpackConfigParser(cp.RawConfigParser): (line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR")): self._sections["comment-%d" % comment] = line comment += 1 - continue # a section header or option header? else: # is it a section header? @@ -381,27 +377,21 @@ class SpackConfigParser(cp.RawConfigParser): raise cp.MissingSectionHeaderError(fpname, lineno, line) # an option line? else: - mo = self._optcre.match(line) + mo = self.OPTCRE.match(line) if mo: optname, vi, optval = mo.group('option', 'vi', 'value') + if vi in ('=', ':') and ';' in optval: + # ';' is a comment delimiter only if it follows + # a spacing character + pos = optval.find(';') + if pos != -1 and optval[pos-1].isspace(): + optval = optval[:pos] + optval = optval.strip() + # allow empty values + if optval == '""': + optval = '' optname = self.optionxform(optname.rstrip()) - # This check is fine because the OPTCRE cannot - # match if it would set optval to None - if optval is not None: - if vi in ('=', ':') and ';' in optval: - # ';' is a comment delimiter only if it follows - # a spacing character - pos = optval.find(';') - if pos != -1 and optval[pos-1].isspace(): - optval = optval[:pos] - optval = optval.strip() - # allow empty values - if optval == '""': - optval = '' - cursect[optname] = [optval] - else: - # valueless option handling - cursect[optname] = optval + cursect[optname] = optval else: # a non-fatal parsing error occurred. set up the # exception but keep going. the exception will be @@ -414,23 +404,13 @@ class SpackConfigParser(cp.RawConfigParser): if e: raise e - # join the multi-line values collected while reading - all_sections = [self._defaults] - all_sections.extend(self._sections.values()) - for options in all_sections: - # skip comments - if isinstance(options, basestring): - continue - for name, val in options.items(): - if isinstance(val, list): - options[name] = '\n'.join(val) def _write(self, fp): """Write an .ini-format representation of the configuration state. - This is taken from the default Python 2.7 source. It writes 4 + This is taken from the default Python 2.6 source. It writes 4 spaces at the beginning of lines instead of no leading space. """ if self._defaults: @@ -449,11 +429,9 @@ class SpackConfigParser(cp.RawConfigParser): # Allow leading whitespace fp.write("[%s]\n" % section) for (key, value) in self._sections[section].items(): - if key == "__name__": - continue - if (value is not None) or (self._optcre == self.OPTCRE): - key = " = ".join((key, str(value).replace('\n', '\n\t'))) - fp.write(" %s\n" % (key)) + if key != "__name__": + fp.write(" %s = %s\n" % + (key, str(value).replace('\n', '\n\t'))) class SpackConfigurationError(spack.error.SpackError): diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 4fc00d536e..9b31aad5fe 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -29,7 +29,10 @@ import hashlib import shutil from contextlib import closing +import llnl.util.tty as tty from llnl.util.filesystem import join_path, mkdirp + +import spack from spack.spec import Spec from spack.error import SpackError @@ -131,12 +134,9 @@ class SpecHashDirectoryLayout(DirectoryLayout): """Prefix size is number of characters in the SHA-1 prefix to use to make each hash unique. """ - prefix_size = kwargs.get('prefix_size', 8) - spec_file = kwargs.get('spec_file', '.spec') - + spec_file_name = kwargs.get('spec_file_name', '.spec') super(SpecHashDirectoryLayout, self).__init__(root) - self.prefix_size = prefix_size - self.spec_file = spec_file + self.spec_file_name = spec_file_name def relative_path_for_spec(self, spec): @@ -154,16 +154,36 @@ class SpecHashDirectoryLayout(DirectoryLayout): def read_spec(self, path): """Read the contents of a file and parse them as a spec""" with closing(open(path)) as spec_file: - string = spec_file.read().replace('\n', '') # Specs from files are assumed normal and concrete - return Spec(string, concrete=True) + spec = Spec(spec_file.read().replace('\n', '')) + + # If we do not have a package on hand for this spec, we know + # it is concrete, and we *assume* that it is normal. This + # prevents us from trying to fetch a non-existing package, and + # allows best effort for commands like spack find. + if not spack.db.exists(spec.name): + spec._normal = True + spec._concrete = True + else: + spec.normalize() + if not spec.concrete: + tty.warn("Spec read from installed package is not concrete:", + path, spec) + + return spec + + + def spec_file_path(self, spec): + """Gets full path to spec file""" + _check_concrete(spec) + return join_path(self.path_for_spec(spec), self.spec_file_name) def make_path_for_spec(self, spec): _check_concrete(spec) path = self.path_for_spec(spec) - spec_file_path = join_path(path, self.spec_file) + spec_file_path = self.spec_file_path(spec) if os.path.isdir(path): if not os.path.isfile(spec_file_path): @@ -177,8 +197,7 @@ class SpecHashDirectoryLayout(DirectoryLayout): spec_hash = self.hash_spec(spec) installed_hash = self.hash_spec(installed_spec) if installed_spec == spec_hash: - raise SpecHashCollisionError( - installed_hash, spec_hash, self.prefix_size) + raise SpecHashCollisionError(installed_hash, spec_hash) else: raise InconsistentInstallDirectoryError( 'Spec file in %s does not match SHA-1 hash!' @@ -195,7 +214,7 @@ class SpecHashDirectoryLayout(DirectoryLayout): for path in traverse_dirs_at_depth(self.root, 3): arch, compiler, last_dir = path spec_file_path = join_path( - self.root, arch, compiler, last_dir, self.spec_file) + self.root, arch, compiler, last_dir, self.spec_file_name) if os.path.exists(spec_file_path): spec = self.read_spec(spec_file_path) yield spec @@ -209,10 +228,10 @@ class DirectoryLayoutError(SpackError): class SpecHashCollisionError(DirectoryLayoutError): """Raised when there is a hash collision in an SpecHashDirectoryLayout.""" - def __init__(self, installed_spec, new_spec, prefix_size): + def __init__(self, installed_spec, new_spec): super(SpecHashDirectoryLayout, self).__init__( - 'Specs %s and %s have the same %d character SHA-1 prefix!' - % prefix_size, installed_spec, new_spec) + 'Specs %s and %s have the same SHA-1 prefix!' + % installed_spec, new_spec) class InconsistentInstallDirectoryError(DirectoryLayoutError): diff --git a/lib/spack/spack/error.py b/lib/spack/spack/error.py index 40e0e75fdb..8bcbd83c28 100644 --- a/lib/spack/spack/error.py +++ b/lib/spack/spack/error.py @@ -28,7 +28,8 @@ class SpackError(Exception): Subclasses can be found in the modules they have to do with. """ def __init__(self, message, long_message=None): - super(SpackError, self).__init__(message) + super(SpackError, self).__init__() + self.message = message self.long_message = long_message diff --git a/lib/spack/spack/hooks/dotkit.py b/lib/spack/spack/hooks/dotkit.py index 10b7732353..4e748ff80a 100644 --- a/lib/spack/spack/hooks/dotkit.py +++ b/lib/spack/spack/hooks/dotkit.py @@ -22,62 +22,14 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os -import re -import textwrap -import shutil -from contextlib import closing - -from llnl.util.filesystem import join_path, mkdirp - -import spack - - -def dotkit_file(pkg): - dk_file_name = pkg.spec.format('$_$@$%@$+$=$#') + ".dk" - return join_path(spack.dotkit_path, dk_file_name) +import spack.modules def post_install(pkg): - if not os.path.exists(spack.dotkit_path): - mkdirp(spack.dotkit_path) - - alterations = [] - for var, path in [ - ('PATH', pkg.prefix.bin), - ('MANPATH', pkg.prefix.man), - ('MANPATH', pkg.prefix.share_man), - ('LD_LIBRARY_PATH', pkg.prefix.lib), - ('LD_LIBRARY_PATH', pkg.prefix.lib64)]: - - if os.path.isdir(path): - alterations.append("dk_alter %s %s\n" % (var, path)) - - if not alterations: - return - - alterations.append("dk_alter CMAKE_PREFIX_PATH %s\n" % pkg.prefix) - - dk_file = dotkit_file(pkg) - with closing(open(dk_file, 'w')) as dk: - # Put everything in the spack category. - dk.write('#c spack\n') - - dk.write('#d %s\n' % pkg.spec.format("$_ $@")) - - # Recycle the description - if pkg.__doc__: - doc = re.sub(r'\s+', ' ', pkg.__doc__) - for line in textwrap.wrap(doc, 72): - dk.write("#h %s\n" % line) - - # Write alterations - for alter in alterations: - dk.write(alter) + dk = spack.modules.Dotkit(pkg.spec) + dk.write() def post_uninstall(pkg): - dk_file = dotkit_file(pkg) - if os.path.exists(dk_file): - shutil.rmtree(dk_file, ignore_errors=True) - + dk = spack.modules.Dotkit(pkg.spec) + dk.remove() diff --git a/lib/spack/spack/hooks/tclmodule.py b/lib/spack/spack/hooks/tclmodule.py new file mode 100644 index 0000000000..0b9fd5a67c --- /dev/null +++ b/lib/spack/spack/hooks/tclmodule.py @@ -0,0 +1,35 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by David Beckingsale, david@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import spack.modules + + +def post_install(pkg): + dk = spack.modules.TclModule(pkg.spec) + dk.write() + + +def post_uninstall(pkg): + dk = spack.modules.TclModule(pkg.spec) + dk.remove() diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py new file mode 100644 index 0000000000..755e9ea900 --- /dev/null +++ b/lib/spack/spack/modules.py @@ -0,0 +1,247 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""This module contains code for creating environment modules, which +can include dotkits, tcl modules, lmod, and others. + +The various types of modules are installed by post-install hooks and +removed after an uninstall by post-uninstall hooks. This class +consolidates the logic for creating an abstract description of the +information that module systems need. Currently that includes a +number directories to be appended to paths in the user's environment: + + * /bin directories to be appended to PATH + * /lib* directories for LD_LIBRARY_PATH + * /man* and /share/man* directories for LD_LIBRARY_PATH + * the package prefix for CMAKE_PREFIX_PATH + +This module also includes logic for coming up with unique names for +the module files so that they can be found by the various +shell-support files in $SPACK/share/spack/setup-env.*. + +Each hook in hooks/ implements the logic for writing its specific type +of module file. +""" +__all__ = ['EnvModule', 'Dotkit', 'TclModule'] + +import os +import re +import textwrap +import shutil +from contextlib import closing + +import llnl.util.tty as tty +from llnl.util.filesystem import join_path, mkdirp + +import spack + +"""Registry of all types of modules. Entries created by EnvModule's + metaclass.""" +module_types = {} + + +def print_help(): + """For use by commands to tell user how to activate shell support.""" + + tty.msg("This command requires spack's shell integration.", + "", + "To initialize spack's shell commands, you must run one of", + "the commands below. Choose the right command for your shell.", + "", + "For bash and zsh:", + " . %s/setup-env.sh" % spack.share_path, + "", + "For csh and tcsh:", + " setenv SPACK_ROOT %s" % spack.prefix, + " source %s/setup-env.csh" % spack.share_path, + "") + + +class EnvModule(object): + name = 'env_module' + + class __metaclass__(type): + def __init__(cls, name, bases, dict): + type.__init__(cls, name, bases, dict) + if cls.name != 'env_module': + module_types[cls.name] = cls + + + def __init__(self, spec=None): + # category in the modules system + # TODO: come up with smarter category names. + self.category = "spack" + + # Descriptions for the module system's UI + self.short_description = "" + self.long_description = "" + + # dict pathname -> list of directories to be prepended to in + # the module file. + self._paths = None + self.spec = spec + + + @property + def paths(self): + if self._paths is None: + self._paths = {} + + def add_path(path_name, directory): + path = self._paths.setdefault(path_name, []) + path.append(directory) + + # Add paths if they exist. + for var, directory in [ + ('PATH', self.spec.prefix.bin), + ('MANPATH', self.spec.prefix.man), + ('MANPATH', self.spec.prefix.share_man), + ('LD_LIBRARY_PATH', self.spec.prefix.lib), + ('LD_LIBRARY_PATH', self.spec.prefix.lib64)]: + + if os.path.isdir(directory): + add_path(var, directory) + + # short description is just the package + version + # TODO: maybe packages can optionally provide it. + self.short_description = self.spec.format("$_ $@") + + # long description is the docstring with reduced whitespace. + if self.spec.package.__doc__: + self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) + + return self._paths + + + def write(self): + """Write out a module file for this object.""" + module_dir = os.path.dirname(self.file_name) + if not os.path.exists(module_dir): + mkdirp(module_dir) + + # If there are no paths, no need for a dotkit. + if not self.paths: + return + + with closing(open(self.file_name, 'w')) as f: + self._write(f) + + + def _write(self, stream): + """To be implemented by subclasses.""" + raise NotImplementedError() + + + @property + def file_name(self): + """Subclasses should implement this to return the name of the file + where this module lives.""" + raise NotImplementedError() + + + @property + def use_name(self): + """Subclasses should implement this to return the name the + module command uses to refer to the package.""" + raise NotImplementedError() + + + def remove(self): + mod_file = self.file_name + if os.path.exists(mod_file): + shutil.rmtree(mod_file, ignore_errors=True) + + +class Dotkit(EnvModule): + name = 'dotkit' + path = join_path(spack.share_path, "dotkit") + + @property + def file_name(self): + return join_path(Dotkit.path, self.spec.architecture, + self.spec.format('$_$@$%@$+$#.dk')) + + @property + def use_name(self): + return self.spec.format('$_$@$%@$+$#') + + + def _write(self, dk_file): + # Category + if self.category: + dk_file.write('#c %s\n' % self.category) + + # Short description + if self.short_description: + dk_file.write('#d %s\n' % self.short_description) + + # Long description + if self.long_description: + for line in textwrap.wrap(self.long_description, 72): + dk_file.write("#h %s\n" % line) + + # Path alterations + for var, dirs in self.paths.items(): + for directory in dirs: + dk_file.write("dk_alter %s %s\n" % (var, directory)) + + # Let CMake find this package. + dk_file.write("dk_alter CMAKE_PREFIX_PATH %s\n" % self.spec.prefix) + + +class TclModule(EnvModule): + name = 'tcl' + path = join_path(spack.share_path, "modules") + + @property + def file_name(self): + return join_path(TclModule.path, self.spec.architecture, self.use_name) + + + @property + def use_name(self): + return self.spec.format('$_$@$%@$+$#') + + + def _write(self, m_file): + # TODO: cateogry? + m_file.write('#%Module1.0\n') + + # Short description + if self.short_description: + m_file.write('module-whatis \"%s\"\n\n' % self.short_description) + + # Long description + if self.long_description: + m_file.write('proc ModulesHelp { } {\n') + doc = re.sub(r'"', '\"', self.long_description) + m_file.write("puts stderr \"%s\"\n" % doc) + m_file.write('}\n\n') + + # Path alterations + for var, dirs in self.paths.items(): + for directory in dirs: + m_file.write("prepend-path %s \"%s\"\n" % (var, directory)) + + m_file.write("prepend-path CMAKE_PREFIX_PATH \"%s\"\n" % self.spec.prefix) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 48074aaebb..361fda1ba6 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -360,7 +360,7 @@ class Package(object): # Version-ize the keys in versions dict try: - self.versions = { Version(v):h for v,h in self.versions.items() } + self.versions = dict((Version(v), h) for v,h in self.versions.items()) except ValueError: raise ValueError("Keys of versions dict in package %s must be versions!" % self.name) @@ -773,7 +773,7 @@ class Package(object): ' '.join(formatted_deps)) self.remove_prefix() - tty.msg("Successfully uninstalled %s." % self.spec) + tty.msg("Successfully uninstalled %s." % self.spec.short_spec) # Once everything else is done, run post install hooks spack.hooks.post_uninstall(self) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 4f9f606c89..72f9403a64 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -70,11 +70,11 @@ class PackageDB(object): if not spec in self.instances: package_class = self.get_class_for_package_name(spec.name) try: - self.instances[spec.name] = package_class(spec) + self.instances[spec.copy()] = package_class(spec) except Exception, e: raise FailedConstructorError(spec.name, e) - return self.instances[spec.name] + return self.instances[spec] @_autospec @@ -118,7 +118,23 @@ class PackageDB(object): """Read installed package names straight from the install directory layout. """ - return spack.install_layout.all_specs() + # Get specs from the directory layout but ensure that they're + # all normalized properly. + installed = [] + for spec in spack.install_layout.all_specs(): + spec.normalize() + installed.append(spec) + return installed + + + def installed_known_package_specs(self): + """Read installed package names straight from the install + directory layout, but return only specs for which the + package is known to this version of spack. + """ + for spec in spack.install_layout.all_specs(): + if self.exists(spec.name): + yield spec @memoized @@ -182,24 +198,6 @@ class PackageDB(object): return cls - def compute_dependents(self): - """Reads in all package files and sets dependence information on - Package objects in memory. - """ - if not hasattr(compute_dependents, index): - compute_dependents.index = {} - - for pkg in all_packages(): - if pkg._dependents is None: - pkg._dependents = [] - - for name, dep in pkg.dependencies.iteritems(): - dpkg = self.get(name) - if dpkg._dependents is None: - dpkg._dependents = [] - dpkg._dependents.append(pkg.name) - - def graph_dependencies(self, out=sys.stdout): """Print out a graph of all the dependencies between package. Graph is in dot format.""" @@ -214,10 +212,17 @@ class PackageDB(object): return '"%s"' % string deps = [] - for pkg in all_packages(): + for pkg in self.all_packages(): out.write(' %-30s [label="%s"]\n' % (quote(pkg.name), pkg.name)) + + # Add edges for each depends_on in the package. for dep_name, dep in pkg.dependencies.iteritems(): deps.append((pkg.name, dep_name)) + + # If the package provides something, add an edge for that. + for provider in set(p.name for p in pkg.provided): + deps.append((provider, pkg.name)) + out.write('\n') for pair in deps: diff --git a/lib/spack/spack/relations.py b/lib/spack/spack/relations.py index a7b46cfb33..5afb7e7624 100644 --- a/lib/spack/spack/relations.py +++ b/lib/spack/spack/relations.py @@ -72,7 +72,6 @@ __all__ = [ 'depends_on', 'provides', 'patch', 'version' ] import re import inspect -import importlib from llnl.util.lang import * diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 45c3402617..4838fd9946 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -94,6 +94,7 @@ import sys import itertools import hashlib from StringIO import StringIO +from operator import attrgetter import llnl.util.tty as tty from llnl.util.lang import * @@ -309,9 +310,8 @@ class DependencyMap(HashableMap): def __str__(self): - sorted_dep_names = sorted(self.keys()) return ''.join( - ["^" + str(self[name]) for name in sorted_dep_names]) + ["^" + str(self[name]) for name in sorted(self.keys())]) @key_ordering @@ -352,10 +352,6 @@ class Spec(object): self._normal = kwargs.get('normal', False) self._concrete = kwargs.get('concrete', False) - # Specs cannot be concrete and non-normal. - if self._concrete: - self._normal = True - # This allows users to construct a spec DAG with literals. # Note that given two specs a and b, Spec(a) copies a, but # Spec(a, b) will copy a but just add b as a dep. @@ -454,10 +450,19 @@ class Spec(object): return self._concrete - def preorder_traversal(self, visited=None, d=0, **kwargs): - """Generic preorder traversal of the DAG represented by this spec. + def traverse(self, visited=None, d=0, **kwargs): + """Generic traversal of the DAG represented by this spec. This will yield each node in the spec. Options: + order [=pre|post] + Order to traverse spec nodes. Defaults to preorder traversal. + Options are: + + 'pre': Pre-order traversal; each node is yielded before its + children in the dependency DAG. + 'post': Post-order traversal; each node is yielded after its + children in the dependency DAG. + cover [=nodes|edges|paths] Determines how extensively to cover the dag. Possible vlaues: @@ -475,7 +480,7 @@ class Spec(object): spec, but also their depth from the root in a (depth, node) tuple. - keyfun [=id] + key [=id] Allow a custom key function to track the identity of nodes in the traversal. @@ -487,44 +492,57 @@ class Spec(object): 'parents', traverses upwards in the DAG towards the root. """ + # get initial values for kwargs depth = kwargs.get('depth', False) key_fun = kwargs.get('key', id) + if isinstance(key_fun, basestring): + key_fun = attrgetter(key_fun) yield_root = kwargs.get('root', True) cover = kwargs.get('cover', 'nodes') direction = kwargs.get('direction', 'children') + order = kwargs.get('order', 'pre') - cover_values = ('nodes', 'edges', 'paths') - if cover not in cover_values: - raise ValueError("Invalid value for cover: %s. Choices are %s" - % (cover, ",".join(cover_values))) - - direction_values = ('children', 'parents') - if direction not in direction_values: - raise ValueError("Invalid value for direction: %s. Choices are %s" - % (direction, ",".join(direction_values))) + # Make sure kwargs have legal values; raise ValueError if not. + def validate(name, val, allowed_values): + if val not in allowed_values: + raise ValueError("Invalid value for %s: %s. Choices are %s" + % (name, val, ",".join(allowed_values))) + validate('cover', cover, ('nodes', 'edges', 'paths')) + validate('direction', direction, ('children', 'parents')) + validate('order', order, ('pre', 'post')) if visited is None: visited = set() + key = key_fun(self) + + # Node traversal does not yield visited nodes. + if key in visited and cover == 'nodes': + return + # Determine whether and what to yield for this node. + yield_me = yield_root or d > 0 result = (d, self) if depth else self - key = key_fun(self) - if key in visited: - if cover == 'nodes': return - if yield_root or d > 0: yield result - if cover == 'edges': return - else: - if yield_root or d > 0: yield result + # Preorder traversal yields before successors + if yield_me and order == 'pre': + yield result - successors = self.dependencies - if direction == 'parents': - successors = self.dependents + # Edge traversal yields but skips children of visited nodes + if not (key in visited and cover == 'edges'): + # This code determines direction and yields the children/parents + successors = self.dependencies + if direction == 'parents': + successors = self.dependents - visited.add(key) - for name in sorted(successors): - child = successors[name] - for elt in child.preorder_traversal(visited, d+1, **kwargs): - yield elt + visited.add(key) + for name in sorted(successors): + child = successors[name] + for elt in child.traverse(visited, d+1, **kwargs): + yield elt + + # Postorder traversal yields after successors + if yield_me and order == 'post': + yield result @property @@ -540,13 +558,14 @@ class Spec(object): def dep_hash(self, length=None): - """Return a hash representing the dependencies of this spec - This will always normalize first so that the hash is consistent. - """ - self.normalize() + """Return a hash representing all dependencies of this spec + (direct and indirect). + If you want this hash to be consistent, you should + concretize the spec first so that it is not ambiguous. + """ sha = hashlib.sha1() - sha.update(str(self.dependencies)) + sha.update(self.dep_string()) full_hash = sha.hexdigest() return full_hash[:length] @@ -609,7 +628,7 @@ class Spec(object): a problem. """ while True: - virtuals =[v for v in self.preorder_traversal() if v.virtual] + virtuals =[v for v in self.traverse() if v.virtual] if not virtuals: return @@ -620,7 +639,7 @@ class Spec(object): spec._replace_with(concrete) # If there are duplicate providers or duplicate provider deps, this - # consolidates them and merges constraints. + # consolidates them and merge constraints. self.normalize(force=True) @@ -654,47 +673,51 @@ class Spec(object): return clone - def flat_dependencies(self): - """Return a DependencyMap containing all of this spec's dependencies - with their constraints merged. If there are any conflicts, throw - an exception. + def flat_dependencies(self, **kwargs): + """Return a DependencyMap containing all of this spec's + dependencies with their constraints merged. + + If copy is True, returns merged copies of its dependencies + without modifying the spec it's called on. - This will work even on specs that are not normalized; i.e. specs - that have two instances of the same dependency in the DAG. - This is used as the first step of normalization. + If copy is False, clears this spec's dependencies and + returns them. """ - # This ensures that the package descriptions themselves are consistent - if not self.virtual: - self.package.validate_dependencies() + copy = kwargs.get('copy', True) - # Once that is guaranteed, we know any constraint violations are due - # to the spec -- so they're the user's fault, not Spack's. flat_deps = DependencyMap() try: - for spec in self.preorder_traversal(): + for spec in self.traverse(root=False): if spec.name not in flat_deps: - new_spec = spec.copy(dependencies=False) - flat_deps[spec.name] = new_spec - + if copy: + flat_deps[spec.name] = spec.copy(deps=False) + else: + flat_deps[spec.name] = spec else: flat_deps[spec.name].constrain(spec) + if not copy: + for dep in flat_deps.values(): + dep.dependencies.clear() + dep.dependents.clear() + self.dependencies.clear() + + return flat_deps + except UnsatisfiableSpecError, e: - # This REALLY shouldn't happen unless something is wrong in spack. - # It means we got a spec DAG with two instances of the same package - # that had inconsistent constraints. There's no way for a user to - # produce a spec like this (the parser adds all deps to the root), - # so this means OUR code is not sane! + # Here, the DAG contains two instances of the same package + # with inconsistent constraints. Users cannot produce + # inconsistent specs like this on the command line: the + # parser doesn't allow it. Spack must be broken! raise InconsistentSpecError("Invalid Spec DAG: %s" % e.message) - return flat_deps - def flatten(self): """Pull all dependencies up to the root (this spec). Merge constraints for dependencies with the same name, and if they conflict, throw an exception. """ - self.dependencies = self.flat_dependencies() + for dep in self.flat_dependencies(copy=False): + self._add_dependency(dep) def _normalize_helper(self, visited, spec_deps, provider_index): @@ -797,11 +820,12 @@ class Spec(object): # Ensure first that all packages & compilers in the DAG exist. self.validate_names() - # Then ensure that the packages referenced are sane, that the - # provided spec is sane, and that all dependency specs are in the - # root node of the spec. flat_dependencies will do this for us. - spec_deps = self.flat_dependencies() - self.dependencies.clear() + # Ensure that the package & dep descriptions are consistent & sane + if not self.virtual: + self.package.validate_dependencies() + + # Get all the dependencies into one DependencyMap + spec_deps = self.flat_dependencies(copy=False) # Figure out which of the user-provided deps provide virtual deps. # Remove virtual deps that are already provided by something in the spec @@ -814,7 +838,7 @@ class Spec(object): # If there are deps specified but not visited, they're not # actually deps of this package. Raise an error. - extra = set(spec_deps.viewkeys()).difference(visited) + extra = set(spec_deps.keys()).difference(visited) # Also subtract out all the packags that provide a needed vpkg vdeps = [v for v in self.package.virtual_dependencies()] @@ -843,7 +867,7 @@ class Spec(object): If they're not, it will raise either UnknownPackageError or UnsupportedCompilerError. """ - for spec in self.preorder_traversal(): + for spec in self.traverse(): # Don't get a package for a virtual name. if not spec.virtual: spack.db.get(spec.name) @@ -911,17 +935,17 @@ class Spec(object): def common_dependencies(self, other): """Return names of dependencies that self an other have in common.""" common = set( - s.name for s in self.preorder_traversal(root=False)) + s.name for s in self.traverse(root=False)) common.intersection_update( - s.name for s in other.preorder_traversal(root=False)) + s.name for s in other.traverse(root=False)) return common def dep_difference(self, other): """Returns dependencies in self that are not in other.""" - mine = set(s.name for s in self.preorder_traversal(root=False)) + mine = set(s.name for s in self.traverse(root=False)) mine.difference_update( - s.name for s in other.preorder_traversal(root=False)) + s.name for s in other.traverse(root=False)) return mine @@ -980,8 +1004,8 @@ class Spec(object): return False # For virtual dependencies, we need to dig a little deeper. - self_index = ProviderIndex(self.preorder_traversal(), restrict=True) - other_index = ProviderIndex(other.preorder_traversal(), restrict=True) + self_index = ProviderIndex(self.traverse(), restrict=True) + other_index = ProviderIndex(other.traverse(), restrict=True) # This handles cases where there are already providers for both vpkgs if not self_index.satisfies(other_index): @@ -1003,7 +1027,7 @@ class Spec(object): def virtual_dependencies(self): """Return list of any virtual deps in this spec.""" - return [spec for spec in self.preorder_traversal() if spec.virtual] + return [spec for spec in self.traverse() if spec.virtual] def _dup(self, other, **kwargs): @@ -1018,22 +1042,29 @@ class Spec(object): Whether deps should be copied too. Set to false to copy a spec but not its dependencies. """ - # TODO: this needs to handle DAGs. + # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() self.variants = other.variants.copy() self.architecture = other.architecture - self.compiler = None - if other.compiler: - self.compiler = other.compiler.copy() - + self.compiler = other.compiler.copy() if other.compiler else None self.dependents = DependencyMap() - copy_deps = kwargs.get('dependencies', True) - if copy_deps: - self.dependencies = other.dependencies.copy() - else: - self.dependencies = DependencyMap() - + self.dependencies = DependencyMap() + + # If we copy dependencies, preserve DAG structure in the new spec + if kwargs.get('deps', True): + # This copies the deps from other using _dup(deps=False) + new_nodes = other.flat_dependencies() + new_nodes[self.name] = self + + # Hook everything up properly here by traversing. + for spec in other.traverse(cover='nodes'): + parent = new_nodes[spec.name] + for child in spec.dependencies: + if child not in parent.dependencies: + parent._add_dependency(new_nodes[child]) + + # Since we preserved structure, we can copy _normal safely. self._normal = other._normal self._concrete = other._concrete @@ -1057,7 +1088,7 @@ class Spec(object): def __getitem__(self, name): """TODO: reconcile __getitem__, _add_dependency, __contains__""" - for spec in self.preorder_traversal(): + for spec in self.traverse(): if spec.name == name: return spec @@ -1068,15 +1099,82 @@ class Spec(object): """True if this spec has any dependency that satisfies the supplied spec.""" spec = self._autospec(spec) - for s in self.preorder_traversal(): + for s in self.traverse(): if s.satisfies(spec): return True return False - def _cmp_key(self): + def sorted_deps(self): + """Return a list of all dependencies sorted by name.""" + deps = self.flat_dependencies() + return tuple(deps[name] for name in sorted(deps)) + + + def _eq_dag(self, other, vs, vo): + """Recursive helper for eq_dag and ne_dag. Does the actual DAG + traversal.""" + vs.add(id(self)) + vo.add(id(other)) + + if self.ne_node(other): + return False + + if len(self.dependencies) != len(other.dependencies): + return False + + ssorted = [self.dependencies[name] for name in sorted(self.dependencies)] + osorted = [other.dependencies[name] for name in sorted(other.dependencies)] + + for s, o in zip(ssorted, osorted): + visited_s = id(s) in vs + visited_o = id(o) in vo + + # Check for duplicate or non-equal dependencies + if visited_s != visited_o: return False + + # Skip visited nodes + if visited_s or visited_o: continue + + # Recursive check for equality + if not s._eq_dag(o, vs, vo): + return False + + return True + + + def eq_dag(self, other): + """True if the full dependency DAGs of specs are equal""" + return self._eq_dag(other, set(), set()) + + + def ne_dag(self, other): + """True if the full dependency DAGs of specs are not equal""" + return not self.eq_dag(other) + + + def _cmp_node(self): + """Comparison key for just *this node* and not its deps.""" return (self.name, self.versions, self.variants, - self.architecture, self.compiler, self.dependencies) + self.architecture, self.compiler) + + + def eq_node(self, other): + """Equality with another spec, not including dependencies.""" + return self._cmp_node() == other._cmp_node() + + + def ne_node(self, other): + """Inequality with another spec, not including dependencies.""" + return self._cmp_node() != other._cmp_node() + + + def _cmp_key(self): + """Comparison key for this node and all dependencies *without* + considering structure. This is the default, as + normalization will restore structure. + """ + return self._cmp_node() + (self.sorted_deps(),) def colorized(self): @@ -1179,12 +1277,12 @@ class Spec(object): return result + def dep_string(self): + return ''.join("^" + dep.format() for dep in self.sorted_deps()) + + def __str__(self): - by_name = lambda d: d.name - deps = self.preorder_traversal(key=by_name, root=False) - sorted_deps = sorted(deps, key=by_name) - dep_string = ''.join("^" + dep.format() for dep in sorted_deps) - return self.format() + dep_string + return self.format() + self.dep_string() def tree(self, **kwargs): @@ -1200,7 +1298,7 @@ class Spec(object): out = "" cur_id = 0 ids = {} - for d, node in self.preorder_traversal(cover=cover, depth=True): + for d, node in self.traverse(order='pre', cover=cover, depth=True): out += " " * indent if depth: out += "%-4d" % d diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 839555d630..3dac798396 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -120,8 +120,7 @@ class Stage(object): if spack.use_tmp_stage: # If we're using a tmp dir, it's a link, and it points at the right spot, # then keep it. - if (os.path.commonprefix((real_path, real_tmp)) == real_tmp - and os.path.exists(real_path)): + if (real_path.startswith(real_tmp) and os.path.exists(real_path)): return False else: # otherwise, just unlink it and start over. diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 5442189c2e..8ddc7f227d 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -45,7 +45,9 @@ test_names = ['versions', 'multimethod', 'install', 'package_sanity', - 'config'] + 'config', + 'directory_layout', + 'python_version'] def list_tests(): @@ -70,7 +72,7 @@ def run(names, verbose=False): runner = unittest.TextTestRunner(verbosity=verbosity) - testsRun = errors = failures = skipped = 0 + testsRun = errors = failures = 0 for test in names: module = 'spack.test.' + test print module @@ -81,12 +83,10 @@ def run(names, verbose=False): testsRun += result.testsRun errors += len(result.errors) failures += len(result.failures) - skipped += len(result.skipped) succeeded = not errors and not failures tty.msg("Tests Complete.", "%5d tests run" % testsRun, - "%5d skipped" % skipped, "%5d failures" % failures, "%5d errors" % errors) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 6ad2ef29d8..a7f4812c8c 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -134,29 +134,29 @@ class ConcretizeTest(MockPackagesTest): def test_virtual_is_fully_expanded_for_callpath(self): # force dependence on fake "zmpi" by asking for MPI 10.0 spec = Spec('callpath ^mpi@10.0') - self.assertIn('mpi', spec.dependencies) - self.assertNotIn('fake', spec) + self.assertTrue('mpi' in spec.dependencies) + self.assertFalse('fake' in spec) spec.concretize() - self.assertIn('zmpi', spec.dependencies) - self.assertNotIn('mpi', spec) - self.assertIn('fake', spec.dependencies['zmpi']) + self.assertTrue('zmpi' in spec.dependencies) + self.assertFalse('mpi' in spec) + self.assertTrue('fake' in spec.dependencies['zmpi']) def test_virtual_is_fully_expanded_for_mpileaks(self): spec = Spec('mpileaks ^mpi@10.0') - self.assertIn('mpi', spec.dependencies) - self.assertNotIn('fake', spec) + self.assertTrue('mpi' in spec.dependencies) + self.assertFalse('fake' in spec) spec.concretize() - self.assertIn('zmpi', spec.dependencies) - self.assertIn('callpath', spec.dependencies) - self.assertIn('zmpi', spec.dependencies['callpath'].dependencies) - self.assertIn('fake', spec.dependencies['callpath'].dependencies['zmpi'].dependencies) + self.assertTrue('zmpi' in spec.dependencies) + self.assertTrue('callpath' in spec.dependencies) + self.assertTrue('zmpi' in spec.dependencies['callpath'].dependencies) + self.assertTrue('fake' in spec.dependencies['callpath'].dependencies['zmpi'].dependencies) - self.assertNotIn('mpi', spec) + self.assertFalse('mpi' in spec) def test_my_dep_depends_on_provider_of_my_virtual_dep(self): diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py new file mode 100644 index 0000000000..3e52954cfe --- /dev/null +++ b/lib/spack/spack/test/directory_layout.py @@ -0,0 +1,155 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""\ +This test verifies that the Spack directory layout works properly. +""" +import unittest +import tempfile +import shutil +import os +from contextlib import closing + +from llnl.util.filesystem import * + +import spack +from spack.spec import Spec +from spack.packages import PackageDB +from spack.directory_layout import SpecHashDirectoryLayout + +class DirectoryLayoutTest(unittest.TestCase): + """Tests that a directory layout works correctly and produces a + consistent install path.""" + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.layout = SpecHashDirectoryLayout(self.tmpdir) + + + def tearDown(self): + shutil.rmtree(self.tmpdir, ignore_errors=True) + self.layout = None + + + def test_read_and_write_spec(self): + """This goes through each package in spack and creates a directory for + it. It then ensures that the spec for the directory's + installed package can be read back in consistently, and + finally that the directory can be removed by the directory + layout. + """ + for pkg in spack.db.all_packages(): + spec = pkg.spec + + # If a spec fails to concretize, just skip it. If it is a + # real error, it will be caught by concretization tests. + try: + spec.concretize() + except: + continue + + self.layout.make_path_for_spec(spec) + + install_dir = self.layout.path_for_spec(spec) + spec_path = self.layout.spec_file_path(spec) + + # Ensure directory has been created in right place. + self.assertTrue(os.path.isdir(install_dir)) + self.assertTrue(install_dir.startswith(self.tmpdir)) + + # Ensure spec file exists when directory is created + self.assertTrue(os.path.isfile(spec_path)) + self.assertTrue(spec_path.startswith(install_dir)) + + # Make sure spec file can be read back in to get the original spec + spec_from_file = self.layout.read_spec(spec_path) + self.assertEqual(spec, spec_from_file) + self.assertTrue(spec.eq_dag, spec_from_file) + self.assertTrue(spec_from_file.concrete) + + # Ensure that specs that come out "normal" are really normal. + with closing(open(spec_path)) as spec_file: + read_separately = Spec(spec_file.read()) + + read_separately.normalize() + self.assertEqual(read_separately, spec_from_file) + + read_separately.concretize() + self.assertEqual(read_separately, spec_from_file) + + # Make sure the dep hash of the read-in spec is the same + self.assertEqual(spec.dep_hash(), spec_from_file.dep_hash()) + + # Ensure directories are properly removed + self.layout.remove_path_for_spec(spec) + self.assertFalse(os.path.isdir(install_dir)) + self.assertFalse(os.path.exists(install_dir)) + + + def test_handle_unknown_package(self): + """This test ensures that spack can at least do *some* + operations with packages that are installed but that it + does not know about. This is actually not such an uncommon + scenario with spack; it can happen when you switch from a + git branch where you're working on a new package. + + This test ensures that the directory layout stores enough + information about installed packages' specs to uninstall + or query them again if the package goes away. + """ + mock_db = PackageDB(spack.mock_packages_path) + + not_in_mock = set(spack.db.all_package_names()).difference( + set(mock_db.all_package_names())) + + # Create all the packages that are not in mock. + installed_specs = {} + for pkg_name in not_in_mock: + spec = spack.db.get(pkg_name).spec + + # If a spec fails to concretize, just skip it. If it is a + # real error, it will be caught by concretization tests. + try: + spec.concretize() + except: + continue + + self.layout.make_path_for_spec(spec) + installed_specs[spec] = self.layout.path_for_spec(spec) + + tmp = spack.db + spack.db = mock_db + + # Now check that even without the package files, we know + # enough to read a spec from the spec file. + for spec, path in installed_specs.items(): + spec_from_file = self.layout.read_spec(join_path(path, '.spec')) + + # To satisfy these conditions, directory layouts need to + # read in concrete specs from their install dirs somehow. + self.assertEqual(path, self.layout.path_for_spec(spec_from_file)) + self.assertEqual(spec, spec_from_file) + self.assertEqual(spec.dep_hash(), spec_from_file.dep_hash()) + + spack.db = tmp diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index a92bd92289..8047ab92e3 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -25,15 +25,18 @@ import os import unittest import shutil +import tempfile from contextlib import closing from llnl.util.filesystem import * import spack from spack.stage import Stage +from spack.directory_layout import SpecHashDirectoryLayout from spack.util.executable import which from spack.test.mock_packages_test import * + dir_name = 'trivial-1.0' archive_name = 'trivial-1.0.tar.gz' install_test_package = 'trivial_install_test_package' @@ -66,9 +69,16 @@ class InstallTest(MockPackagesTest): tar = which('tar') tar('-czf', archive_name, dir_name) - # We use a fake pacakge, so skip the checksum. + # We use a fake package, so skip the checksum. spack.do_checksum = False + # Use a fake install directory to avoid conflicts bt/w + # installed pkgs and mock packages. + self.tmpdir = tempfile.mkdtemp() + self.orig_layout = spack.install_layout + spack.install_layout = SpecHashDirectoryLayout(self.tmpdir) + + def tearDown(self): super(InstallTest, self).tearDown() @@ -78,6 +88,10 @@ class InstallTest(MockPackagesTest): # Turn checksumming back on spack.do_checksum = True + # restore spack's layout. + spack.install_layout = self.orig_layout + shutil.rmtree(self.tmpdir, ignore_errors=True) + def test_install_and_uninstall(self): # Get a basic concrete spec for the trivial install package. diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index adde70ff6c..e948376039 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -39,7 +39,6 @@ def set_pkg_dep(pkg, spec): class MockPackagesTest(unittest.TestCase): - @classmethod def setUp(self): # Use the mock packages database for these tests. This allows # us to set up contrived packages that don't interfere with @@ -52,7 +51,7 @@ class MockPackagesTest(unittest.TestCase): 'site' : spack.mock_site_config, 'user' : spack.mock_user_config } - @classmethod + def tearDown(self): """Restore the real packages path after any test.""" spack.db = self.real_db diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py new file mode 100644 index 0000000000..04b4eadf34 --- /dev/null +++ b/lib/spack/spack/test/python_version.py @@ -0,0 +1,97 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +""" +This test ensures that all Spack files are Python version 2.6 or less. + +Spack was originally 2.7, but enough systems in 2014 are still using +2.6 on their frontend nodes that we need 2.6 to get adopted. +""" +import unittest +import os +import re +from contextlib import closing + +import llnl.util.tty as tty + +from external import pyqver2 +import spack + +spack_max_version = (2,6) + +class PythonVersionTest(unittest.TestCase): + + def spack_python_files(self): + # first file is the spack script. + yield spack.spack_file + yield os.path.join(spack.build_env_path, 'cc') + + # Next files are all the source files and package files. + search_paths = [spack.lib_path, spack.var_path] + + # Iterate through the whole spack source tree. + for path in search_paths: + for root, dirnames, filenames in os.walk(path): + for filename in filenames: + if re.match(r'^[^.#].*\.py$', filename): + yield os.path.join(root, filename) + + + def test_python_versions(self): + # dict version -> filename -> reasons + all_issues = {} + + for fn in self.spack_python_files(): + with closing(open(fn)) as pyfile: + versions = pyqver2.get_versions(pyfile.read()) + for ver, reasons in versions.items(): + if ver > spack_max_version: + if not ver in all_issues: + all_issues[ver] = {} + all_issues[ver][fn] = reasons + + if all_issues: + tty.error("Spack must run on Python version %d.%d" + % spack_max_version) + + for v in sorted(all_issues.keys(), reverse=True): + msgs = [] + for fn in sorted(all_issues[v].keys()): + short_fn = fn + if fn.startswith(spack.prefix): + short_fn = fn[len(spack.prefix):] + + reasons = [r for r in set(all_issues[v][fn]) if r] + for r in reasons: + msgs.append(("%s:%s" % ('spack' + short_fn, r[0]), r[1])) + + tty.error("These files require version %d.%d:" % v) + maxlen = max(len(f) for f, prob in msgs) + fmt = "%%-%ds%%s" % (maxlen+3) + print fmt % ('File', 'Reason') + print fmt % ('-' * (maxlen), '-' * 20) + for msg in msgs: + print fmt % msg + + self.assertTrue(len(all_issues) == 0) diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 0c0b214ab7..fb67aa8a8d 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -48,7 +48,7 @@ class SpecDagTest(MockPackagesTest): spec.package.validate_dependencies) - def test_unique_node_traversal(self): + def test_preorder_node_traversal(self): dag = Spec('mpileaks ^zmpi') dag.normalize() @@ -56,14 +56,14 @@ class SpecDagTest(MockPackagesTest): 'zmpi', 'fake'] pairs = zip([0,1,2,3,4,2,3], names) - traversal = dag.preorder_traversal() - self.assertListEqual([x.name for x in traversal], names) + traversal = dag.traverse() + self.assertEqual([x.name for x in traversal], names) - traversal = dag.preorder_traversal(depth=True) - self.assertListEqual([(x, y.name) for x,y in traversal], pairs) + traversal = dag.traverse(depth=True) + self.assertEqual([(x, y.name) for x,y in traversal], pairs) - def test_unique_edge_traversal(self): + def test_preorder_edge_traversal(self): dag = Spec('mpileaks ^zmpi') dag.normalize() @@ -71,14 +71,14 @@ class SpecDagTest(MockPackagesTest): 'libelf', 'zmpi', 'fake', 'zmpi'] pairs = zip([0,1,2,3,4,3,2,3,1], names) - traversal = dag.preorder_traversal(cover='edges') - self.assertListEqual([x.name for x in traversal], names) + traversal = dag.traverse(cover='edges') + self.assertEqual([x.name for x in traversal], names) - traversal = dag.preorder_traversal(cover='edges', depth=True) - self.assertListEqual([(x, y.name) for x,y in traversal], pairs) + traversal = dag.traverse(cover='edges', depth=True) + self.assertEqual([(x, y.name) for x,y in traversal], pairs) - def test_unique_path_traversal(self): + def test_preorder_path_traversal(self): dag = Spec('mpileaks ^zmpi') dag.normalize() @@ -86,11 +86,56 @@ class SpecDagTest(MockPackagesTest): 'libelf', 'zmpi', 'fake', 'zmpi', 'fake'] pairs = zip([0,1,2,3,4,3,2,3,1,2], names) - traversal = dag.preorder_traversal(cover='paths') - self.assertListEqual([x.name for x in traversal], names) + traversal = dag.traverse(cover='paths') + self.assertEqual([x.name for x in traversal], names) - traversal = dag.preorder_traversal(cover='paths', depth=True) - self.assertListEqual([(x, y.name) for x,y in traversal], pairs) + traversal = dag.traverse(cover='paths', depth=True) + self.assertEqual([(x, y.name) for x,y in traversal], pairs) + + + def test_postorder_node_traversal(self): + dag = Spec('mpileaks ^zmpi') + dag.normalize() + + names = ['libelf', 'libdwarf', 'dyninst', 'fake', 'zmpi', + 'callpath', 'mpileaks'] + pairs = zip([4,3,2,3,2,1,0], names) + + traversal = dag.traverse(order='post') + self.assertEqual([x.name for x in traversal], names) + + traversal = dag.traverse(depth=True, order='post') + self.assertEqual([(x, y.name) for x,y in traversal], pairs) + + + def test_postorder_edge_traversal(self): + dag = Spec('mpileaks ^zmpi') + dag.normalize() + + names = ['libelf', 'libdwarf', 'libelf', 'dyninst', 'fake', 'zmpi', + 'callpath', 'zmpi', 'mpileaks'] + pairs = zip([4,3,3,2,3,2,1,1,0], names) + + traversal = dag.traverse(cover='edges', order='post') + self.assertEqual([x.name for x in traversal], names) + + traversal = dag.traverse(cover='edges', depth=True, order='post') + self.assertEqual([(x, y.name) for x,y in traversal], pairs) + + + def test_postorder_path_traversal(self): + dag = Spec('mpileaks ^zmpi') + dag.normalize() + + names = ['libelf', 'libdwarf', 'libelf', 'dyninst', 'fake', 'zmpi', + 'callpath', 'fake', 'zmpi', 'mpileaks'] + pairs = zip([4,3,3,2,3,2,1,2,1,0], names) + + traversal = dag.traverse(cover='paths', order='post') + self.assertEqual([x.name for x in traversal], names) + + traversal = dag.traverse(cover='paths', depth=True, order='post') + self.assertEqual([(x, y.name) for x,y in traversal], pairs) def test_conflicting_spec_constraints(self): @@ -142,7 +187,7 @@ class SpecDagTest(MockPackagesTest): # make sure nothing with the same name occurs twice counts = {} - for spec in dag.preorder_traversal(keyfun=id): + for spec in dag.traverse(key=id): if not spec.name in counts: counts[spec.name] = 0 counts[spec.name] += 1 @@ -152,15 +197,15 @@ class SpecDagTest(MockPackagesTest): def check_links(self, spec_to_check): - for spec in spec_to_check.preorder_traversal(): + for spec in spec_to_check.traverse(): for dependent in spec.dependents.values(): - self.assertIn( - spec.name, dependent.dependencies, + self.assertTrue( + spec.name in dependent.dependencies, "%s not in dependencies of %s" % (spec.name, dependent.name)) for dependency in spec.dependencies.values(): - self.assertIn( - spec.name, dependency.dependents, + self.assertTrue( + spec.name in dependency.dependents, "%s not in dependents of %s" % (spec.name, dependency.name)) @@ -221,30 +266,53 @@ class SpecDagTest(MockPackagesTest): def test_equal(self): - spec = Spec('mpileaks ^callpath ^libelf ^libdwarf') - self.assertNotEqual(spec, Spec( - 'mpileaks', Spec('callpath', - Spec('libdwarf', - Spec('libelf'))))) - self.assertNotEqual(spec, Spec( - 'mpileaks', Spec('callpath', - Spec('libelf', - Spec('libdwarf'))))) + # Different spec structures to test for equality + flat = Spec('mpileaks ^callpath ^libelf ^libdwarf') + + flat_init = Spec( + 'mpileaks', Spec('callpath'), Spec('libdwarf'), Spec('libelf')) + + flip_flat = Spec( + 'mpileaks', Spec('libelf'), Spec('libdwarf'), Spec('callpath')) + + dag = Spec('mpileaks', Spec('callpath', + Spec('libdwarf', + Spec('libelf')))) - self.assertEqual(spec, Spec( - 'mpileaks', Spec('callpath'), Spec('libdwarf'), Spec('libelf'))) + flip_dag = Spec('mpileaks', Spec('callpath', + Spec('libelf', + Spec('libdwarf')))) - self.assertEqual(spec, Spec( - 'mpileaks', Spec('libelf'), Spec('libdwarf'), Spec('callpath'))) + # All these are equal to each other with regular == + specs = (flat, flat_init, flip_flat, dag, flip_dag) + for lhs, rhs in zip(specs, specs): + self.assertEqual(lhs, rhs) + self.assertEqual(str(lhs), str(rhs)) + + # Same DAGs constructed different ways are equal + self.assertTrue(flat.eq_dag(flat_init)) + + # order at same level does not matter -- (dep on same parent) + self.assertTrue(flat.eq_dag(flip_flat)) + + # DAGs should be unequal if nesting is different + self.assertFalse(flat.eq_dag(dag)) + self.assertFalse(flat.eq_dag(flip_dag)) + self.assertFalse(flip_flat.eq_dag(dag)) + self.assertFalse(flip_flat.eq_dag(flip_dag)) + self.assertFalse(dag.eq_dag(flip_dag)) def test_normalize_mpileaks(self): + # Spec parsed in from a string spec = Spec('mpileaks ^mpich ^callpath ^dyninst ^libelf@1.8.11 ^libdwarf') + # What that spec should look like after parsing expected_flat = Spec( 'mpileaks', Spec('mpich'), Spec('callpath'), Spec('dyninst'), Spec('libelf@1.8.11'), Spec('libdwarf')) + # What it should look like after normalization mpich = Spec('mpich') libelf = Spec('libelf@1.8.11') expected_normalized = Spec( @@ -257,7 +325,10 @@ class SpecDagTest(MockPackagesTest): mpich), mpich) - expected_non_unique_nodes = Spec( + # Similar to normalized spec, but now with copies of the same + # libelf node. Normalization should result in a single unique + # node for each package, so this is the wrong DAG. + non_unique_nodes = Spec( 'mpileaks', Spec('callpath', Spec('dyninst', @@ -267,21 +338,33 @@ class SpecDagTest(MockPackagesTest): mpich), Spec('mpich')) - self.assertEqual(expected_normalized, expected_non_unique_nodes) - - self.assertEqual(str(expected_normalized), str(expected_non_unique_nodes)) - self.assertEqual(str(spec), str(expected_non_unique_nodes)) - self.assertEqual(str(expected_normalized), str(spec)) + # All specs here should be equal under regular equality + specs = (spec, expected_flat, expected_normalized, non_unique_nodes) + for lhs, rhs in zip(specs, specs): + self.assertEqual(lhs, rhs) + self.assertEqual(str(lhs), str(rhs)) + # Test that equal and equal_dag are doing the right thing self.assertEqual(spec, expected_flat) - self.assertNotEqual(spec, expected_normalized) - self.assertNotEqual(spec, expected_non_unique_nodes) + self.assertTrue(spec.eq_dag(expected_flat)) + + self.assertEqual(spec, expected_normalized) + self.assertFalse(spec.eq_dag(expected_normalized)) + + self.assertEqual(spec, non_unique_nodes) + self.assertFalse(spec.eq_dag(non_unique_nodes)) spec.normalize() - self.assertNotEqual(spec, expected_flat) + # After normalizing, spec_dag_equal should match the normalized spec. + self.assertEqual(spec, expected_flat) + self.assertFalse(spec.eq_dag(expected_flat)) + self.assertEqual(spec, expected_normalized) - self.assertEqual(spec, expected_non_unique_nodes) + self.assertTrue(spec.eq_dag(expected_normalized)) + + self.assertEqual(spec, non_unique_nodes) + self.assertFalse(spec.eq_dag(non_unique_nodes)) def test_normalize_with_virtual_package(self): @@ -302,10 +385,63 @@ class SpecDagTest(MockPackagesTest): def test_contains(self): spec = Spec('mpileaks ^mpi ^libelf@1.8.11 ^libdwarf') - self.assertIn(Spec('mpi'), spec) - self.assertIn(Spec('libelf'), spec) - self.assertIn(Spec('libelf@1.8.11'), spec) - self.assertNotIn(Spec('libelf@1.8.12'), spec) - self.assertIn(Spec('libdwarf'), spec) - self.assertNotIn(Spec('libgoblin'), spec) - self.assertIn(Spec('mpileaks'), spec) + self.assertTrue(Spec('mpi') in spec) + self.assertTrue(Spec('libelf') in spec) + self.assertTrue(Spec('libelf@1.8.11') in spec) + self.assertFalse(Spec('libelf@1.8.12') in spec) + self.assertTrue(Spec('libdwarf') in spec) + self.assertFalse(Spec('libgoblin') in spec) + self.assertTrue(Spec('mpileaks') in spec) + + + def test_copy_simple(self): + orig = Spec('mpileaks') + copy = orig.copy() + + self.check_links(copy) + + self.assertEqual(orig, copy) + self.assertTrue(orig.eq_dag(copy)) + self.assertEqual(orig._normal, copy._normal) + self.assertEqual(orig._concrete, copy._concrete) + + # ensure no shared nodes bt/w orig and copy. + orig_ids = set(id(s) for s in orig.traverse()) + copy_ids = set(id(s) for s in copy.traverse()) + self.assertFalse(orig_ids.intersection(copy_ids)) + + + def test_copy_normalized(self): + orig = Spec('mpileaks') + orig.normalize() + copy = orig.copy() + + self.check_links(copy) + + self.assertEqual(orig, copy) + self.assertTrue(orig.eq_dag(copy)) + self.assertEqual(orig._normal, copy._normal) + self.assertEqual(orig._concrete, copy._concrete) + + # ensure no shared nodes bt/w orig and copy. + orig_ids = set(id(s) for s in orig.traverse()) + copy_ids = set(id(s) for s in copy.traverse()) + self.assertFalse(orig_ids.intersection(copy_ids)) + + + def test_copy_concretized(self): + orig = Spec('mpileaks') + orig.concretize() + copy = orig.copy() + + self.check_links(copy) + + self.assertEqual(orig, copy) + self.assertTrue(orig.eq_dag(copy)) + self.assertEqual(orig._normal, copy._normal) + self.assertEqual(orig._concrete, copy._concrete) + + # ensure no shared nodes bt/w orig and copy. + orig_ids = set(id(s) for s in orig.traverse()) + copy_ids = set(id(s) for s in copy.traverse()) + self.assertFalse(orig_ids.intersection(copy_ids)) diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index 08899f9810..a412549dc7 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -51,28 +51,20 @@ readme_text = "hello world!\n" stage_name = 'spack-test-stage' -class with_tmp(object): - """Decorator that executes a function with or without spack set to use - a temp dir. Spack allows builds to happen directly in the - stage directory or in a tmp dir and symlinked into the stage - directory, so this lets us use the same test in both cases. +@contextmanager +def use_tmp(use_tmp): + """Allow some test code to be executed with spack.use_tmp_stage + set to a certain value. Context manager makes sure it's reset + on failure. """ - def __init__(self, use_tmp): - self.use_tmp = use_tmp - - def __call__(self, fun): - use_tmp = self.use_tmp - def new_test_function(self): - old_tmp = spack.use_tmp_stage - spack.use_tmp_stage = use_tmp - fun(self) - spack.use_tmp_stage = old_tmp - return new_test_function + old_tmp = spack.use_tmp_stage + spack.use_tmp_stage = use_tmp + yield + spack.use_tmp_stage = old_tmp class StageTest(unittest.TestCase): - @classmethod - def setUpClass(cls): + def setUp(self): """This sets up a mock archive to fetch, and a mock temp space for use by the Stage class. It doesn't actually create the Stage -- that is done by individual tests. @@ -92,52 +84,58 @@ class StageTest(unittest.TestCase): tar('czf', archive_name, archive_dir) # Make spack use the test environment for tmp stuff. - cls.old_tmp_dirs = spack.tmp_dirs + self.old_tmp_dirs = spack.tmp_dirs spack.tmp_dirs = [test_tmp_path] + # record this since this test changes to directories that will + # be removed. + self.working_dir = os.getcwd() + - @classmethod - def tearDownClass(cls): + def tearDown(self): """Blows away the test environment directory.""" shutil.rmtree(test_files_dir) + # chdir back to original working dir + os.chdir(self.working_dir) + # restore spack's original tmp environment - spack.tmp_dirs = cls.old_tmp_dirs + spack.tmp_dirs = self.old_tmp_dirs def get_stage_path(self, stage, stage_name): - """Figure out based on a stage and an intended name where it should - be living. This depends on whether it's named or not. + """Figure out where a stage should be living. This depends on + whether it's named. """ - if stage_name: + if stage_name is not None: # If it is a named stage, we know where the stage should be - stage_path = join_path(spack.stage_path, stage_name) + return join_path(spack.stage_path, stage_name) else: # If it's unnamed, ensure that we ran mkdtemp in the right spot. - stage_path = stage.path - self.assertIsNotNone(stage_path) - self.assertEqual( - os.path.commonprefix((stage_path, spack.stage_path)), - spack.stage_path) - return stage_path + self.assertTrue(stage.path is not None) + self.assertTrue(stage.path.startswith(spack.stage_path)) + return stage.path def check_setup(self, stage, stage_name): """Figure out whether a stage was set up correctly.""" stage_path = self.get_stage_path(stage, stage_name) + + # Ensure stage was created in the spack stage directory self.assertTrue(os.path.isdir(stage_path)) if spack.use_tmp_stage: - # Make sure everything was created and linked correctly for - # a tmp stage. + # Check that the stage dir is really a symlink. self.assertTrue(os.path.islink(stage_path)) + # Make sure it points to a valid directory target = os.path.realpath(stage_path) self.assertTrue(os.path.isdir(target)) self.assertFalse(os.path.islink(target)) - self.assertEqual( - os.path.commonprefix((target, test_tmp_path)), - test_tmp_path) + + # Make sure the directory is in the place we asked it to + # be (see setUp and tearDown) + self.assertTrue(target.startswith(test_tmp_path)) else: # Make sure the stage path is NOT a link for a non-tmp stage @@ -146,15 +144,15 @@ class StageTest(unittest.TestCase): def check_fetch(self, stage, stage_name): stage_path = self.get_stage_path(stage, stage_name) - self.assertIn(archive_name, os.listdir(stage_path)) + self.assertTrue(archive_name in os.listdir(stage_path)) self.assertEqual(join_path(stage_path, archive_name), stage.archive_file) def check_expand_archive(self, stage, stage_name): stage_path = self.get_stage_path(stage, stage_name) - self.assertIn(archive_name, os.listdir(stage_path)) - self.assertIn(archive_dir, os.listdir(stage_path)) + self.assertTrue(archive_name in os.listdir(stage_path)) + self.assertTrue(archive_dir in os.listdir(stage_path)) self.assertEqual( join_path(stage_path, archive_dir), @@ -192,32 +190,40 @@ class StageTest(unittest.TestCase): self.assertFalse(os.path.exists(target)) - def checkSetupAndDestroy(self, stage_name=None): - stage = Stage(archive_url, name=stage_name) - self.check_setup(stage, stage_name) - - stage.destroy() - self.check_destroy(stage, stage_name) - - - @with_tmp(True) def test_setup_and_destroy_name_with_tmp(self): - self.checkSetupAndDestroy(stage_name) + with use_tmp(True): + stage = Stage(archive_url, name=stage_name) + self.check_setup(stage, stage_name) + + stage.destroy() + self.check_destroy(stage, stage_name) - @with_tmp(False) def test_setup_and_destroy_name_without_tmp(self): - self.checkSetupAndDestroy(stage_name) + with use_tmp(False): + stage = Stage(archive_url, name=stage_name) + self.check_setup(stage, stage_name) + + stage.destroy() + self.check_destroy(stage, stage_name) - @with_tmp(True) def test_setup_and_destroy_no_name_with_tmp(self): - self.checkSetupAndDestroy(None) + with use_tmp(True): + stage = Stage(archive_url) + self.check_setup(stage, None) + + stage.destroy() + self.check_destroy(stage, None) - @with_tmp(False) def test_setup_and_destroy_no_name_without_tmp(self): - self.checkSetupAndDestroy(None) + with use_tmp(False): + stage = Stage(archive_url) + self.check_setup(stage, None) + + stage.destroy() + self.check_destroy(stage, None) def test_chdir(self): @@ -286,7 +292,7 @@ class StageTest(unittest.TestCase): with closing(open('foobar', 'w')) as file: file.write("this file is to be destroyed.") - self.assertIn('foobar', os.listdir(stage.expanded_archive_path)) + self.assertTrue('foobar' in os.listdir(stage.expanded_archive_path)) # Make sure the file is not there after restage. stage.restage() @@ -295,7 +301,7 @@ class StageTest(unittest.TestCase): stage.chdir_to_archive() self.check_chdir_to_archive(stage, stage_name) - self.assertNotIn('foobar', os.listdir(stage.expanded_archive_path)) + self.assertFalse('foobar' in os.listdir(stage.expanded_archive_path)) stage.destroy() self.check_destroy(stage, stage_name) diff --git a/lib/spack/spack/util/crypto.py b/lib/spack/spack/util/crypto.py index 4d8681bed9..950e807596 100644 --- a/lib/spack/spack/util/crypto.py +++ b/lib/spack/spack/util/crypto.py @@ -35,7 +35,7 @@ _acceptable_hashes = [ hashlib.sha512 ] """Index for looking up hasher for a digest.""" -_size_to_hash = { h().digest_size : h for h in _acceptable_hashes } +_size_to_hash = dict((h().digest_size, h) for h in _acceptable_hashes) def checksum(hashlib_algo, filename, **kwargs): diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index bc27b25889..923c7c19a5 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -121,7 +121,7 @@ def which(name, **kwargs): for dir in path: exe = os.path.join(dir, name) - if os.access(exe, os.X_OK): + if os.path.isfile(exe) and os.access(exe, os.X_OK): return Executable(exe) if required: diff --git a/lib/spack/spack/util/string.py b/lib/spack/spack/util/string.py index e7a9a6862e..234163bf52 100644 --- a/lib/spack/spack/util/string.py +++ b/lib/spack/spack/util/string.py @@ -33,7 +33,9 @@ def comma_list(sequence, article=''): return sequence[0] else: out = ', '.join(str(s) for s in sequence[:-1]) - out += ', ' + if len(sequence) != 2: + out += ',' # oxford comma + out += ' ' if article: out += article + ' ' out += str(sequence[-1]) diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 4558f88384..fbf86db8e1 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -47,7 +47,8 @@ import os import sys import re from bisect import bisect_left -from functools import total_ordering, wraps +from functools import wraps +from external.functools import total_ordering import llnl.util.compare.none_high as none_high import llnl.util.compare.none_low as none_low |