Merge branch 'features/better-find' into develop

7 files changed, 210 insertions, 140 deletions

diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py
index ce7d0197f0..049d158c6d 100644
--- a/lib/spack/llnl/util/lang.py
+++ b/lib/spack/llnl/util/lang.py
@@ -68,6 +68,12 @@ def index_by(objects, *funcs):
 
            index1 = index_by(list_of_specs, 'arch', 'compiler')
            index2 = index_by(list_of_specs, 'compiler')
+
+       You can also index by tuples by passing tuples:
+
+           index1 = index_by(list_of_specs, ('arch', 'compiler'))
+
+       Keys in the resulting dict will look like ('gcc', 'bgqos_0').
     """
     if not funcs:
         return objects
@@ -75,6 +81,8 @@ def index_by(objects, *funcs):
     f = funcs[0]
     if isinstance(f, basestring):
         f = lambda x: getattr(x, funcs[0])
+    elif isinstance(f, tuple):
+        f = lambda x: tuple(getattr(x, p) for p in funcs[0])
 
     result = {}
     for o in objects:
diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py
index 652d1cc1a8..aba9e61f4f 100644
--- a/lib/spack/llnl/util/tty/__init__.py
+++ b/lib/spack/llnl/util/tty/__init__.py
@@ -25,6 +25,9 @@
 import sys
 import os
 import textwrap
+import fcntl
+import termios
+import struct
 from StringIO import StringIO
 
 from llnl.util.tty.color import *
@@ -142,20 +145,18 @@ def get_yes_or_no(prompt, **kwargs):
 
 
 def hline(label=None, **kwargs):
-    """Draw an optionally colored or labeled horizontal line.
+    """Draw a labeled horizontal line.
        Options:
-
        char       Char to draw the line with.  Default '-'
-       color      Color of the label.  Default is no color.
        max_width  Maximum width of the line.  Default is 64 chars.
-
-       See tty.color for possible color formats.
     """
-    char      = kwargs.get('char', '-')
-    color     = kwargs.get('color', '')
-    max_width = kwargs.get('max_width', 64)
+    char      = kwargs.pop('char', '-')
+    max_width = kwargs.pop('max_width', 64)
+    if kwargs:
+        raise TypeError("'%s' is an invalid keyword argument for this function."
+                        % next(kwargs.iterkeys()))
 
-    cols, rows = terminal_size()
+    rows, cols = terminal_size()
     if not cols:
         cols = max_width
     else:
@@ -163,37 +164,34 @@ def hline(label=None, **kwargs):
     cols = min(max_width, cols)
 
     label = str(label)
-    prefix = char * 2 + " " + label + " "
-    suffix = (cols - len(prefix)) * char
+    prefix = char * 2 + " "
+    suffix = " " + (cols - len(prefix) - clen(label)) * char
 
     out = StringIO()
-    if color:
-        prefix = char * 2 + " " + color + cescape(label) + "@. "
-        cwrite(prefix, stream=out, color=True)
-    else:
-        out.write(prefix)
+    out.write(prefix)
+    out.write(label)
     out.write(suffix)
 
     print out.getvalue()
 
 
 def terminal_size():
-    """Gets the dimensions of the console: cols, rows."""
+    """Gets the dimensions of the console: (rows, cols)."""
     def ioctl_GWINSZ(fd):
         try:
-            cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
+            rc = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
         except:
             return
-        return cr
-    cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
-    if not cr:
+        return rc
+    rc = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
+    if not rc:
         try:
             fd = os.open(os.ctermid(), os.O_RDONLY)
-            cr = ioctl_GWINSZ(fd)
+            rc = ioctl_GWINSZ(fd)
             os.close(fd)
         except:
             pass
-    if not cr:
-        cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80))
+    if not rc:
+        rc = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80))
 
-    return int(cr[1]), int(cr[0])
+    return int(rc[0]), int(rc[1])
diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py
index ff06241937..6b2909990c 100644
--- a/lib/spack/llnl/util/tty/colify.py
+++ b/lib/spack/llnl/util/tty/colify.py
@@ -22,16 +22,9 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-# colify
-# By Todd Gamblin, tgamblin@llnl.gov
-#
-# Takes a list of items as input and finds a good columnization of them,
-# similar to how gnu ls does.  You can pipe output to this script and
-# get a tight display for it.  This supports both uniform-width and
-# variable-width (tighter) columns.
-#
-# Run colify -h for more information.
-#
+"""
+Routines for printing columnar output.  See colify() for more information.
+"""
 import os
 import sys
 import fcntl
@@ -40,6 +33,7 @@ import struct
 from StringIO import StringIO
 
 from llnl.util.tty import terminal_size
+from llnl.util.tty.color import clen
 
 
 class ColumnConfig:
@@ -47,32 +41,52 @@ class ColumnConfig:
         self.cols = cols
         self.line_length = 0
         self.valid = True
-        self.widths = [0] * cols
+        self.widths = [0] * cols   # does not include ansi colors
+        self.cwidths = [0] * cols  # includes ansi colors
 
     def __repr__(self):
         attrs = [(a,getattr(self, a)) for a in dir(self) if not a.startswith("__")]
         return "<Config: %s>" % ", ".join("%s: %r" % a for a in attrs)
 
 
-def config_variable_cols(elts, console_cols, padding):
+def config_variable_cols(elts, console_width, padding, cols=0):
+    """Variable-width column fitting algorithm.
+
+       This function determines the most columns that can fit in the
+       screen width.  Unlike uniform fitting, where all columns take
+       the width of the longest element in the list, each column takes
+       the width of its own longest element. This packs elements more
+       efficiently on screen.
+
+       If cols is nonzero, force
+    """
+    if cols < 0:
+        raise ValueError("cols must be non-negative.")
+
     # Get a bound on the most columns we could possibly have.
-    lengths = [len(elt) for elt in elts]
-    max_cols = max(1, console_cols / (min(lengths) + padding))
+    # 'clen' ignores length of ansi color sequences.
+    lengths = [clen(e) for e in elts]
+    clengths = [len(e) for e in elts]
+
+    max_cols = max(1, console_width / (min(lengths) + padding))
     max_cols = min(len(elts), max_cols)
 
-    configs = [ColumnConfig(c) for c in xrange(1, max_cols+1)]
-    for elt, length in enumerate(lengths):
-        for i, conf in enumerate(configs):
+    # Range of column counts to try.  If forced, use the supplied value.
+    col_range = [cols] if cols else xrange(1, max_cols+1)
+
+    # Determine the most columns possible for the console width.
+    configs = [ColumnConfig(c) for c in col_range]
+    for i, length in enumerate(lengths):
+        for conf in configs:
             if conf.valid:
-                col = elt / ((len(elts) + i) / (i + 1))
-                padded = length
-                if col < i:
-                    padded += padding
+                col = i / ((len(elts) + conf.cols - 1) / conf.cols)
+                p = padding if col < (conf.cols - 1) else 0
 
-                if conf.widths[col] < padded:
-                    conf.line_length += padded - conf.widths[col]
-                    conf.widths[col] = padded
-                    conf.valid = (conf.line_length < console_cols)
+                if conf.widths[col] < (length + p):
+                    conf.line_length += length + p - conf.widths[col]
+                    conf.widths[col]  = length + p
+                    conf.cwidths[col] = clengths[i] + p
+                    conf.valid = (conf.line_length < console_width)
 
     try:
         config = next(conf for conf in reversed(configs) if conf.valid)
@@ -85,57 +99,98 @@ def config_variable_cols(elts, console_cols, padding):
     return config
 
 
-def config_uniform_cols(elts, console_cols, padding):
-    max_len = max(len(elt) for elt in elts) + padding
-    cols = max(1, console_cols / max_len)
-    cols = min(len(elts), cols)
+def config_uniform_cols(elts, console_width, padding, cols=0):
+    """Uniform-width column fitting algorithm.
+
+       Determines the longest element in the list, and determines how
+       many columns of that width will fit on screen.  Returns a
+       corresponding column config.
+    """
+    if cols < 0:
+        raise ValueError("cols must be non-negative.")
+
+    # 'clen' ignores length of ansi color sequences.
+    max_len = max(clen(e) for e in elts) + padding
+    max_clen = max(len(e) for e in elts) + padding
+    if cols == 0:
+        cols = max(1, console_width / max_len)
+        cols = min(len(elts), cols)
+
     config = ColumnConfig(cols)
     config.widths = [max_len] * cols
+    config.cwidths = [max_clen] * cols
+
     return config
 
 
-def isatty(ostream):
-    force = os.environ.get('COLIFY_TTY', 'false').lower() != 'false'
-    return force or ostream.isatty()
+def colify(elts, **options):
+    """Takes a list of elements as input and finds a good columnization
+    of them, similar to how gnu ls does. This supports both
+    uniform-width and variable-width (tighter) columns.
 
+    If elts is not a list of strings, each element is first conveted
+    using str().
 
-def colify(elts, **options):
+    Keyword arguments:
+
+    output=<stream>   A file object to write to.  Default is sys.stdout.
+    indent=<int>      Optionally indent all columns by some number of spaces.
+    padding=<int>     Spaces between columns.  Default is 2.
+    width=<int>       Width of the output.  Default is 80 if tty is not detected.
+
+    cols=<int>        Force number of columns. Default is to size to terminal,
+                      or single-column if no tty
+
+    tty=<bool>        Whether to attempt to write to a tty.  Default is to
+                      autodetect a tty. Set to False to force single-column output.
+
+    method=<string>   Method to use to fit columns.  Options are variable or uniform.
+                      Variable-width columns are tighter, uniform columns are all the
+                      same width and fit less data on the screen.
+
+    len=<func>        Function to use for calculating string length.
+                      Useful for ignoring ansi color. Default is 'len'.
+    """
     # Get keyword arguments or set defaults
-    output       = options.get("output", sys.stdout)
-    indent       = options.get("indent", 0)
-    padding      = options.get("padding", 2)
-    tty          = options.get('tty', None)
+    cols         = options.pop("cols", 0)
+    output       = options.pop("output", sys.stdout)
+    indent       = options.pop("indent", 0)
+    padding      = options.pop("padding", 2)
+    tty          = options.pop('tty', None)
+    method       = options.pop("method", "variable")
+    console_cols = options.pop("width", None)
+
+    if options:
+        raise TypeError("'%s' is an invalid keyword argument for this function."
+                        % next(options.iterkeys()))
 
     # elts needs to be an array of strings so we can count the elements
     elts = [str(elt) for elt in elts]
     if not elts:
         return (0, ())
 
+    # Use only one column if not a tty.
     if not tty:
-        if tty is False or not isatty(output):
-            for elt in elts:
-                output.write("%s\n" % elt)
-
-            maxlen = max(len(str(s)) for s in elts)
-            return (1, (maxlen,))
+        if tty is False or not output.isatty():
+            cols = 1
 
-    console_cols = options.get("cols", None)
+    # Specify the number of character columns to use.
     if not console_cols:
-        console_cols, console_rows = terminal_size()
+        console_rows, console_cols = terminal_size()
     elif type(console_cols) != int:
         raise ValueError("Number of columns must be an int")
     console_cols = max(1, console_cols - indent)
 
-    method = options.get("method", "variable")
+    # Choose a method.  Variable-width colums vs uniform-width.
     if method == "variable":
-        config = config_variable_cols(elts, console_cols, padding)
+        config = config_variable_cols(elts, console_cols, padding, cols)
     elif method == "uniform":
-        config = config_uniform_cols(elts, console_cols, padding)
+        config = config_uniform_cols(elts, console_cols, padding, cols)
     else:
         raise ValueError("method must be one of: " + allowed_methods)
 
     cols = config.cols
-    formats = ["%%-%ds" % width for width in config.widths[:-1]]
+    formats = ["%%-%ds" % width for width in config.cwidths[:-1]]
     formats.append("%s")  # last column has no trailing space
 
     rows = (len(elts) + cols - 1) / cols
@@ -155,6 +210,25 @@ def colify(elts, **options):
     return (config.cols, tuple(config.widths))
 
 
+def colify_table(table, **options):
+    if table is None:
+        raise TypeError("Can't call colify_table on NoneType")
+    elif not table or not table[0]:
+        raise ValueError("Table is empty in colify_table!")
+
+    columns = len(table[0])
+    def transpose():
+        for i in xrange(columns):
+            for row in table:
+                yield row[i]
+
+    if 'cols' in options:
+        raise ValueError("Cannot override columsn in colify_table.")
+    options['cols'] = columns
+
+    colify(transpose(), **options)
+
+
 def colified(elts, **options):
     """Invokes the colify() function but returns the result as a string
        instead of writing it to an output string."""
@@ -162,29 +236,3 @@ def colified(elts, **options):
     options['output'] = sio
     colify(elts, **options)
     return sio.getvalue()
-
-
-if __name__ == "__main__":
-    import optparse
-
-    cols, rows = terminal_size()
-    parser = optparse.OptionParser()
-    parser.add_option("-u", "--uniform", action="store_true", default=False,
-                      help="Use uniformly sized columns instead of variable-size.")
-    parser.add_option("-p", "--padding", metavar="PADDING", action="store",
-                      type=int, default=2, help="Spaces to add between columns.  Default is 2.")
-    parser.add_option("-i", "--indent", metavar="SPACES", action="store",
-                      type=int, default=0, help="Indent the output by SPACES.  Default is 0.")
-    parser.add_option("-w", "--width", metavar="COLS", action="store",
-                      type=int, default=cols, help="Indent the output by SPACES.  Default is 0.")
-    options, args = parser.parse_args()
-
-    method = "variable"
-    if options.uniform:
-        method = "uniform"
-
-    if sys.stdin.isatty():
-        parser.print_help()
-        sys.exit(1)
-    else:
-        colify([line.strip() for line in sys.stdin], method=method, **options.__dict__)
diff --git a/lib/spack/llnl/util/tty/color.py b/lib/spack/llnl/util/tty/color.py
index 14974a1014..598e9d44f5 100644
--- a/lib/spack/llnl/util/tty/color.py
+++ b/lib/spack/llnl/util/tty/color.py
@@ -149,6 +149,11 @@ def colorize(string, **kwargs):
     return re.sub(color_re, match_to_ansi(color), string)
 
 
+def clen(string):
+    """Return the length of a string, excluding ansi color sequences."""
+    return len(re.sub(r'\033[^m]*m', '', string))
+
+
 def cwrite(string, stream=sys.stdout, color=None):
     """Replace all color expressions in string with ANSI control
        codes and write the result to the stream.  If color is
diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py
index 5c46a3536d..e37f44b3b7 100644
--- a/lib/spack/spack/cmd/compiler.py
+++ b/lib/spack/spack/cmd/compiler.py
@@ -25,6 +25,7 @@
 from external import argparse
 
 import llnl.util.tty as tty
+from llnl.util.tty.color import colorize
 from llnl.util.tty.colify import colify
 from llnl.util.lang import index_by
 
@@ -96,9 +97,12 @@ def compiler_info(args):
 def compiler_list(args):
     tty.msg("Available compilers")
     index = index_by(spack.compilers.all_compilers(), 'name')
-    for name, compilers in index.items():
-        tty.hline(name, char='-', color=spack.spec.compiler_color)
-        colify(reversed(sorted(compilers)), indent=4)
+    for i, (name, compilers) in enumerate(index.items()):
+        if i >= 1: print
+
+        cname = "%s{%s}" % (spack.spec.compiler_color, name)
+        tty.hline(colorize(cname), char='-')
+        colify(reversed(sorted(compilers)))
 
 
 def compiler(parser, args):
diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py
index 2238484a21..1de3413d42 100644
--- a/lib/spack/spack/cmd/find.py
+++ b/lib/spack/spack/cmd/find.py
@@ -24,13 +24,14 @@
 ##############################################################################
 import sys
 import collections
+import itertools
 from external import argparse
 from StringIO import StringIO
 
 import llnl.util.tty as tty
-from llnl.util.tty.colify import colify
+from llnl.util.tty.colify import *
 from llnl.util.tty.color import *
-from llnl.util.lang import partition_list, index_by
+from llnl.util.lang import *
 
 import spack
 import spack.spec
@@ -40,11 +41,14 @@ description ="Find installed spack packages"
 def setup_parser(subparser):
     format_group = subparser.add_mutually_exclusive_group()
     format_group.add_argument(
+        '-l', '--long', action='store_true', dest='long',
+        help='Show dependency hashes as well as versions.')
+    format_group.add_argument(
         '-p', '--paths', action='store_true', dest='paths',
         help='Show paths to package install directories')
     format_group.add_argument(
-        '-l', '--long', action='store_true', dest='full_specs',
-        help='Show full-length specs of installed packages')
+        '-d', '--deps', action='store_true', dest='full_deps',
+        help='Show full dependency DAG of installed packages')
 
     subparser.add_argument(
         'query_specs', nargs=argparse.REMAINDER,
@@ -65,39 +69,43 @@ def find(parser, args):
         if not query_specs:
             return
 
-    specs = [s for s in spack.db.installed_package_specs()
-             if not query_specs or any(s.satisfies(q) for q in query_specs)]
+    # Get all the specs the user asked for
+    if not query_specs:
+        specs = set(spack.db.installed_package_specs())
+    else:
+        results = [set(spack.db.get_installed(qs)) for qs in query_specs]
+        specs = set.union(*results)
 
     # Make a dict with specs keyed by architecture and compiler.
-    index = index_by(specs, 'architecture', 'compiler')
+    index = index_by(specs, ('architecture', 'compiler'))
 
     # Traverse the index and print out each package
-    for architecture in index:
-        tty.hline(architecture, char='=', color=spack.spec.architecture_color)
-        for compiler in index[architecture]:
-            tty.hline(compiler, char='-', color=spack.spec.compiler_color)
-
-            specs = index[architecture][compiler]
-            specs.sort()
-
-            abbreviated = [s.format('$_$@$+$#', color=True) for s in specs]
-
-            if args.paths:
-                # Print one spec per line along with prefix path
-                width = max(len(s) for s in abbreviated)
-                width += 2
-                format = "    %-{}s%s".format(width)
-
-                for abbrv, spec in zip(abbreviated, specs):
-                    print format % (abbrv, spec.prefix)
-
-            elif args.full_specs:
-                for spec in specs:
-                    print spec.tree(indent=4, format='$_$@$+', color=True),
-            else:
-                max_len = max([len(s.name) for s in specs])
-                max_len += 4
-
-                for spec in specs:
-                    format = '$-' + str(max_len) + '_$@$+$#'
-                    print "   " + spec.format(format, color=True)
+    for i, (architecture, compiler) in enumerate(sorted(index)):
+        if i > 0: print
+
+        header = "%s{%s} / %s{%s}" % (
+            spack.spec.architecture_color, architecture,
+            spack.spec.compiler_color, compiler)
+        tty.hline(colorize(header), char='-')
+
+        specs = index[(architecture,compiler)]
+        specs.sort()
+
+        abbreviated = [s.format('$_$@$+', color=True) for s in specs]
+        if args.paths:
+            # Print one spec per line along with prefix path
+            width = max(len(s) for s in abbreviated)
+            width += 2
+            format = "    %-{}s%s".format(width)
+
+            for abbrv, spec in zip(abbreviated, specs):
+                print format % (abbrv, spec.prefix)
+
+        elif args.full_deps:
+            for spec in specs:
+                print spec.tree(indent=4, format='$_$@$+', color=True),
+        else:
+            fmt = '$-_$@$+'
+            if args.long:
+                fmt += '$#'
+            colify(s.format(fmt, color=True) for s in specs)
diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py
index 5c7051d6a9..1f0978a18e 100644
--- a/lib/spack/spack/cmd/list.py
+++ b/lib/spack/spack/cmd/list.py
@@ -61,5 +61,4 @@ def list(parser, args):
     indent=0
     if sys.stdout.isatty():
         tty.msg("%d packages." % len(sorted_packages))
-        indent=2
     colify(sorted_packages, indent=indent)