First cut concretization works, with tests.

Mock packages now all have their own version lists.

18 files changed, 268 insertions, 119 deletions

diff --git a/lib/spack/spack/cmd/spec.py b/lib/spack/spack/cmd/spec.py
index f87e94dae6..36bfc5e154 100644
--- a/lib/spack/spack/cmd/spec.py
+++ b/lib/spack/spack/cmd/spec.py
@@ -14,3 +14,6 @@ def spec(parser, args):
     for spec in specs:
         spec.normalize()
         print spec.tree()
+
+        spec.concretize()
+        print spec.tree()
diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py
index c9752fbbac..b1c8df4eb3 100644
--- a/lib/spack/spack/cmd/test.py
+++ b/lib/spack/spack/cmd/test.py
@@ -17,13 +17,21 @@ def setup_parser(subparser):
         help="verbose output")
 
 
+def find_test_modules():
+    """Include all the modules under test, unless they set skip_test=True"""
+    for name in list_modules(spack.test_path):
+        module = __import__('spack.test.' + name, fromlist='skip_test')
+        if not getattr(module, 'skip_test', False):
+            yield name
+
+
 def test(parser, args):
     if args.list:
         print "Available tests:"
-        colify(list_modules(spack.test_path, directories=False))
+        colify(find_test_modules())
 
     elif not args.names:
-        for name in list_modules(spack.test_path, directories=False):
+        for name in find_test_modules():
             print "Running Tests: %s" % name
             spack.test.run(name, verbose=args.verbose)
 
diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index ac1d4421da..6acae39039 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -24,12 +24,12 @@ def concretize_version(spec):
     if spec.versions.concrete:
         return
 
-    pkg = speck.package
-    available = pkg.available_versions
+    pkg = spec.package
 
     # If there are known avaialble versions, return the most recent
-    if versions:
-        spec.versions = ver([avaialble[-1]])
+    available_versions = pkg.available_versions
+    if available_versions:
+        spec.versions = ver([available_versions[-1]])
     else:
         spec.versions = ver([pkg.version])
 
@@ -73,7 +73,7 @@ def concretize_compiler(spec):
        build with the compiler that will be used by libraries that
        link to this one, to maximize compatibility.
     """
-    if spec.compiler.concrete:
+    if spec.compiler and spec.compiler.concrete:
         if spec.compiler != spack.compilers.default_compiler():
             raise spack.spec.UnknownCompilerError(str(spec.compiler))
     else:
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 3658277634..d61ab4620d 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -298,6 +298,11 @@ class Package(object):
         # This is set by scraping a web page.
         self._available_versions = None
 
+        # This list overrides available_versions if set by the user.
+        attr.setdefault(self, 'versions', None)
+        if self.versions and type(self.versions) != VersionList:
+            self.versions = VersionList(self.versions)
+
         # stage used to build this package.
         self.stage = Stage("%s-%s" % (self.name, self.version), self.url)
 
@@ -637,6 +642,11 @@ class Package(object):
 
     @property
     def available_versions(self):
+        # If the package overrode available_versions, then use that.
+        if self.versions is not None:
+            return self.versions
+
+        # If not, then try to fetch using list_url
         if not self._available_versions:
             self._available_versions = ver([self.version])
             try:
@@ -656,7 +666,7 @@ class Package(object):
                              "to the package to tell Spack where to look for versions.")
 
             except subprocess.CalledProcessError:
-                tty.warn("Fetching %s failed." % self.list_url,
+                tty.warn("Could not connect to %s" % self.list_url,
                          "Package.available_versions requires an internet connection.",
                          "Version list may be incomplete.")
 
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index f7f4299942..e4203e85fb 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -170,7 +170,7 @@ class Compiler(object):
     def __str__(self):
         out = self.name
         if self.versions:
-            vlist = ",".join(str(v) for v in sorted(self.versions))
+            vlist = ",".join(str(v) for v in self.versions)
             out += "@%s" % vlist
         return out
 
@@ -190,6 +190,10 @@ class Variant(object):
         return (self.name, self.enabled)
 
 
+    def copy(self):
+        return Variant(self.name, self.enabled)
+
+
     def __str__(self):
         out = '+' if self.enabled else '~'
         return out + self.name
@@ -350,43 +354,58 @@ class Spec(object):
                 yield spec
 
 
-    def _concretize(self):
+    def _concretize_helper(self, presets):
+        for name in sorted(self.dependencies.keys()):
+            self.dependencies[name]._concretize_helper(presets)
+
+        if self.name in presets:
+            self.constrain(presets[self.name])
+        else:
+            spack.concretize.concretize_architecture(self)
+            spack.concretize.concretize_compiler(self)
+            spack.concretize.concretize_version(self)
+            presets[self.name] = self
+
+
+    def concretize(self, *presets):
         """A spec is concrete if it describes one build of a package uniquely.
            This will ensure that this spec is concrete.
 
            If this spec could describe more than one version, variant, or build
-           of a package, this will resolve it to be concrete.
-
-           Ensures that the spec is in canonical form.
+           of a package, this will add constraints to make it concrete.
 
-           This means:
-           1. All dependencies of this package and of its dependencies are
-              in the dependencies list (transitive closure of deps).
-           2. All dependencies in the dependencies list are canonicalized.
-
-           This function also serves to validate the spec, in that it makes sure
-           that each package exists an that spec criteria don't violate package
-           criteria.
+           Some rigorous validation and checks are also performed on the spec.
+           Concretizing ensures that it is self-consistent and that it's consistent
+           with requirements of its pacakges.  See flatten() and normalize() for
+           more details on this.
         """
-        # TODO: modularize the process of selecting concrete versions.
-        # There should be a set of user-configurable policies for these decisions.
-        self.validate()
-
-        # take the system's architecture for starters
-        if not self.architecture:
-             self.architecture = arch.sys_type()
-
-        if self.compiler:
-            self.compiler._concretize()
-
-        # TODO: handle variants.
-
-        # Take the highest version in a range
-        if not self.versions.concrete:
-            preferred = self.versions.highest() or self.package.version
-            self.versions = VersionList([preferred])
-
-        # Ensure dependencies have right versions
+        # Build specs out of user-provided presets
+        specs = [Spec(p) for p in presets]
+
+        # Concretize the presets first.  They could be partial specs, like just
+        # a particular version that the caller wants.
+        for spec in specs:
+            if not spec.concrete:
+                try:
+                    spec.concretize()
+                except UnsatisfiableSpecError, e:
+                    e.message = ("Unsatisfiable preset in concretize: %s."
+                                 % e.message)
+                    raise e
+
+        # build preset specs into a map
+        preset_dict = {spec.name : spec for spec in specs}
+
+        # Concretize bottom up, passing in presets to force concretization
+        # for certain specs.
+        self.normalize()
+        self._concretize_helper(preset_dict)
+
+
+    def concretized(self, *presets):
+        clone = self.copy()
+        clone.concretize(*presets)
+        return clone
 
 
     def flat_dependencies(self):
@@ -406,7 +425,9 @@ class Spec(object):
         try:
             for spec in self.preorder_traversal():
                 if spec.name not in flat_deps:
-                    flat_deps[spec.name] = spec
+                    new_spec = spec.copy(dependencies=False)
+                    flat_deps[spec.name] = new_spec
+
                 else:
                     flat_deps[spec.name].constrain(spec)
 
@@ -466,7 +487,7 @@ class Spec(object):
         # provided spec is sane, and that all dependency specs are in the
         # root node of the spec.  flat_dependencies will do this for us.
         spec_deps = self.flat_dependencies()
-        self.dependencies = DependencyMap()
+        self.dependencies.clear()
 
         visited = set()
         self._normalize_helper(visited, spec_deps)
@@ -523,30 +544,37 @@ class Spec(object):
                 self.dependencies.satisfies(other.dependencies))
 
 
-    def concretized(self):
-        clone = self.copy()
-        clone._concretize()
-        return clone
-
-
-    def _dup(self, other):
+    def _dup(self, other, **kwargs):
         """Copy the spec other into self.  This is a
-           first-party, overwriting copy."""
+           first-party, overwriting copy.  This does not copy
+           parent; if the other spec has a parent, this one will not.
+           To duplicate an entire DAG, Duplicate the root of the DAG.
+        """
         # TODO: this needs to handle DAGs.
         self.name = other.name
+        self.parent = None
         self.versions = other.versions.copy()
         self.variants = other.variants.copy()
         self.architecture = other.architecture
         self.compiler = None
         if other.compiler:
             self.compiler = other.compiler.copy()
-        self.dependencies = other.dependencies.copy()
 
+        copy_deps = kwargs.get('dependencies', True)
+        if copy_deps:
+            self.dependencies = other.dependencies.copy()
+        else:
+            self.dependencies = DependencyMap()
 
-    def copy(self):
-        """Return a deep copy of this spec."""
-        return Spec(self)
 
+    def copy(self, **kwargs):
+        """Return a copy of this spec.
+           By default, returns a deep copy.  Supply dependencies=False
+           to get a shallow copy.
+        """
+        clone = Spec.__new__(Spec)
+        clone._dup(self, **kwargs)
+        return clone
 
     @property
     def version(self):
@@ -564,7 +592,7 @@ class Spec(object):
         return colorize_spec(self)
 
 
-    def str_no_deps(self):
+    def str_no_deps(self, **kwargs):
         out = self.name
 
         # If the version range is entirely open, omit it
@@ -579,12 +607,17 @@ class Spec(object):
         if self.architecture:
             out += "=%s" % self.architecture
 
-        return out
+        if kwargs.get('color', False):
+            return colorize_spec(out)
+        else:
+            return out
 
 
-    def tree(self):
+    def tree(self, **kwargs):
         """Prints out this spec and its dependencies, tree-formatted
            with indentation."""
+        color = kwargs.get('color', False)
+
         out = ""
         cur_id = 0
         ids = {}
@@ -594,7 +627,7 @@ class Spec(object):
                 ids[id(node)] = cur_id
             out += str(ids[id(node)])
             out += " "+ ("    " * d)
-            out += node.str_no_deps() + "\n"
+            out += node.str_no_deps(color=color) + "\n"
         return out
 
 
diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py
index f010d09c0e..c95db03609 100644
--- a/lib/spack/spack/test/concretize.py
+++ b/lib/spack/spack/test/concretize.py
@@ -1,17 +1,60 @@
 import unittest
 from spack.spec import Spec
+from spack.test.mock_packages_test import *
 
+class ConcretizeTest(MockPackagesTest):
 
-class ConcretizeTest(unittest.TestCase):
+    def check_spec(self, abstract, concrete):
+        if abstract.versions.concrete:
+            self.assertEqual(abstract.versions, concrete.versions)
 
-    def check_concretize(self, abstract_spec):
+        if abstract.variants:
+            self.assertEqual(abstract.versions, concrete.versions)
+
+        if abstract.compiler and abstract.compiler.concrete:
+            self.assertEqual(abstract.compiler, concrete.compiler)
+
+        if abstract.architecture and abstract.architecture.concrete:
+            self.assertEqual(abstract.architecture, concrete.architecture)
+
+
+    def check_concretize(self, abstract_spec, *presets):
         abstract = Spec(abstract_spec)
-        print abstract
-        print abstract.concretized()
-        print abstract.concretized().concrete
-        self.assertTrue(abstract.concretized().concrete)
+        concrete = abstract.concretized(*presets)
+
+        self.assertFalse(abstract.concrete)
+        self.assertTrue(concrete.concrete)
+        self.check_spec(abstract, concrete)
+
+        return concrete
+
+
+    def check_presets(self, abstract, *presets):
+        abstract = Spec(abstract)
+        concrete = self.check_concretize(abstract, *presets)
+
+        flat_deps = concrete.flat_dependencies()
+        for preset in presets:
+            preset_spec = Spec(preset)
+            name = preset_spec.name
+
+            self.assertTrue(name in flat_deps)
+            self.check_spec(preset_spec, flat_deps[name])
+
+        return concrete
+
+
+    def test_concretize_no_deps(self):
+        self.check_concretize('libelf')
+        self.check_concretize('libelf@0.8.13')
+
+
+    def test_concretize_dag(self):
+        self.check_concretize('mpileaks')
+        self.check_concretize('callpath')
 
 
-    def test_packages(self):
-        pass
-        #self.check_concretize("libelf")
+    def test_concretize_with_presets(self):
+        self.check_presets('mpileaks', 'callpath@0.8')
+        self.check_presets('mpileaks', 'callpath@0.9', 'dyninst@8.0+debug')
+        self.check_concretize('callpath', 'libelf@0.8.13+debug~foo', 'mpich@1.0')
diff --git a/lib/spack/spack/test/mock_packages/__init__.py b/lib/spack/spack/test/mock_packages/__init__.py
index e69de29bb2..e6a8f467e6 100644
--- a/lib/spack/spack/test/mock_packages/__init__.py
+++ b/lib/spack/spack/test/mock_packages/__init__.py
@@ -0,0 +1 @@
+skip_test = True
diff --git a/lib/spack/spack/test/mock_packages/callpath.py b/lib/spack/spack/test/mock_packages/callpath.py
index 958960e0ab..4ca1a57007 100644
--- a/lib/spack/spack/test/mock_packages/callpath.py
+++ b/lib/spack/spack/test/mock_packages/callpath.py
@@ -5,6 +5,8 @@ class Callpath(Package):
     url      = "http://github.com/tgamblin/callpath-0.2.tar.gz"
     md5      = "foobarbaz"
 
+    versions = [0.8, 0.9, 1.0]
+
     depends_on("dyninst")
     depends_on("mpich")
 
diff --git a/lib/spack/spack/test/mock_packages/dyninst.py b/lib/spack/spack/test/mock_packages/dyninst.py
index f550cde54f..99bbd1507b 100644
--- a/lib/spack/spack/test/mock_packages/dyninst.py
+++ b/lib/spack/spack/test/mock_packages/dyninst.py
@@ -5,6 +5,8 @@ class Dyninst(Package):
     url      = "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1.2/DyninstAPI-8.1.2.tgz"
     md5      = "bf03b33375afa66fe0efa46ce3f4b17a"
 
+    versions = '7.0, 7.0.1, 8.0, 8.1.1, 8.1.2'
+
     depends_on("libelf")
     depends_on("libdwarf")
 
diff --git a/lib/spack/spack/test/mock_packages/libdwarf.py b/lib/spack/spack/test/mock_packages/libdwarf.py
index bae701b38b..a096eef0b8 100644
--- a/lib/spack/spack/test/mock_packages/libdwarf.py
+++ b/lib/spack/spack/test/mock_packages/libdwarf.py
@@ -10,6 +10,7 @@ class Libdwarf(Package):
     md5      = "64b42692e947d5180e162e46c689dfbf"
 
     list_url = "http://reality.sgiweb.org/davea/dwarf.html"
+    versions = '20111030, 20120410, 20130207'
 
     depends_on("libelf")
 
diff --git a/lib/spack/spack/test/mock_packages/libelf.py b/lib/spack/spack/test/mock_packages/libelf.py
index 7e3046b174..efc3ebc98b 100644
--- a/lib/spack/spack/test/mock_packages/libelf.py
+++ b/lib/spack/spack/test/mock_packages/libelf.py
@@ -5,6 +5,8 @@ class Libelf(Package):
     url      = "http://www.mr511.de/software/libelf-0.8.13.tar.gz"
     md5      = "4136d7b4c04df68b686570afa26988ac"
 
+    versions = '0.8.10, 0.8.12, 0.8.13'
+
     def install(self, prefix):
         configure("--prefix=%s" % prefix,
                   "--enable-shared",
diff --git a/lib/spack/spack/test/mock_packages/mpich.py b/lib/spack/spack/test/mock_packages/mpich.py
index d8cd67d528..ab235b1e43 100644
--- a/lib/spack/spack/test/mock_packages/mpich.py
+++ b/lib/spack/spack/test/mock_packages/mpich.py
@@ -5,6 +5,8 @@ class Mpich(Package):
     url      = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz"
     md5      = "9c5d5d4fe1e17dd12153f40bc5b6dbc0"
 
+    versions = '1.0.3, 1.3.2p1, 1.4.1p1, 3.0.4, 3.1b1'
+
     def install(self, prefix):
         configure("--prefix=%s" % prefix)
         make()
diff --git a/lib/spack/spack/test/mock_packages/mpileaks.py b/lib/spack/spack/test/mock_packages/mpileaks.py
index 224557cc52..d006ff61ed 100644
--- a/lib/spack/spack/test/mock_packages/mpileaks.py
+++ b/lib/spack/spack/test/mock_packages/mpileaks.py
@@ -5,6 +5,8 @@ class Mpileaks(Package):
     url      = "http://www.llnl.gov/mpileaks-1.0.tar.gz"
     md5      = "foobarbaz"
 
+    versions = [1.0, 2.1, 2.2, 2.3]
+
     depends_on("mpich")
     depends_on("callpath")
 
diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py
new file mode 100644
index 0000000000..d58e1d1124
--- /dev/null
+++ b/lib/spack/spack/test/mock_packages_test.py
@@ -0,0 +1,55 @@
+import unittest
+
+import spack
+import spack.packages as packages
+from spack.spec import Spec
+from spack.util.lang import new_path, list_modules
+
+mock_packages_path = new_path(spack.module_path, 'test', 'mock_packages')
+original_deps = None
+
+
+def set_pkg_dep(pkg, spec):
+    """Alters dependence information for a pacakge.
+       Use this to mock up constraints.
+    """
+    spec = Spec(spec)
+    packages.get(pkg).dependencies[spec.name] = spec
+
+
+def restore_dependencies():
+    # each time through restore original dependencies & constraints
+    global original_deps
+    for pkg_name, deps in original_deps.iteritems():
+        packages.get(pkg_name).dependencies.clear()
+        for dep in deps:
+            set_pkg_dep(pkg_name, dep)
+
+
+class MockPackagesTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        # Use a different packages directory for these tests.  We want to use
+        # mocked up packages that don't interfere with the real ones.
+        cls.real_packages_path = spack.packages_path
+        spack.packages_path = mock_packages_path
+
+        # First time through, record original relationships bt/w packages
+        global original_deps
+        original_deps = {}
+        for name in list_modules(mock_packages_path):
+            pkg = packages.get(name)
+            original_deps[name] = [
+                spec for spec in pkg.dependencies.values()]
+
+
+    @classmethod
+    def tearDownClass(cls):
+        """Restore the real packages path after any test."""
+        restore_dependencies()
+        spack.packages_path = cls.real_packages_path
+
+
+    def setUp(self):
+        """Before each test, restore deps between packages to original state."""
+        restore_dependencies()
diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py
index bdb64fee80..452884e539 100644
--- a/lib/spack/spack/test/spec_dag.py
+++ b/lib/spack/spack/test/spec_dag.py
@@ -6,61 +6,18 @@ packages directories that these tests use in:
 
 Each test validates conditions with the packages in those directories.
 """
-import unittest
-
 import spack
 import spack.package
 import spack.packages as packages
 
 from spack.util.lang import new_path, list_modules
 from spack.spec import Spec
+from spack.test.mock_packages_test import *
 
 mock_packages_path = new_path(spack.module_path, 'test', 'mock_packages')
 
 
-def set_pkg_dep(pkg, spec):
-    """Alters dependence information for a pacakge.
-       Use this to mock up constraints.
-    """
-    spec = Spec(spec)
-    packages.get(pkg).dependencies[spec.name] = spec
-
-
-class ValidationTest(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        # Use a different packages directory for these tests.  We want to use
-        # mocked up packages that don't interfere with the real ones.
-        cls.real_packages_path = spack.packages_path
-        spack.packages_path = mock_packages_path
-
-        # First time through, record original relationships bt/w packages
-        cls.original_deps = {}
-        for name in list_modules(mock_packages_path):
-            pkg = packages.get(name)
-            cls.original_deps[name] = [
-                spec for spec in pkg.dependencies.values()]
-
-
-    @classmethod
-    def restore(cls):
-        # each time through restore original dependencies & constraints
-        for pkg_name, deps in cls.original_deps.iteritems():
-            packages.get(pkg_name).dependencies.clear()
-            for dep in deps:
-                set_pkg_dep(pkg_name, dep)
-
-    @classmethod
-    def tearDownClass(cls):
-        """Restore the real packages path after any test."""
-        cls.restore()
-        spack.packages_path = cls.real_packages_path
-
-
-    def setUp(self):
-        """Before each test, restore deps between packages to original state."""
-        ValidationTest.restore()
-
+class ValidationTest(MockPackagesTest):
 
     def test_conflicting_package_constraints(self):
         set_pkg_dep('mpileaks', 'mpich@1.0')
@@ -87,6 +44,25 @@ class ValidationTest(unittest.TestCase):
         self.assertRaises(spack.spec.InconsistentSpecError, mpileaks.flatten)
 
 
+    def test_normalize_twice(self):
+        """Make sure normalize can be run twice on the same spec,
+           and that it is idempotent."""
+        spec = Spec('mpileaks')
+        spec.normalize()
+        n1 = spec.copy()
+
+        spec.normalize()
+        self.assertEqual(n1, spec)
+
+
+    def test_normalize_a_lot(self):
+        spec = Spec('mpileaks')
+        spec.normalize()
+        spec.normalize()
+        spec.normalize()
+        spec.normalize()
+
+
     def test_unsatisfiable_version(self):
         set_pkg_dep('mpileaks', 'mpich@1.0')
         spec = Spec('mpileaks ^mpich@2.0 ^callpath ^dyninst ^libelf ^libdwarf')
diff --git a/lib/spack/spack/tty.py b/lib/spack/spack/tty.py
index d6f64f7203..4cad0914e6 100644
--- a/lib/spack/spack/tty.py
+++ b/lib/spack/spack/tty.py
@@ -24,15 +24,15 @@ def verbose(message, *args):
 
 def debug(*args):
     if spack.debug:
-        info(message, *args, format='*c')
+        info("Debug: " + message, *args, format='*c')
 
 
 def error(message, *args):
-    info(message, *args, format='*r')
+    info("Error: " + message, *args, format='*r')
 
 
 def warn(message, *args):
-    info(message, *args, format='*Y')
+    info("Warning: " + message, *args, format='*Y')
 
 
 def die(message, *args):
diff --git a/lib/spack/spack/util/lang.py b/lib/spack/spack/util/lang.py
index 90546c3f2f..d983141045 100644
--- a/lib/spack/spack/util/lang.py
+++ b/lib/spack/spack/util/lang.py
@@ -97,5 +97,5 @@ class HashableMap(dict):
 
         # Copy everything from this dict into it.
         for key in self:
-            clone[key] = self[key]
+            clone[key] = self[key].copy()
         return clone
diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py
index 15b4027606..0be19e96ed 100644
--- a/lib/spack/spack/version.py
+++ b/lib/spack/spack/version.py
@@ -347,9 +347,16 @@ class VersionList(object):
     def __init__(self, vlist=None):
         self.versions = []
         if vlist != None:
-            vlist = list(vlist)
-            for v in vlist:
-                self.add(ver(v))
+            if type(vlist) == str:
+                vlist = _string_to_version(vlist)
+                if type(vlist) == VersionList:
+                    self.versions = vlist.versions
+                else:
+                    self.versions = [vlist]
+            else:
+                vlist = list(vlist)
+                for v in vlist:
+                    self.add(ver(v))
 
 
     def add(self, version):
@@ -540,6 +547,8 @@ def ver(obj):
         return VersionList(obj)
     elif t == str:
         return _string_to_version(obj)
+    elif t in (int, float):
+        return _string_to_version(str(obj))
     elif t in (Version, VersionRange, VersionList):
         return obj
     else: