Save progress. import gov.llnl.spack.mpich works.

15 files changed, 433 insertions, 229 deletions

diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py
index 66c52c3968..acf64c1e13 100644
--- a/lib/spack/llnl/util/tty/colify.py
+++ b/lib/spack/llnl/util/tty/colify.py
@@ -220,6 +220,13 @@ def colify(elts, **options):
 
 
 def colify_table(table, **options):
+    """Version of colify() for data expressed in rows, (list of lists).
+
+       Same as regular colify but takes a list of lists, where each
+       sub-list must be the same length, and each is interpreted as a
+       row in a table.  Regular colify displays a sequential list of
+       values in columns.
+    """
     if table is None:
         raise TypeError("Can't call colify_table on NoneType")
     elif not table or not table[0]:
diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index 09bc9ca52a..71e3ac3715 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -23,8 +23,10 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import os
+import sys
 import tempfile
 from llnl.util.filesystem import *
+import llnl.util.tty as tty
 
 # This lives in $prefix/lib/spack/spack/__file__
 prefix = ancestor(__file__, 4)
@@ -42,6 +44,7 @@ test_path      = join_path(module_path, "test")
 hooks_path     = join_path(module_path, "hooks")
 var_path       = join_path(prefix, "var", "spack")
 stage_path     = join_path(var_path, "stage")
+packages_path  = join_path(var_path, "packages")
 opt_path       = join_path(prefix, "opt")
 install_path   = join_path(opt_path, "spack")
 share_path     = join_path(prefix, "share", "spack")
@@ -55,9 +58,12 @@ repos = RepoNamespace()
 #
 # Set up the default packages database.
 #
-from spack.packages import PackageDB
-packages_path = join_path(var_path, "packages")
-db = PackageDB()
+import spack.packages
+_repo_paths = spack.config.get_repos_config()
+if not _repo_paths:
+    tty.die("Spack configuration contains no package repositories.")
+db = spack.packages.PackageFinder(*_repo_paths)
+sys.meta_path.append(db)
 
 #
 # Paths to mock files for testing.
diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py
index 1261c7ada9..e290f60b7b 100644
--- a/lib/spack/spack/cmd/repo.py
+++ b/lib/spack/spack/cmd/repo.py
@@ -32,7 +32,7 @@ from llnl.util.filesystem import join_path, mkdirp
 import spack.spec
 import spack.config
 from spack.util.environment import get_path
-from spack.packages import repo_config
+from spack.packages import repo_config_filename
 
 import os
 import exceptions
@@ -50,13 +50,8 @@ def setup_parser(subparser):
     create_parser = sp.add_parser('create', help=repo_create.__doc__)
     create_parser.add_argument('directory', help="Directory containing the packages.")
     create_parser.add_argument('name', help="Name of new package repository.")
-<<<<<<< HEAD:lib/spack/spack/cmd/packagerepo.py
-    
-    remove_parser = sp.add_parser('remove', help=packagerepo_remove.__doc__)
-=======
 
     remove_parser = sp.add_parser('remove', help=repo_remove.__doc__)
->>>>>>> Save changes to external repo integration:lib/spack/spack/cmd/repo.py
     remove_parser.add_argument('name')
 
     list_parser = sp.add_parser('list', help=repo_list.__doc__)
@@ -81,7 +76,7 @@ def repo_add(args):
     """Add package sources to the Spack configuration."""
     if not add_to_config(args.directory):
         tty.die('Repo directory %s already exists in the repo list' % dir)
-        
+
 
 def repo_create(args):
     """Create a new package repo at a directory and name"""
@@ -95,13 +90,13 @@ def repo_create(args):
             mkdirp(dir)
         except exceptions.OSError, e:
             tty.die('Failed to create new directory %s' % dir)
-    path = os.path.join(dir, repo_config)
+    path = os.path.join(dir, repo_config_filename)
     try:
         with closing(open(path, 'w')) as repofile:
             repofile.write(name + '\n')
     except exceptions.IOError, e:
         tty.die('Could not create new file %s' % path)
-            
+
     if not add_to_config(args.directory):
         tty.warn('Repo directory %s already exists in the repo list' % dir)
 
@@ -118,8 +113,8 @@ def repo_list(args):
     fmt = "%%-%ds%%s" % (max_len + 4)
     for root in root_names:
         print fmt % (root[0], root[1])
-        
-    
+
+
 
 def repo(parser, args):
     action = { 'add'    : repo_add,
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index dc59f9a5a3..66da91f629 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -269,7 +269,9 @@ def get_repos_config():
     config = get_config()
     if 'repos' not in config:
         return []
-    return config['repos']
+
+    repo_list = config['repos']
+    return [substitute_spack_prefix(repo) for repo in repo_list]
 
 
 def get_mirror_config():
diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py
index c414234386..df54b12324 100644
--- a/lib/spack/spack/packages.py
+++ b/lib/spack/spack/packages.py
@@ -28,7 +28,6 @@ import sys
 import inspect
 import glob
 import imp
-import spack.config
 import re
 import itertools
 import traceback
@@ -41,149 +40,327 @@ from llnl.util.lang import *
 import spack.error
 import spack.spec
 from spack.virtual import ProviderIndex
-from spack.util.naming import mod_to_class, validate_module_name
-from sets import Set
-from spack.repo_loader import RepoLoader, imported_packages_module, package_file_name
+from spack.util.naming import *
 
 # Filename for package repo names
-repo_config = 'repo.yaml'
+repo_config_filename = '_repo.yaml'
+
+# Filename for packages in repos.
+package_file_name = 'package.py'
 
 def _autospec(function):
     """Decorator that automatically converts the argument of a single-arg
        function to a Spec."""
-    def converter(self, spec_like, **kwargs):
+    def converter(self, spec_like, *args, **kwargs):
         if not isinstance(spec_like, spack.spec.Spec):
             spec_like = spack.spec.Spec(spec_like)
-        return function(self, spec_like, **kwargs)
+        return function(self, spec_like, *args, **kwargs)
     return converter
 
 
-def sliding_window(seq, n):
-    it = iter(seq)
-    result = tuple(itertools.islice(it, n))
-    if len(result) == n:
-        yield result
-    for elem in it:
-        result = result[1:] + (elem,)
-        yield result
+class NamespaceTrie(object):
+    def __init__(self):
+        self._elements = {}
 
 
-class PackageDB(object):
+    def __setitem__(self, namespace, repo):
+        parts = namespace.split('.')
+        cur = self._elements
+        for p in parts[:-1]:
+            if p not in cur:
+                cur[p] = {}
+            cur = cur[p]
+
+        cur[parts[-1]] = repo
+
+
+    def __getitem__(self, namespace):
+        parts = namespace.split('.')
+        cur = self._elements
+        for p in parts:
+            if p not in cur:
+                raise KeyError("Can't find namespace %s in trie" % namespace)
+            cur = cur[p]
+        return cur
+
+
+    def __contains__(self, namespace):
+        parts = namespace.split('.')
+        cur = self._elements
+        for p in parts:
+            if not isinstance(cur, dict):
+                return False
+            if p not in cur:
+                return False
+            cur  = cur[p]
+        return True
+
+
+
+class PackageFinder(object):
+    """A PackageFinder is a wrapper around a list of PackageDBs.
+
+       It functions exactly like a PackageDB, but it operates on the
+       combined results of the PackageDBs in its list instead of on a
+       single package repository.
+    """
     def __init__(self, *repo_dirs):
-        """Construct a new package database from a list of directories.
+        self.repos = []
+        self.by_namespace = NamespaceTrie()
+        self.by_path = {}
+
+        for root in repo_dirs:
+            repo = PackageDB(root)
+            self.put_last(repo)
+
+
+    def _check_repo(self, repo):
+        if repo.root in self.by_path:
+            raise DuplicateRepoError("Package repos are the same",
+                                     repo, self.by_path[repo.root])
+
+        if repo.namespace in self.by_namespace:
+            tty.error("Package repos cannot have the same name",
+                      repo, self.by_namespace[repo.namespace])
+
+
+    def _add(self, repo):
+        self._check_repo(repo)
+        self.by_namespace[repo.namespace] = repo
+        self.by_path[repo.root] = repo
+
+
+    def put_first(self, repo):
+        self._add(repo)
+        self.repos.insert(0, repo)
+
+
+    def put_last(self, repo):
+        self._add(repo)
+        self.repos.append(repo)
+
 
-        Args:
-          repo_dirs   List of directories containing packages.
+    def remove(self, repo):
+        if repo in self.repos:
+            self.repos.remove(repo)
 
-        If ``repo_dirs`` is empty, gets repository list from Spack configuration.
+
+    def swap(self, other):
+        repos = self.repos
+        by_namespace = self.by_namespace
+        by_path = self.by_path
+
+        self.repos = other.repos
+        self.by_namespace = other.by_namespace
+        self.by_pah = other.by_path
+
+        other.repos = repos
+        other.by_namespace = by_namespace
+        other.by_path = by_path
+
+
+    def all_package_names(self):
+        all_pkgs = set()
+        for repo in self.repos:
+            all_pkgs.update(set(repo.all_package_names()))
+        return all_pkgs
+
+
+    def all_packages(self):
+        for name in self.all_package_names():
+            yield self.get(name)
+
+
+    def providers_for(self, vpkg_name):
+        # TODO: USE MORE THAN FIRST REPO
+        return self.repos[0].providers_for(vpkg_name)
+
+
+    def _get_spack_pkg_name(self, repo, py_module_name):
+        """Allow users to import Spack packages using legal Python identifiers.
+
+        A python identifier might map to many different Spack package
+        names due to hyphen/underscore ambiguity.
+
+        Easy example:
+            num3proxy   -> 3proxy
+
+        Ambiguous:
+            foo_bar -> foo_bar, foo-bar
+
+        More ambiguous:
+            foo_bar_baz -> foo_bar_baz, foo-bar-baz, foo_bar-baz, foo-bar_baz
         """
-        if not repo_dirs:
-            repo_dirs = spack.config.get_repos_config()
-            if not repo_dirs:
-                tty.die("Spack configuration contains no package repositories.")
+        if py_module_name in repo:
+            return py_module_name
 
-        # Collect the repos from the config file and read their names
-        # from the file system
-        repo_dirs = [spack.config.substitute_spack_prefix(rd) for rd in repo_dirs]
+        options = possible_spack_module_names(py_module_name)
+        options.remove(py_module_name)
+        for name in options:
+            if name in repo:
+                return name
 
-        self.repos = []
-        for rdir in repo_dirs:
-            rname = self._read_reponame_from_directory(rdir)
-            if rname:
-                self.repos.append((self._read_reponame_from_directory(rdir), rdir))
+        return None
 
 
-        by_path = sorted(self.repos, key=lambda r:r[1])
-        by_name = sorted(self.repos, key=lambda r:r[0])
+    def find_module(self, fullname, path=None):
+        if fullname in self.by_namespace:
+            return self
 
-        for r1, r2 in by_path:
-            if r1[1] == r2[1]:
-                tty.die("Package repos are the same:",
-                        "  %20s  %s" % r1, "  %20s %s" % r2)
+        namespace, dot, module_name = fullname.rpartition('.')
+        if namespace not in self.by_namespace:
+            return None
 
-        for r1, r2 in by_name:
-            if r1[0] == r2[0]:
-                tty.die("Package repos cannot have the same name:",
-                        "  %20s  %s" % r1, "  %20s %s" % r2)
+        repo = self.by_namespace[namespace]
+        name = self._get_spack_pkg_name(repo, module_name)
+        if not name:
+            return None
 
-        # For each repo, create a RepoLoader
-        self.repo_loaders = dict((name, RepoLoader(name, path))
-                                 for name, path in self.repos)
+        return self
 
-        self.instances = {}
-        self.provider_index = None
 
+    def load_module(self, fullname):
+        if fullname in sys.modules:
+            return sys.modules[fullname]
 
-    def _read_reponame_from_directory(self, dir):
-        """For a packagerepo directory, read the repo name from the
-        $root/repo.yaml file"""
-        path = os.path.join(dir, repo_config)
+        if fullname in self.by_namespace:
+            ns = self.by_namespace[fullname]
+            module = imp.new_module(fullname)
+            module.__file__ = "<spack-namespace>"
+            module.__path__ = []
+            module.__package__ = fullname
 
+        else:
+            namespace, dot, module_name = fullname.rpartition('.')
+            if namespace not in self.by_namespace:
+                raise ImportError(
+                    "No Spack repository with namespace %s" % namespace)
+
+            repo = self.by_namespace[namespace]
+            name = self._get_spack_pkg_name(repo, module_name)
+            if not name:
+                raise ImportError(
+                    "No module %s in Spack repository %s" % (module_name, repo))
+
+            fullname = namespace + '.' + name
+            file_path = os.path.join(repo.root, name, package_file_name)
+            module = imp.load_source(fullname, file_path)
+            module.__package__ = namespace
+
+        module.__loader__ = self
+        sys.modules[fullname] = module
+        return module
+
+
+    @_autospec
+    def get(self, spec, new=False):
+        for repo in self.repos:
+            if spec.name in repo:
+                return repo.get(spec, new)
+        raise UnknownPackageError(spec.name)
+
+
+    def get_repo(self, namespace):
+        if namespace in self.by_namespace:
+            repo = self.by_namespace[namespace]
+            if isinstance(repo, PackageDB):
+                return repo
+        return None
+
+
+    def exists(self, pkg_name):
+        return any(repo.exists(pkg_name) for repo in self.repos)
+
+
+    def __contains__(self, pkg_name):
+        return self.exists(pkg_name)
+
+
+
+class PackageDB(object):
+    """Class representing a package repository in the filesystem.
+
+    Each package repository must have a top-level configuration file
+    called `_repo.yaml`.
+
+    Currently, `_repo.yaml` this must define:
+
+    `namespace`:
+        A Python namespace where the repository's packages should live.
+
+    """
+    def __init__(self, root):
+        """Instantiate a package repository from a filesystem path."""
+        # Root directory, containing _repo.yaml and package dirs
+        self.root = root
+
+        # Config file in <self.root>/_repo.yaml
+        self.config_file = os.path.join(self.root, repo_config_filename)
+
+        # Read configuration from _repo.yaml
+        config = self._read_config()
+        if not 'namespace' in config:
+            tty.die('Package repo in %s must define a namespace in %s.'
+                    % (self.root, repo_config_filename))
+
+        # Check namespace in the repository configuration.
+        self.namespace = config['namespace']
+        if not re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', self.namespace):
+            tty.die(("Invalid namespace '%s' in '%s'. Namespaces must be "
+                     "valid python identifiers separated by '.'")
+                    % (self.namespace, self.root))
+
+        # These are internal cache variables.
+        self._instances = {}
+        self._provider_index = None
+
+
+    def _read_config(self):
+        """Check for a YAML config file in this db's root directory."""
         try:
-            with open(path) as reponame_file:
+            with open(self.config_file) as reponame_file:
                 yaml_data = yaml.load(reponame_file)
 
-                if (not yaml_data or
-                    'repo' not in yaml_data or
-                    'namespace' not in yaml_data['repo']):
-                    tty.die("Invalid %s in %s" % (repo_config, dir))
-
-                name = yaml_data['repo']['namespace']
-                if not re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', name):
-                    tty.die(
-                        "Package repo name '%s', read from %s, is an invalid name. "
-                        "Repo names must began with a letter and only contain "
-                        "letters and numbers." % (name, path))
-                return name
+                if (not yaml_data or 'repo' not in yaml_data or
+                    not isinstance(yaml_data['repo'], dict)):
+                    tty.die("Invalid %s in repository %s"
+                            % (repo_config_filename, self.root))
+
+                return yaml_data['repo']
+
         except exceptions.IOError, e:
-            tty.die("Error reading %s when opening %s" % (repo_config, dir))
+            tty.die("Error reading %s when opening %s"
+                    % (self.config_file, self.root))
 
 
     @_autospec
-    def get(self, spec, **kwargs):
+    def get(self, spec, new=False):
         if spec.virtual:
             raise UnknownPackageError(spec.name)
 
-        if kwargs.get('new', False):
-            if spec in self.instances:
-                del self.instances[spec]
+        if new:
+            if spec in self._instances:
+                del self._instances[spec]
 
-        if not spec in self.instances:
+        if not spec in self._instances:
             package_class = self.get_class_for_package_name(spec.name, spec.repo)
             try:
                 copy = spec.copy()
-                self.instances[copy] = package_class(copy)
+                self._instances[copy] = package_class(copy)
             except Exception, e:
                 if spack.debug:
                     sys.excepthook(*sys.exc_info())
                 raise FailedConstructorError(spec.name, *sys.exc_info())
 
-        return self.instances[spec]
-
-
-    @_autospec
-    def delete(self, spec):
-        """Force a package to be recreated."""
-        del self.instances[spec]
-
-
-    def purge(self):
-        """Clear entire package instance cache."""
-        self.instances.clear()
-
-
-    @_autospec
-    def get_installed(self, spec):
-        """Get all the installed specs that satisfy the provided spec constraint."""
-        return [s for s in self.installed_package_specs() if s.satisfies(spec)]
+        return self._instances[spec]
 
 
     @_autospec
     def providers_for(self, vpkg_spec):
-        if self.provider_index is None:
-            self.provider_index = ProviderIndex(self.all_package_names())
+        if self._provider_index is None:
+            self._provider_index = ProviderIndex(self.all_package_names())
 
-        providers = self.provider_index.providers_for(vpkg_spec)
+        providers = self._provider_index.providers_for(vpkg_spec)
         if not providers:
             raise UnknownPackageError(vpkg_spec.name)
         return providers
@@ -194,46 +371,13 @@ class PackageDB(object):
         return [p for p in self.all_packages() if p.extends(extendee_spec)]
 
 
-    @_autospec
-    def installed_extensions_for(self, extendee_spec):
-        for s in self.installed_package_specs():
-            try:
-                if s.package.extends(extendee_spec):
-                    yield s.package
-            except UnknownPackageError, e:
-                # Skip packages we know nothing about
-                continue
-                # TODO: add some conditional way to do this instead of
-                # catching exceptions.
-
-
-    def repo_for_package_name(self, pkg_name, packagerepo_name=None):
-        """Find the dirname for a package and the packagerepo it came from
-           if packagerepo_name is not None, then search for the package in the
-           specified packagerepo"""
-        #Look for an existing package under any matching packagerepos
-        roots = [pkgrepo for pkgrepo in self.repos
-                 if not packagerepo_name or packagerepo_name == pkgrepo[0]]
-
-        if not roots:
-            tty.die("Package repo %s does not exist" % packagerepo_name)
-
-        for pkgrepo in roots:
-            path = join_path(pkgrepo[1], pkg_name)
-            if os.path.exists(path):
-                return (pkgrepo[0], path)
-
-        repo_to_add_to = roots[-1]
-        return (repo_to_add_to[0], join_path(repo_to_add_to[1], pkg_name))
-
-
-    def dirname_for_package_name(self, pkg_name, packagerepo_name=None):
+    def dirname_for_package_name(self, pkg_name):
         """Get the directory name for a particular package.  This is the
            directory that contains its package.py file."""
-        return self.repo_for_package_name(pkg_name, packagerepo_name)[1]
+        return join_path(self.root, pkg_name)
 
 
-    def filename_for_package_name(self, pkg_name, packagerepo_name=None):
+    def filename_for_package_name(self, pkg_name):
         """Get the filename for the module we should load for a particular
            package.  Packages for a pacakge DB live in
            ``$root/<package_name>/package.py``
@@ -241,57 +385,25 @@ class PackageDB(object):
            This will return a proper package.py path even if the
            package doesn't exist yet, so callers will need to ensure
            the package exists before importing.
-
-           If a packagerepo is specified, then return existing
-           or new paths in the specified packagerepo directory.  If no
-           package repo is supplied, return an existing path from any
-           package repo, and new paths in the default package repo.
         """
         validate_module_name(pkg_name)
-        pkg_dir = self.dirname_for_package_name(pkg_name, packagerepo_name)
+        pkg_dir = self.dirname_for_package_name(pkg_name)
         return join_path(pkg_dir, package_file_name)
 
 
-    def installed_package_specs(self):
-        """Read installed package names straight from the install directory
-           layout.
-        """
-        # Get specs from the directory layout but ensure that they're
-        # all normalized properly.
-        installed = []
-        for spec in spack.install_layout.all_specs():
-            spec.normalize()
-            installed.append(spec)
-        return installed
-
-
-    def installed_known_package_specs(self):
-        """Read installed package names straight from the install
-           directory layout, but return only specs for which the
-           package is known to this version of spack.
-        """
-        for spec in spack.install_layout.all_specs():
-            if self.exists(spec.name):
-                yield spec
-
-
     @memoized
     def all_package_names(self):
         """Generator function for all packages.  This looks for
            ``<pkg_name>/package.py`` files within the repo direcotories"""
-        all_packages = Set()
-        for repo in self.repos:
-            dir = repo[1]
-            if not os.path.isdir(dir):
-                continue
-            for pkg_name in os.listdir(dir):
-                pkg_dir  = join_path(dir, pkg_name)
-                pkg_file = join_path(pkg_dir, package_file_name)
-                if os.path.isfile(pkg_file):
-                    all_packages.add(pkg_name)
-        all_package_names = list(all_packages)
-        all_package_names.sort()
-        return all_package_names
+        all_package_names = []
+
+        for pkg_name in os.listdir(self.root):
+            pkg_dir  = join_path(self.root, pkg_name)
+            pkg_file = join_path(pkg_dir, package_file_name)
+            if os.path.isfile(pkg_file):
+                all_package_names.append(pkg_name)
+
+        return sorted(all_package_names)
 
 
     def all_packages(self):
@@ -301,19 +413,25 @@ class PackageDB(object):
 
     @memoized
     def exists(self, pkg_name):
-        """Whether a package with the supplied name exists ."""
+        """Whether a package with the supplied name exists."""
         return os.path.exists(self.filename_for_package_name(pkg_name))
 
 
     @memoized
     def get_class_for_package_name(self, pkg_name, reponame = None):
         """Get an instance of the class for a particular package."""
-        (reponame, repodir) = self.repo_for_package_name(pkg_name, reponame)
-        module_name = imported_packages_module + '.' + reponame + '.' + pkg_name
+        file_path = self.filename_for_package_name(pkg_name)
 
-        module = self.repo_loaders[reponame].get_module(pkg_name)
+        if os.path.exists(file_path):
+            if not os.path.isfile(file_path):
+                tty.die("Something's wrong. '%s' is not a file!" % file_path)
+            if not os.access(file_path, os.R_OK):
+                tty.die("Cannot read '%s'!" % file_path)
+        else:
+            raise UnknownPackageError(pkg_name, self.namespace)
 
         class_name = mod_to_class(pkg_name)
+        module = __import__(self.namespace + '.' + pkg_name, fromlist=[class_name])
         cls = getattr(module, class_name)
         if not inspect.isclass(cls):
             tty.die("%s.%s is not a class" % (pkg_name, class_name))
@@ -321,6 +439,63 @@ class PackageDB(object):
         return cls
 
 
+    def __str__(self):
+        return "<PackageDB '%s' from '%s'>" % (self.namespace, self.root)
+
+
+    def __repr__(self):
+        return self.__str__()
+
+
+    def __contains__(self, pkg_name):
+        return self.exists(pkg_name)
+
+
+    #
+    # Below functions deal with installed packages, and should be
+    # moved to some other part of Spack (conbine with
+    # directory_layout?)
+    #
+    @_autospec
+    def get_installed(self, spec):
+        """Get all the installed specs that satisfy the provided spec constraint."""
+        return [s for s in self.installed_package_specs() if s.satisfies(spec)]
+
+
+    @_autospec
+    def installed_extensions_for(self, extendee_spec):
+        for s in self.installed_package_specs():
+            try:
+                if s.package.extends(extendee_spec):
+                    yield s.package
+            except UnknownPackageError, e:
+                # Skip packages we know nothing about
+                continue
+
+
+    def installed_package_specs(self):
+        """Read installed package names straight from the install directory
+           layout.
+        """
+        # Get specs from the directory layout but ensure that they're
+        # all normalized properly.
+        installed = []
+        for spec in spack.install_layout.all_specs():
+            spec.normalize()
+            installed.append(spec)
+        return installed
+
+
+    def installed_known_package_specs(self):
+        """Read installed package names straight from the install
+           directory layout, but return only specs for which the
+           package is known to this version of spack.
+        """
+        for spec in spack.install_layout.all_specs():
+            if self.exists(spec.name):
+                yield spec
+
+
 class UnknownPackageError(spack.error.SpackError):
     """Raised when we encounter a package spack doesn't have."""
     def __init__(self, name, repo=None):
@@ -333,6 +508,13 @@ class UnknownPackageError(spack.error.SpackError):
         self.name = name
 
 
+class DuplicateRepoError(spack.error.SpackError):
+    """Raised when duplicate repos are added to a PackageFinder."""
+    def __init__(self, msg, repo1, repo2):
+        super(UnknownPackageError, self).__init__(
+            "%s: %s, %s" % (msg, repo1, repo2))
+
+
 class FailedConstructorError(spack.error.SpackError):
     """Raised when a package's class constructor fails."""
     def __init__(self, name, exc_type, exc_obj, exc_tb):
diff --git a/lib/spack/spack/repo_loader.py b/lib/spack/spack/repo_loader.py
index 92da1cf709..441011cf98 100644
--- a/lib/spack/spack/repo_loader.py
+++ b/lib/spack/spack/repo_loader.py
@@ -12,7 +12,6 @@ imported_packages_module = 'spack.repos'
 # Name of the package file inside a package directory
 package_file_name = 'package.py'
 
-import sys
 class LazyLoader:
     """The LazyLoader handles cases when repo modules or classes
        are imported.  It watches for 'spack.repos.*' loads, then
@@ -21,15 +20,6 @@ class LazyLoader:
         if not fullname.startswith(imported_packages_module):
             return None
 
-        print "HERE ==="
-        print
-        for line in traceback.format_stack():
-            print "    ", line.strip()
-        print
-        print "full: ", fullname
-        print "path: ", pathname
-        print
-
         partial_name = fullname[len(imported_packages_module)+1:]
 
         print "partial: ", partial_name
@@ -50,7 +40,7 @@ class LazyLoader:
     def load_module(self, fullname):
         return self.mod
 
-sys.meta_path.append(LazyLoader())
+#sys.meta_path.append(LazyLoader())
 
 _reponames = {}
 class RepoNamespace(types.ModuleType):
@@ -59,7 +49,6 @@ class RepoNamespace(types.ModuleType):
        this class will use __getattr__ to translate the 'original'
        into one of spack's known repositories"""
     def __init__(self):
-        import sys
         sys.modules[imported_packages_module] = self
 
     def __getattr__(self, name):
@@ -89,7 +78,6 @@ class RepoLoader(types.ModuleType):
         if not reponame in _reponames:
             _reponames[reponame] = self
 
-        import sys
         sys.modules[self.module_name] = self
 
 
@@ -110,14 +98,6 @@ class RepoLoader(types.ModuleType):
         import imp
         import llnl.util.tty as tty
 
-        file_path = os.path.join(self.path, pkg_name, package_file_name)
-        if os.path.exists(file_path):
-            if not os.path.isfile(file_path):
-                tty.die("Something's wrong. '%s' is not a file!" % file_path)
-            if not os.access(file_path, os.R_OK):
-                tty.die("Cannot read '%s'!" % file_path)
-        else:
-            raise spack.packages.UnknownPackageError(pkg_name, self.reponame if self.reponame != 'original' else None)
 
         try:
             module_name = imported_packages_module + '.' + self.reponame + '.' + pkg_name
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 972ba9ccbb..1666457502 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -1714,7 +1714,7 @@ class SpecParser(spack.parse.Parser):
             spec_repo = lst[-2]
         else:
             spec_name = self.token.value
-            (spec_repo, repodir) = spack.db.repo_for_package_name(spec_name)
+            spec_repo = 'gov.llnl.spack'
 
         self.check_identifier(spec_name)
 
diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py
index b3ad8efec4..55b3f0b18f 100644
--- a/lib/spack/spack/test/directory_layout.py
+++ b/lib/spack/spack/test/directory_layout.py
@@ -34,7 +34,7 @@ from llnl.util.filesystem import *
 
 import spack
 from spack.spec import Spec
-from spack.packages import PackageDB
+from spack.packages import PackageFinder
 from spack.directory_layout import YamlDirectoryLayout
 
 # number of packages to test (to reduce test time)
@@ -123,7 +123,7 @@ class DirectoryLayoutTest(unittest.TestCase):
            information about installed packages' specs to uninstall
            or query them again if the package goes away.
         """
-        mock_db = PackageDB(spack.mock_packages_path)
+        mock_db = PackageFinder(spack.mock_packages_path)
 
         not_in_mock = set.difference(
             set(spack.db.all_package_names()),
@@ -145,8 +145,7 @@ class DirectoryLayoutTest(unittest.TestCase):
             self.layout.create_install_directory(spec)
             installed_specs[spec] = self.layout.path_for_spec(spec)
 
-        tmp = spack.db
-        spack.db = mock_db
+        spack.db.swap(mock_db)
 
         # Now check that even without the package files, we know
         # enough to read a spec from the spec file.
@@ -161,7 +160,7 @@ class DirectoryLayoutTest(unittest.TestCase):
             self.assertTrue(spec.eq_dag(spec_from_file))
             self.assertEqual(spec.dag_hash(), spec_from_file.dag_hash())
 
-        spack.db = tmp
+        spack.db.swap(mock_db)
 
 
     def test_find(self):
diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py
index 00f81114af..1f46d65790 100644
--- a/lib/spack/spack/test/mock_packages_test.py
+++ b/lib/spack/spack/test/mock_packages_test.py
@@ -22,11 +22,12 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+import sys
 import unittest
 
 import spack
 import spack.config
-from spack.packages import PackageDB
+from spack.packages import PackageFinder
 from spack.spec import Spec
 
 
@@ -43,8 +44,8 @@ class MockPackagesTest(unittest.TestCase):
         # Use the mock packages database for these tests.  This allows
         # us to set up contrived packages that don't interfere with
         # real ones.
-        self.real_db = spack.db
-        spack.db = PackageDB(spack.mock_packages_path)
+        self.db = PackageFinder(spack.mock_packages_path)
+        spack.db.swap(self.db)
 
         spack.config.clear_config_caches()
         self.real_scopes = spack.config.config_scopes
@@ -55,7 +56,8 @@ class MockPackagesTest(unittest.TestCase):
 
     def cleanmock(self):
         """Restore the real packages path after any test."""
-        spack.db = self.real_db
+        spack.db.swap(self.db)
+
         spack.config.config_scopes = self.real_scopes
         spack.config.clear_config_caches()
 
@@ -66,5 +68,3 @@ class MockPackagesTest(unittest.TestCase):
 
     def tearDown(self):
         self.cleanmock()
-
-
diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py
index 6222e7b5f8..70b5d6a478 100644
--- a/lib/spack/spack/test/package_sanity.py
+++ b/lib/spack/spack/test/package_sanity.py
@@ -47,10 +47,10 @@ class PackageSanityTest(unittest.TestCase):
 
     def test_get_all_mock_packages(self):
         """Get the mock packages once each too."""
-        tmp = spack.db
-        spack.db = PackageDB(spack.mock_packages_path)
+        db = PackageFinder(spack.mock_packages_path)
+        spack.db.swap(db)
         self.check_db()
-        spack.db = tmp
+        spack.db.swap(db)
 
 
     def test_url_versions(self):
diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py
index a8183cf6a6..42bd91ec5c 100644
--- a/lib/spack/spack/test/packages.py
+++ b/lib/spack/spack/test/packages.py
@@ -44,7 +44,8 @@ class PackagesTest(MockPackagesTest):
 
 
     def test_package_filename(self):
-        filename = spack.db.filename_for_package_name('mpich')
+        repo = spack.db.get_repo('gov.llnl.spack.mock')
+        filename = repo.filename_for_package_name('mpich')
         self.assertEqual(filename, join_path(spack.mock_packages_path, 'mpich', 'package.py'))
 
 
@@ -54,7 +55,8 @@ class PackagesTest(MockPackagesTest):
 
 
     def test_nonexisting_package_filename(self):
-        filename = spack.db.filename_for_package_name('some-nonexisting-package')
+        repo = spack.db.get_repo('gov.llnl.spack.mock')
+        filename = repo.filename_for_package_name('some-nonexisting-package')
         self.assertEqual(filename, join_path(spack.mock_packages_path, 'some-nonexisting-package', 'package.py'))
 
 
diff --git a/lib/spack/spack/util/naming.py b/lib/spack/spack/util/naming.py
index 782afbd4bb..a7b6e2b436 100644
--- a/lib/spack/spack/util/naming.py
+++ b/lib/spack/spack/util/naming.py
@@ -1,10 +1,14 @@
 # Need this because of spack.util.string
 from __future__ import absolute_import
 import string
+import itertools
 import re
 
 import spack
 
+__all__ = ['mod_to_class', 'spack_module_to_python_module', 'valid_module_name',
+           'validate_module_name', 'possible_spack_module_names']
+
 # Valid module names can contain '-' but can't start with it.
 _valid_module_re = r'^\w[\w-]*$'
 
@@ -42,6 +46,33 @@ def mod_to_class(mod_name):
     return class_name
 
 
+def spack_module_to_python_module(mod_name):
+    """Given a Spack module name, returns the name by which it can be
+       imported in Python.
+    """
+    if re.match(r'[0-9]', mod_name):
+        mod_name = 'num' + mod_name
+
+    return mod_name.replace('-', '_')
+
+
+def possible_spack_module_names(python_mod_name):
+    """Given a Python module name, return a list of all possible spack module
+       names that could correspond to it."""
+    mod_name = re.sub(r'^num(\d)', r'\1', python_mod_name)
+
+    parts = re.split(r'(_)', mod_name)
+    options = [['_', '-']] * mod_name.count('_')
+
+    results = []
+    for subs in itertools.product(*options):
+        s = list(parts)
+        s[1::2] = subs
+        results.append(''.join(s))
+
+    return results
+
+
 def valid_module_name(mod_name):
     """Return whether the mod_name is valid for use in Spack."""
     return bool(re.match(_valid_module_re, mod_name))
diff --git a/var/spack/mock_packages/_repo.yaml b/var/spack/mock_packages/_repo.yaml
new file mode 100644
index 0000000000..b97b978de3
--- /dev/null
+++ b/var/spack/mock_packages/_repo.yaml
@@ -0,0 +1,2 @@
+repo:
+  namespace: gov.llnl.spack.mock
diff --git a/var/spack/mock_packages/repo.yaml b/var/spack/mock_packages/repo.yaml
deleted file mode 100644
index d065896006..0000000000
--- a/var/spack/mock_packages/repo.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
-repo:
-  namespace: mock
diff --git a/var/spack/packages/repo.yaml b/var/spack/packages/_repo.yaml
index 4a371e1cad..4a371e1cad 100644
--- a/var/spack/packages/repo.yaml
+++ b/var/spack/packages/_repo.yaml