Eliminated all calls that relied on finding all packages in the opt directory

Replaced them all with references to the database

Implemented caching in the database. The database now only re-reads data
if the database file exists and was changed since this file last wrote to it.

Added the installed_db field to the spack instance

Left the call to all_specs from testdirectory_layout.py for now.

11 files changed, 121 insertions, 95 deletions

diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index caa09eb6e0..4d10bc2da6 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -54,6 +54,12 @@ packages_path = join_path(var_path, "packages")
 db = PackageDB(packages_path)
 
 #
+# Set up the installed packages database
+#
+from spack.database import Database
+installed_db = Database(install_path)
+
+#
 # Paths to mock files for testing.
 #
 mock_packages_path = join_path(var_path, "mock_packages")
diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py
index b96ac5af51..fd3ef3ed27 100644
--- a/lib/spack/spack/cmd/__init__.py
+++ b/lib/spack/spack/cmd/__init__.py
@@ -124,7 +124,7 @@ def elide_list(line_list, max_num=10):
 
 
 def disambiguate_spec(spec):
-    matching_specs = spack.db.get_installed(spec)
+    matching_specs = spack.installed_db.get_installed(spec)
     if not matching_specs:
         tty.die("Spec '%s' matches no installed packages." % spec)
 
diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py
index e44be41029..1f0e303cdf 100644
--- a/lib/spack/spack/cmd/deactivate.py
+++ b/lib/spack/spack/cmd/deactivate.py
@@ -54,7 +54,7 @@ def deactivate(parser, args):
     if args.all:
         if pkg.extendable:
             tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec)
-            ext_pkgs = spack.db.installed_extensions_for(spec)
+            ext_pkgs = spack.installed_db.installed_extensions_for(spec)
 
             for ext_pkg in ext_pkgs:
                 ext_pkg.spec.normalize()
diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py
index fc8e6842c3..e919b1c4fb 100644
--- a/lib/spack/spack/cmd/extensions.py
+++ b/lib/spack/spack/cmd/extensions.py
@@ -80,7 +80,7 @@ def extensions(parser, args):
     colify(ext.name for ext in extensions)
 
     # List specs of installed extensions.
-    installed  = [s.spec for s in spack.db.installed_extensions_for(spec)]
+    installed  = [s.spec for s in spack.installed_db.installed_extensions_for(spec)]
     print
     if not installed:
         tty.msg("None installed.")
diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py
index 3c993990b1..6f9072e311 100644
--- a/lib/spack/spack/cmd/find.py
+++ b/lib/spack/spack/cmd/find.py
@@ -138,9 +138,9 @@ def find(parser, args):
 
     # Get all the specs the user asked for
     if not query_specs:
-        specs = set(spack.db.installed_package_specs())
+        specs = set(spack.installed_db.installed_package_specs())
     else:
-        results = [set(spack.db.get_installed(qs)) for qs in query_specs]
+        results = [set(spack.installed_db.get_installed(qs)) for qs in query_specs]
         specs = set.union(*results)
 
     if not args.mode:
diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py
index 34f0855a50..215d877bd0 100644
--- a/lib/spack/spack/cmd/module.py
+++ b/lib/spack/spack/cmd/module.py
@@ -65,7 +65,7 @@ def module_find(mtype, spec_array):
         tty.die("You can only pass one spec.")
     spec = specs[0]
 
-    specs = [s for s in spack.db.installed_package_specs() if s.satisfies(spec)]
+    specs = [s for s in spack.installed_db.installed_package_specs() if s.satisfies(spec)]
     if len(specs) == 0:
         tty.die("No installed packages match spec %s" % spec)
 
@@ -86,7 +86,7 @@ def module_find(mtype, spec_array):
 def module_refresh():
     """Regenerate all module files for installed packages known to
        spack (some packages may no longer exist)."""
-    specs = [s for s in spack.db.installed_known_package_specs()]
+    specs = [s for s in spack.installed_db.installed_known_package_specs()]
 
     for name, cls in module_types.items():
         tty.msg("Regenerating %s module files." % name)
diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py
index aa62510fed..4870712eb6 100644
--- a/lib/spack/spack/cmd/uninstall.py
+++ b/lib/spack/spack/cmd/uninstall.py
@@ -59,7 +59,7 @@ def uninstall(parser, args):
     # Fail and ask user to be unambiguous if it doesn't
     pkgs = []
     for spec in specs:
-        matching_specs = spack.db.get_installed(spec)
+        matching_specs = spack.installed_db.get_installed(spec)
         if not args.all and len(matching_specs) > 1:
             tty.error("%s matches multiple packages:" % spec)
             print
diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py
index 4646530d52..d027db312f 100644
--- a/lib/spack/spack/database.py
+++ b/lib/spack/spack/database.py
@@ -28,6 +28,9 @@ import inspect
 import glob
 import imp
 
+import time
+import copy
+
 from external import yaml
 from external.yaml.error import MarkedYAMLError
 
@@ -43,8 +46,18 @@ from spack.virtual import ProviderIndex
 from spack.util.naming import mod_to_class, validate_module_name
 
 
+def _autospec(function):
+    """Decorator that automatically converts the argument of a single-arg
+       function to a Spec."""
+    def converter(self, spec_like, **kwargs):
+        if not isinstance(spec_like, spack.spec.Spec):
+            spec_like = spack.spec.Spec(spec_like)
+        return function(self, spec_like, **kwargs)
+    return converter
+
+
 class Database(object):
-    def __init__(self,file_name="specDB.yaml"):
+    def __init__(self,root,file_name="specDB.yaml"):
         """
         Create an empty Database
         Location defaults to root/specDB.yaml
@@ -53,13 +66,16 @@ class Database(object):
         path: the path to the install of that package
         dep_hash: a hash of the dependence DAG for that package
         """
+        self.root = root
         self.file_name = file_name
+        self.file_path = join_path(self.root,self.file_name)
         self.data = []
+        self.last_write_time = 0
 
 
     def from_yaml(self,stream):
         """
-        Fill database from YAML
+        Fill database from YAML, do not maintain old data
         Translate the spec portions from node-dict form to spec from
         """
         try:
@@ -70,6 +86,7 @@ class Database(object):
         if file==None:
             return
 
+        self.data = []
         for sp in file['database']:
             spec = Spec.from_node_dict(sp['spec'])
             path = sp['path']
@@ -78,19 +95,14 @@ class Database(object):
             self.data.append(db_entry)
 
 
-    @staticmethod
-    def read_database(root):
-        """Create a Database from the data in the standard location"""
-        database = Database()
-        full_path = join_path(root,database.file_name)
-        if os.path.isfile(full_path):
-            with open(full_path,'r') as f:
-                database.from_yaml(f)
+    def read_database(self):
+        """Reread Database from the data in the set location"""
+        if os.path.isfile(self.file_path):
+            with open(self.file_path,'r') as f:
+                self.from_yaml(f)
         else:
-            with open(full_path,'w+') as f:
-                database.from_yaml(f)
-
-        return database
+            #The file doesn't exist, construct empty data.
+            self.data = []
 
 
     def write_database_to_yaml(self,stream):
@@ -110,48 +122,104 @@ class Database(object):
                          stream=stream, default_flow_style=False)
 
 
-    def write(self,root):
+    def write(self):
         """Write the database to the standard location"""
-        full_path = join_path(root,self.file_name)
-        #test for file existence
-        with open(full_path,'w') as f:
+        #creates file if necessary
+        with open(self.file_path,'w') as f:
+            self.last_write_time = int(time.time())
             self.write_database_to_yaml(f)
 
 
-    @staticmethod
-    def add(root, spec, path):
-        """Read the database from the standard location
+    def is_dirty(self):
+        """
+        Returns true iff the database file exists
+        and was most recently written to by another spack instance.
+        """
+        return (os.path.isfile(self.file_path) and (os.path.getmtime(self.file_path) > self.last_write_time))
+
+
+#    @_autospec
+    def add(self, spec, path):
+        """Re-read the database from the set location if data is dirty
         Add the specified entry as a dict
         Write the database back to memory
-
-        TODO: Caching databases
         """
-        database = Database.read_database(root)
+        if self.is_dirty():
+            self.read_database()
 
         sph = {}
         sph['spec']=spec
         sph['path']=path
         sph['hash']=spec.dag_hash()
 
-        database.data.append(sph)
+        self.data.append(sph)
 
-        database.write(root)
+        self.write()
 
 
-    @staticmethod
-    def remove(root, spec):
+    @_autospec
+    def remove(self, spec):
         """
-        Reads the database from the standard location
+        Re-reads the database from the set location if data is dirty
         Searches for and removes the specified spec
         Writes the database back to memory
+        """
+        if self.is_dirty():
+            self.read_database()
+
+        for sp in self.data:
+
+            if sp['hash'] == spec.dag_hash() and sp['spec'] == Spec.from_node_dict(spec.to_node_dict()):
+                self.data.remove(sp)
+
+        self.write()
 
-        TODO: Caching databases
+
+    @_autospec
+    def get_installed(self, spec):
+        """
+        Get all the installed specs that satisfy the provided spec constraint
+        """
+        return [s for s in self.installed_package_specs() if s.satisfies(spec)]
+
+
+    @_autospec
+    def installed_extensions_for(self, extendee_spec):
+        """
+        Return the specs of all packages that extend
+        the given spec
+        """
+        for s in self.installed_package_specs():
+            try:
+                if s.package.extends(extendee_spec):
+                    yield s.package
+            except UnknownPackageError, e:
+                continue
+            #skips unknown packages
+            #TODO: conditional way to do this instead of catching exceptions
+
+
+    def installed_package_specs(self):
         """
-        database = Database.read_database(root)
+        Read installed package names from the database
+        and return their specs
+        """
+        if self.is_dirty():
+            self.read_database()
+
+        installed = []
+        for sph in self.data:
+            sph['spec'].normalize()
+            sph['spec'].concretize()
+            installed.append(sph['spec'])
+        return installed
 
-        for sp in database.data:
-            #This requires specs w/o dependencies, is that sustainable?
-            if sp['spec'] == spec:
-                database.data.remove(sp)
 
-        database.write(root)
+    def installed_known_package_specs(self):
+        """
+        Read installed package names from the database.
+        Return only the specs for which the package is known
+        to this version of spack
+        """
+        return [s for s in self.installed_package_specs() if spack.db.exists(s.name)]
+
diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py
index 0bbe6c9d85..6dc1b0e550 100644
--- a/lib/spack/spack/directory_layout.py
+++ b/lib/spack/spack/directory_layout.py
@@ -153,7 +153,6 @@ class DirectoryLayout(object):
                 os.rmdir(path)
             path = os.path.dirname(path)
 
-        Database.remove(self.root,spec)
 
 
 class YamlDirectoryLayout(DirectoryLayout):
@@ -266,11 +265,6 @@ class YamlDirectoryLayout(DirectoryLayout):
         self.write_spec(spec, spec_file_path)
 
 
-    def add_to_database(self, spec):
-        """Simply adds a spec to the database"""
-        Database.add(self.root, spec, self.path_for_spec(spec))
-
-
     @memoized
     def all_specs(self):
         if not os.path.isdir(self.root):
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index a425c555a2..acf558d639 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -565,7 +565,7 @@ class Package(object):
         """Return a list of the specs of all installed packages that depend
            on this one."""
         dependents = []
-        for spec in spack.db.installed_package_specs():
+        for spec in spack.installed_db.installed_package_specs():
             if self.name == spec.name:
                 continue
             for dep in spec.traverse():
@@ -601,7 +601,7 @@ class Package(object):
     def remove_prefix(self):
         """Removes the prefix for a package along with any empty parent directories."""
         spack.install_layout.remove_install_directory(self.spec)
-
+        spack.installed_db.remove(self.spec)
 
     def do_fetch(self):
         """Creates a stage directory and downloads the taball for this package.
@@ -818,7 +818,7 @@ class Package(object):
                     install(log_path, log_install_path)
 
                 #Update the database once we know install successful
-                spack.install_layout.add_to_database(self.spec)
+                spack.installed_db.add(self.spec, spack.install_layout.path_for_spec(self.spec))
 
                 # On successful install, remove the stage.
                 if not keep_stage:
diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py
index adfbc26c1d..2e3e95ca40 100644
--- a/lib/spack/spack/packages.py
+++ b/lib/spack/spack/packages.py
@@ -96,12 +96,6 @@ class PackageDB(object):
 
 
     @_autospec
-    def get_installed(self, spec):
-        """Get all the installed specs that satisfy the provided spec constraint."""
-        return [s for s in self.installed_package_specs() if s.satisfies(spec)]
-
-
-    @_autospec
     def providers_for(self, vpkg_spec):
         if self.provider_index is None:
             self.provider_index = ProviderIndex(self.all_package_names())
@@ -117,19 +111,6 @@ class PackageDB(object):
         return [p for p in self.all_packages() if p.extends(extendee_spec)]
 
 
-    @_autospec
-    def installed_extensions_for(self, extendee_spec):
-        for s in self.installed_package_specs():
-            try:
-                if s.package.extends(extendee_spec):
-                    yield s.package
-            except UnknownPackageError, e:
-                # Skip packages we know nothing about
-                continue
-                # TODO: add some conditional way to do this instead of
-                # catching exceptions.
-
-
     def dirname_for_package_name(self, pkg_name):
         """Get the directory name for a particular package.  This is the
            directory that contains its package.py file."""
@@ -150,29 +131,6 @@ class PackageDB(object):
         return join_path(pkg_dir, _package_file_name)
 
 
-    def installed_package_specs(self):
-        """Read installed package names straight from the install directory
-           layout.
-        """
-        # Get specs from the directory layout but ensure that they're
-        # all normalized properly.
-        installed = []
-        for spec in spack.install_layout.all_specs():
-            spec.normalize()
-            installed.append(spec)
-        return installed
-
-
-    def installed_known_package_specs(self):
-        """Read installed package names straight from the install
-           directory layout, but return only specs for which the
-           package is known to this version of spack.
-        """
-        for spec in spack.install_layout.all_specs():
-            if self.exists(spec.name):
-                yield spec
-
-
     @memoized
     def all_package_names(self):
         """Generator function for all packages.  This looks for