From 1da56e5290ba7272ea4bd0f18cdf5cae2b83f093 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 21 Aug 2015 11:32:12 -0700 Subject: Added a database of installed packages. No methods use the database so far. Also, a bug fix: Previous version did not remove the staging directory on a failed install This led to spack refusing to uninstall dependencies of the failed install Added to cleanup() to blow away the staging directory on failed install. --- lib/spack/spack/database.py | 150 ++++++++++++++++++++++++++++++++++++ lib/spack/spack/directory_layout.py | 8 ++ lib/spack/spack/package.py | 12 +++ lib/spack/spack/spec.py | 8 +- 4 files changed, 175 insertions(+), 3 deletions(-) create mode 100644 lib/spack/spack/database.py (limited to 'lib') diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py new file mode 100644 index 0000000000..8ae71a4385 --- /dev/null +++ b/lib/spack/spack/database.py @@ -0,0 +1,150 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import sys +import inspect +import glob +import imp + +from external import yaml +from external.yaml.error import MarkedYAMLError + +import llnl.util.tty as tty +from llnl.util.filesystem import join_path +from llnl.util.lang import * + +import spack.error +import spack.spec +from spack.spec import Spec +from spack.error import SpackError +from spack.virtual import ProviderIndex +from spack.util.naming import mod_to_class, validate_module_name + + +class Database(object): + def __init__(self,file_name="specDB.yaml"): + """ + Create an empty Database + Location defaults to root/specDB.yaml + """ + self.file_name = file_name + self.data = [] + + + def from_yaml(self,stream): + """ + Fill database from YAML + Translate the spec portions from node-dict form to spec from + """ + try: + file = yaml.load(stream) + except MarkedYAMLError, e: + raise SpackYAMLError("error parsing YAML database:", str(e)) + + if file==None: + return + + for sp in file['database']: + spec = Spec.from_node_dict(sp['spec']) + path = sp['path'] + db_entry = {'spec': spec, 'path': path} + self.data.append(db_entry) + + + @staticmethod + def read_database(root): + """Create a Database from the data in the standard location""" + database = Database() + full_path = join_path(root,database.file_name) + if os.path.isfile(full_path): + with open(full_path,'r') as f: + database.from_yaml(f) + else: + with open(full_path,'w+') as f: + database.from_yaml(f) + + return database + + + def write_database_to_yaml(self,stream): + """ + Replace each spec with its dict-node form + Then stream all data to YAML + """ + node_list = [] + for sp in self.data: + node = {} + node['spec']=Spec.to_node_dict(sp['spec']) + node['spec'][sp['spec'].name]['hash']=sp['spec'].dag_hash() + node['path']=sp['path'] + node_list.append(node) + return yaml.dump({ 'database' : node_list}, + stream=stream, default_flow_style=False) + + + def write(self,root): + """Write the database to the standard location""" + full_path = join_path(root,self.file_name) + #test for file existence + with open(full_path,'w') as f: + self.write_database_to_yaml(f) + + + @staticmethod + def add(root, spec, path): + """Read the database from the standard location + Add the specified entry as a dict + Write the database back to memory + + TODO: Caching databases + """ + database = Database.read_database(root) + + spec_and_path = {} + spec_and_path['spec']=spec + spec_and_path['path']=path + + database.data.append(spec_and_path) + + database.write(root) + + + @staticmethod + def remove(root, spec): + """ + Reads the database from the standard location + Searches for and removes the specified spec + Writes the database back to memory + + TODO: Caching databases + """ + database = Database.read_database(root) + + for sp in database.data: + #This requires specs w/o dependencies, is that sustainable? + if sp['spec'] == spec: + database.data.remove(sp) + + database.write(root) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index e61929d8fd..0bbe6c9d85 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -37,6 +37,7 @@ from llnl.util.filesystem import join_path, mkdirp from spack.spec import Spec from spack.error import SpackError +from spack.database import Database def _check_concrete(spec): @@ -152,6 +153,8 @@ class DirectoryLayout(object): os.rmdir(path) path = os.path.dirname(path) + Database.remove(self.root,spec) + class YamlDirectoryLayout(DirectoryLayout): """Lays out installation directories like this:: @@ -263,6 +266,11 @@ class YamlDirectoryLayout(DirectoryLayout): self.write_spec(spec, spec_file_path) + def add_to_database(self, spec): + """Simply adds a spec to the database""" + Database.add(self.root, spec, self.path_for_spec(spec)) + + @memoized def all_specs(self): if not os.path.isdir(self.root): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 3507807373..a425c555a2 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -779,6 +779,15 @@ class Package(object): "Manually remove this directory to fix:", self.prefix) + if not (keep_prefix and keep_stage): + self.do_clean() + else: + tty.warn("Keeping stage in place despite error.", + "Spack will refuse to uninstall dependencies of this package." + + "Manually remove this directory to fix:", + self.stage.path) + + def real_work(): try: tty.msg("Building %s." % self.name) @@ -808,6 +817,9 @@ class Package(object): log_install_path = spack.install_layout.build_log_path(self.spec) install(log_path, log_install_path) + #Update the database once we know install successful + spack.install_layout.add_to_database(self.spec) + # On successful install, remove the stage. if not keep_stage: self.stage.destroy() diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index e1fbb84423..df74b6064e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -427,7 +427,6 @@ class Spec(object): spec = dep if isinstance(dep, Spec) else Spec(dep) self._add_dependency(spec) - # # Private routines here are called by the parser when building a spec. # @@ -640,7 +639,10 @@ class Spec(object): def dag_hash(self, length=None): - """Return a hash of the entire spec DAG, including connectivity.""" + """ + Return a hash of the entire spec DAG, including connectivity. + Stores the hash iff the spec is concrete. + """ yaml_text = yaml.dump( self.to_node_dict(), default_flow_style=True, width=sys.maxint) sha = hashlib.sha1(yaml_text) @@ -710,7 +712,7 @@ class Spec(object): try: yfile = yaml.load(stream) except MarkedYAMLError, e: - raise SpackYAMLError("error parsing YMAL spec:", str(e)) + raise SpackYAMLError("error parsing YAML spec:", str(e)) for node in yfile['spec']: name = next(iter(node)) -- cgit v1.2.3-60-g2f50