From 9347f86939f3dea9d8d8ea723c238cd4f8c518a6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 27 Oct 2016 10:57:27 -0700 Subject: Rename install.yaml -> config.yaml, install_area -> "store" - Added a schema for config.yaml - Moved install tree configuration to config.yaml - Moved etc/spack/install.yaml to etc/spack/defaults/config.yaml - renamed install_area to "store", to use a term in common with guix/nix. - in `config.yaml` file, it's called the `install_tree` to be more intuitive to users. - `install_tree` might've worked in the code, but `install_tree` is already a global function in the spack namespace, from llnl.util.filesystem. --- lib/spack/docs/configuration.rst | 4 +- lib/spack/spack/__init__.py | 16 ------- lib/spack/spack/build_environment.py | 4 +- lib/spack/spack/cmd/__init__.py | 4 +- lib/spack/spack/cmd/common/arguments.py | 4 +- lib/spack/spack/cmd/deactivate.py | 4 +- lib/spack/spack/cmd/extensions.py | 6 +-- lib/spack/spack/cmd/find.py | 6 +-- lib/spack/spack/cmd/reindex.py | 4 +- lib/spack/spack/cmd/uninstall.py | 4 +- lib/spack/spack/config.py | 2 +- lib/spack/spack/database.py | 6 +-- lib/spack/spack/install_area.py | 36 ---------------- lib/spack/spack/package.py | 70 +++++++++++++++--------------- lib/spack/spack/schema/config.py | 62 +++++++++++++++++++++++++++ lib/spack/spack/schema/install.py | 43 ------------------- lib/spack/spack/spec.py | 8 ++-- lib/spack/spack/store.py | 75 +++++++++++++++++++++++++++++++++ lib/spack/spack/test/cmd/uninstall.py | 4 +- lib/spack/spack/test/database.py | 30 ++++++------- lib/spack/spack/test/install.py | 16 +++---- lib/spack/spack/test/mock_database.py | 28 ++++++------ 22 files changed, 241 insertions(+), 195 deletions(-) delete mode 100644 lib/spack/spack/install_area.py create mode 100644 lib/spack/spack/schema/config.py delete mode 100644 lib/spack/spack/schema/install.py create mode 100644 lib/spack/spack/store.py (limited to 'lib') diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index c797e7e948..1244a27d2f 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -10,8 +10,8 @@ Install options ---------------------------- By default, Spack will install software into ``opt/spack``. -To set a custom install directory, the option ``path`` in -``install.yaml`` can be used. This file can be found +To set a custom install directory, the option ``install_tree`` in +``config.yaml`` can be used. This file can be found in a Spack installation's ``etc/spack/`` or a user's ``~/.spack/`` directory. diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 8a0733871f..ab03c0c848 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -145,22 +145,6 @@ for path in _tmp_candidates: # for. do_checksum = True -# -# SYS_TYPE to use for the spack installation. -# Value of this determines what platform spack thinks it is by -# default. You can assign three types of values: -# 1. None -# Spack will try to determine the sys_type automatically. -# -# 2. A string -# Spack will assume that the sys_type is hardcoded to the value. -# -# 3. A function that returns a string: -# Spack will use this function to determine the sys_type. -# -sys_type = None - - # # When packages call 'from spack import *', this extra stuff is brought in. # diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 942b525bf6..f626bab1af 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -58,7 +58,7 @@ import sys import llnl.util.tty as tty import spack -import spack.install_area +import spack.store from llnl.util.filesystem import * from spack.environment import EnvironmentModifications, validate from spack.util.environment import * @@ -277,7 +277,7 @@ def set_build_environment_variables(pkg, env, dirty=False): env.set(SPACK_PREFIX, pkg.prefix) # Install root prefix - env.set(SPACK_INSTALL, spack.install_area.path) + env.set(SPACK_INSTALL, spack.store.root) # Stuff in here sanitizes the build environemnt to eliminate # anything the user has set that may interfere. diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 0da542e930..8000c9f1f2 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -34,7 +34,7 @@ from llnl.util.tty.color import * import spack import spack.config import spack.spec -import spack.install_area +import spack.store # # Settings for commands that modify configuration @@ -137,7 +137,7 @@ def elide_list(line_list, max_num=10): def disambiguate_spec(spec): - matching_specs = spack.install_area.db.query(spec) + matching_specs = spack.store.db.query(spec) if not matching_specs: tty.die("Spec '%s' matches no installed packages." % spec) diff --git a/lib/spack/spack/cmd/common/arguments.py b/lib/spack/spack/cmd/common/arguments.py index d7d2f9770c..5ab54bdb61 100644 --- a/lib/spack/spack/cmd/common/arguments.py +++ b/lib/spack/spack/cmd/common/arguments.py @@ -25,7 +25,7 @@ import argparse -import spack.install_area +import spack.store import spack.modules from spack.util.pattern import Bunch __all__ = ['add_common_arguments'] @@ -54,7 +54,7 @@ class ConstraintAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): # Query specs from command line d = self.qualifiers.get(namespace.subparser_name, {}) - specs = [s for s in spack.install_area.db.query(**d)] + specs = [s for s in spack.store.db.query(**d)] values = ' '.join(values) if values: specs = [x for x in specs if x.satisfies(values, strict=True)] diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index 859025034a..fedd078972 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -27,7 +27,7 @@ import llnl.util.tty as tty import spack import spack.cmd -import spack.install_area +import spack.store from spack.graph import topological_sort description = "Deactivate a package extension." @@ -57,7 +57,7 @@ def deactivate(parser, args): if args.all: if pkg.extendable: tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec) - ext_pkgs = spack.install_area.db.installed_extensions_for(spec) + ext_pkgs = spack.store.db.installed_extensions_for(spec) for ext_pkg in ext_pkgs: ext_pkg.spec.normalize() diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index 49572263c5..bd149044ca 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -30,7 +30,7 @@ from llnl.util.tty.colify import colify import spack import spack.cmd import spack.cmd.find -import spack.install_area +import spack.store description = "List extensions for package." @@ -88,7 +88,7 @@ def extensions(parser, args): # List specs of installed extensions. # installed = [s.spec - for s in spack.install_area.db.installed_extensions_for(spec)] + for s in spack.store.db.installed_extensions_for(spec)] print if not installed: @@ -100,7 +100,7 @@ def extensions(parser, args): # # List specs of activated extensions. # - activated = spack.install_area.layout.extension_map(spec) + activated = spack.store.layout.extension_map(spec) print if not activated: tty.msg("None activated.") diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 9e631d6e24..50e6112486 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -28,7 +28,7 @@ import sys import llnl.util.tty as tty import spack import spack.spec -import spack.install_area +import spack.store from llnl.util.lang import * from llnl.util.tty.colify import * from llnl.util.tty.color import * @@ -146,9 +146,9 @@ def find(parser, args): # Get all the specs the user asked for if not query_specs: - specs = set(spack.install_area.db.query(**q_args)) + specs = set(spack.store.db.query(**q_args)) else: - results = [set(spack.install_area.db.query(qs, **q_args)) + results = [set(spack.store.db.query(qs, **q_args)) for qs in query_specs] specs = set.union(*results) diff --git a/lib/spack/spack/cmd/reindex.py b/lib/spack/spack/cmd/reindex.py index 10c5ea4c05..7dddda2ffb 100644 --- a/lib/spack/spack/cmd/reindex.py +++ b/lib/spack/spack/cmd/reindex.py @@ -23,9 +23,9 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import spack -import spack.install_area +import spack.store description = "Rebuild Spack's package database." def reindex(parser, args): - spack.install_area.db.reindex(spack.install_area.layout) + spack.store.db.reindex(spack.store.layout) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 26d6080ffb..f48b28ff73 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -29,7 +29,7 @@ import argparse import llnl.util.tty as tty import spack import spack.cmd -import spack.install_area +import spack.store import spack.repository description = "Remove an installed package" @@ -90,7 +90,7 @@ def concretize_specs(specs, allow_multiple_matches=False, force=False): specs_from_cli = [] has_errors = False for spec in specs: - matching = spack.install_area.db.query(spec) + matching = spack.store.db.query(spec) # For each spec provided, make sure it refers to only one package. # Fail and ask user to be unambiguous if it doesn't if not allow_multiple_matches and len(matching) > 1: diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 89e02e33df..1bd1689150 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -180,7 +180,7 @@ section_schemas = { 'repos': spack.schema.repos.schema, 'packages': spack.schema.packages.schema, 'modules': spack.schema.modules.schema, - 'install': spack.schema.install.schema, + 'config': spack.schema.config.schema, } """OrderedDict of config scopes keyed by name. diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index e0db8c3167..e8902ec024 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -48,7 +48,7 @@ import llnl.util.tty as tty from llnl.util.filesystem import * from llnl.util.lock import * -import spack.install_area +import spack.store import spack.repository from spack.directory_layout import DirectoryLayoutError from spack.version import Version @@ -280,7 +280,7 @@ class Database(object): if version > _db_version: raise InvalidDatabaseVersionError(_db_version, version) elif version < _db_version: - self.reindex(spack.install_area.layout) + self.reindex(spack.store.layout) installs = dict((k, v.to_dict()) for k, v in self._data.items()) def invalid_record(hash_key, error): @@ -446,7 +446,7 @@ class Database(object): else: # The file doesn't exist, try to traverse the directory. # reindex() takes its own write lock, so no lock here. - self.reindex(spack.install_area.layout) + self.reindex(spack.store.layout) def _add(self, spec, directory_layout=None, explicit=False): """Add an install record for this spec to the database. diff --git a/lib/spack/spack/install_area.py b/lib/spack/spack/install_area.py deleted file mode 100644 index b99df4c570..0000000000 --- a/lib/spack/spack/install_area.py +++ /dev/null @@ -1,36 +0,0 @@ -__author__ = "Benedikt Hegner (CERN)" - -import re - -import llnl.util.tty as tty -from llnl.util.filesystem import * - -# -# Read in the config -# -import spack.config -config = spack.config.get_config("install") - -# -# Set up the install path -# -path = re.sub(r'^\$spack', spack.prefix, config['path']) - -# -# Set up the installed packages database -# -from spack.database import Database -db = Database(path) - -# -# This controls how spack lays out install prefixes and -# stage directories. -# -import spack.directory_layout - -try: - layout_name = config["layout"] - layout_class = getattr(spack.directory_layout,layout_name) - layout = layout_class(path) -except: - tty.die("Invalid install directory layout %s chosen." %layout_name) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index d9145508e3..6bc3362639 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -47,7 +47,7 @@ from StringIO import StringIO import llnl.util.lock import llnl.util.tty as tty import spack -import spack.install_area +import spack.store import spack.compilers import spack.directives import spack.error @@ -832,7 +832,7 @@ class PackageBase(object): if not self.is_extension: raise ValueError( "is_extension called on package that is not an extension.") - exts = spack.install_area.layout.extension_map(self.extendee_spec) + exts = spack.store.layout.extension_map(self.extendee_spec) return (self.name in exts) and (exts[self.name] == self.spec) def provides(self, vpkg_name): @@ -853,7 +853,7 @@ class PackageBase(object): TODO: move this method to database.py? """ dependents = [] - for spec in spack.install_area.db.query(): + for spec in spack.store.db.query(): if self.name == spec.name: continue # XXX(deptype): Should build dependencies not count here? @@ -867,7 +867,7 @@ class PackageBase(object): def prefix_lock(self): """Prefix lock is a byte range lock on the nth byte of a file. - The lock file is ``spack.install_area.db.prefix_lock`` -- the DB + The lock file is ``spack.store.db.prefix_lock`` -- the DB tells us what to call it and it lives alongside the install DB. n is the sys.maxsize-bit prefix of the DAG hash. This makes @@ -879,7 +879,7 @@ class PackageBase(object): prefix = self.spec.prefix if prefix not in Package.prefix_locks: Package.prefix_locks[prefix] = llnl.util.lock.Lock( - spack.install_area.db.prefix_lock_path, + spack.store.db.prefix_lock_path, self.spec.dag_hash_bit_prefix(bit_length(sys.maxsize)), 1) self._prefix_lock = Package.prefix_locks[prefix] @@ -916,7 +916,7 @@ class PackageBase(object): Removes the prefix for a package along with any empty parent directories """ - spack.install_area.layout.remove_install_directory(self.spec) + spack.store.layout.remove_install_directory(self.spec) def do_fetch(self, mirror_only=False): """ @@ -1153,15 +1153,15 @@ class PackageBase(object): return # Ensure package is not already installed - layout = spack.install_area.layout + layout = spack.store.layout with self._prefix_read_lock(): if layout.check_installed(self.spec): tty.msg( "%s is already installed in %s" % (self.name, self.prefix)) - rec = spack.install_area.db.get_record(self.spec) + rec = spack.store.db.get_record(self.spec) if (not rec.explicit) and explicit: - with spack.install_area.db.write_transaction(): - rec = spack.install_area.db.get_record(self.spec) + with spack.store.db.write_transaction(): + rec = spack.store.db.get_record(self.spec) rec.explicit = True return @@ -1266,15 +1266,15 @@ class PackageBase(object): try: # Create the install prefix and fork the build process. - spack.install_area.layout.create_install_directory(self.spec) + spack.store.layout.create_install_directory(self.spec) # Fork a child to do the actual installation spack.build_environment.fork(self, build_process, dirty=dirty) # If we installed then we should keep the prefix keep_prefix = True if self.last_phase is None else keep_prefix # note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. - spack.install_area.db.add( - self.spec, spack.install_area.layout, explicit=explicit + spack.store.db.add( + self.spec, spack.store.layout, explicit=explicit ) except directory_layout.InstallDirectoryAlreadyExistsError: # Abort install if install directory exists. @@ -1303,15 +1303,16 @@ class PackageBase(object): """ self.last_phase = kwargs.pop('stop_at', None) if self.last_phase is not None and self.last_phase not in self.phases: - tty.die('\'{0.last_phase}\' is not among the allowed phases for package {0.name}'.format(self)) # NOQA: ignore=E501 + tty.die('\'{0}\' is not an allowed phase for package {1}' + .format(self.last_phase, self.name)) def log(self): # Copy provenance into the install directory on success - log_install_path = spack.install_area.layout.build_log_path( + log_install_path = spack.store.layout.build_log_path( self.spec) - env_install_path = spack.install_area.layout.build_env_path( + env_install_path = spack.store.layout.build_env_path( self.spec) - packages_dir = spack.install_area.layout.build_packages_path( + packages_dir = spack.store.layout.build_packages_path( self.spec) # Remove first if we're overwriting another build @@ -1345,7 +1346,8 @@ class PackageBase(object): check_paths(self.sanity_check_is_dir, 'directory', os.path.isdir) installed = set(os.listdir(self.prefix)) - installed.difference_update(spack.install_area.layout.hidden_file_paths) + installed.difference_update( + spack.store.layout.hidden_file_paths) if not installed: raise InstallError( "Install failed for %s. Nothing was installed!" % self.name) @@ -1353,7 +1355,7 @@ class PackageBase(object): @property def build_log_path(self): if self.installed: - return spack.install_area.layout.build_log_path(self.spec) + return spack.store.layout.build_log_path(self.spec) else: return join_path(self.stage.source_path, 'spack-build.out') @@ -1483,9 +1485,9 @@ class PackageBase(object): if not self.installed: # prefix may not exist, but DB may be inconsistent. Try to fix by # removing, but omit hooks. - specs = spack.install_area.db.query(self.spec, installed=True) + specs = spack.store.db.query(self.spec, installed=True) if specs: - spack.install_area.db.remove(specs[0]) + spack.store.db.remove(specs[0]) tty.msg("Removed stale DB entry for %s" % self.spec.short_spec) return else: @@ -1502,7 +1504,7 @@ class PackageBase(object): # Uninstalling in Spack only requires removing the prefix. self.remove_prefix() # - spack.install_area.db.remove(self.spec) + spack.store.db.remove(self.spec) tty.msg("Successfully uninstalled %s" % self.spec.short_spec) # Once everything else is done, run post install hooks @@ -1536,8 +1538,8 @@ class PackageBase(object): """ self._sanity_check_extension() - spack.install_area.layout.check_extension_conflict(self.extendee_spec, - self.spec) + spack.store.layout.check_extension_conflict( + self.extendee_spec, self.spec) # Activate any package dependencies that are also extensions. if not force: @@ -1548,7 +1550,7 @@ class PackageBase(object): self.extendee_spec.package.activate(self, **self.extendee_args) - spack.install_area.layout.add_extension(self.extendee_spec, self.spec) + spack.store.layout.add_extension(self.extendee_spec, self.spec) tty.msg("Activated extension %s for %s" % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) @@ -1563,7 +1565,7 @@ class PackageBase(object): """ def ignore(filename): - return (filename in spack.install_area.layout.hidden_file_paths or + return (filename in spack.store.layout.hidden_file_paths or kwargs.get('ignore', lambda f: False)(filename)) tree = LinkTree(extension.prefix) @@ -1581,9 +1583,11 @@ class PackageBase(object): # Allow a force deactivate to happen. This can unlink # spurious files if something was corrupted. if not force: - spack.install_area.layout.check_activated(self.extendee_spec, self.spec) + spack.store.layout.check_activated( + self.extendee_spec, self.spec) - activated = spack.install_area.layout.extension_map(self.extendee_spec) + activated = spack.store.layout.extension_map( + self.extendee_spec) for name, aspec in activated.items(): if aspec == self.spec: continue @@ -1599,8 +1603,8 @@ class PackageBase(object): # redundant activation check -- makes SURE the spec is not # still activated even if something was wrong above. if self.activated: - spack.install_area.layout.remove_extension(self.extendee_spec, - self.spec) + spack.store.layout.remove_extension( + self.extendee_spec, self.spec) tty.msg("Deactivated extension %s for %s" % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) @@ -1616,7 +1620,7 @@ class PackageBase(object): """ def ignore(filename): - return (filename in spack.install_area.layout.hidden_file_paths or + return (filename in spack.store.layout.hidden_file_paths or kwargs.get('ignore', lambda f: False)(filename)) tree = LinkTree(extension.prefix) @@ -1717,7 +1721,7 @@ def flatten_dependencies(spec, flat_dir): for dep in spec.traverse(root=False): name = dep.name - dep_path = spack.install_area.layout.path_for_spec(dep) + dep_path = spack.store.layout.path_for_spec(dep) dep_files = LinkTree(dep_path) os.mkdir(flat_dir + '/' + name) @@ -1746,7 +1750,7 @@ def dump_packages(spec, path): if node is not spec: # Locate the dependency package in the install tree and find # its provenance information. - source = spack.install_area.layout.build_packages_path(node) + source = spack.store.layout.build_packages_path(node) source_repo_root = join_path(source, node.namespace) # There's no provenance installed for the source package. Skip it. diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py new file mode 100644 index 0000000000..7ae3b75c3e --- /dev/null +++ b/lib/spack/spack/schema/config.py @@ -0,0 +1,62 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Schema for packages.yaml configuration files.""" + +schema = { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack module file configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + 'config': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'properties': { + 'install_tree': {'type': 'string'}, + 'build_stage': { + 'oneOf': [ + {'type': 'string'}, + {'type': 'array', + 'items': {'type': 'string'}}], + }, + 'source_cache': {'type': 'string'}, + 'misc_cache': {'type': 'string'}, + 'verify_ssl': { + 'type': 'boolean', + 'default': True, + }, + 'checksum': { + 'type': 'boolean', + 'default': True, + }, + 'dirty': { + 'type': 'boolean', + 'default': False, + }, + } + }, + }, +} diff --git a/lib/spack/spack/schema/install.py b/lib/spack/spack/schema/install.py deleted file mode 100644 index 2c43ad1adf..0000000000 --- a/lib/spack/spack/schema/install.py +++ /dev/null @@ -1,43 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -"""Schema for packages.yaml configuration files.""" - -schema = { - '$schema': 'http://json-schema.org/schema#', - 'title': 'Spack module file configuration file schema', - 'type': 'object', - 'additionalProperties': False, - 'patternProperties': { - 'install': { - 'type': 'object', - 'default' : {}, - 'additionalProperties': False, - 'properties': { - 'path' : { 'type': 'string' }, - 'layout' : { 'type': 'string' } - } - }, - }, -} diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 1d9fd8d97a..17f206d81b 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -111,7 +111,7 @@ from llnl.util.tty.color import * import spack import spack.architecture -import spack.install_area +import spack.store import spack.compilers as compilers import spack.error import spack.parse @@ -965,7 +965,7 @@ class Spec(object): @property def prefix(self): - return Prefix(spack.install_area.layout.path_for_spec(self)) + return Prefix(spack.store.layout.path_for_spec(self)) def dag_hash(self, length=None): """Return a hash of the entire spec DAG, including connectivity.""" @@ -2329,7 +2329,7 @@ class Spec(object): elif named_str == 'SPACK_ROOT': out.write(fmt % spack.prefix) elif named_str == 'SPACK_INSTALL': - out.write(fmt % spack.install_area.path) + out.write(fmt % spack.store.root) elif named_str == 'PREFIX': out.write(fmt % self.prefix) elif named_str.startswith('HASH'): @@ -2533,7 +2533,7 @@ class SpecParser(spack.parse.Parser): def spec_by_hash(self): self.expect(ID) - specs = spack.install_area.db.query() + specs = spack.store.db.query() matches = [spec for spec in specs if spec.dag_hash()[:len(self.token.value)] == self.token.value] diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py new file mode 100644 index 0000000000..3f559315d2 --- /dev/null +++ b/lib/spack/spack/store.py @@ -0,0 +1,75 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Components that manage Spack's installation tree. + +An install tree, or "build store" consists of two parts: + + 1. A package database that tracks what is installed. + 2. A directory layout that determines how the installations + are laid out. + +The store contains all the install prefixes for packages installed by +Spack. The simplest store could just contain prefixes named by DAG hash, +but we use a fancier directory layout to make browsing the store and +debugging easier. + +The directory layout is currently hard-coded to be a YAMLDirectoryLayout, +so called because it stores build metadata within each prefix, in +`spec.yaml` files. In future versions of Spack we may consider allowing +install trees to define their own layouts with some per-tree +configuration. + +""" +import os +import spack +import spack.config +from spack.util.path import canonicalize_path +from spack.database import Database +from spack.directory_layout import YamlDirectoryLayout + +__author__ = "Benedikt Hegner (CERN)" +__all__ = ['db', 'layout', 'root'] + +# +# Read in the config +# +config = spack.config.get_config("config") + +# +# Set up the install path +# +root = canonicalize_path( + config.get('install_tree', os.path.join(spack.opt_path, 'spack'))) + +# +# Set up the installed packages database +# +db = Database(root) + +# +# This controls how spack lays out install prefixes and +# stage directories. +# +layout = YamlDirectoryLayout(root) diff --git a/lib/spack/spack/test/cmd/uninstall.py b/lib/spack/spack/test/cmd/uninstall.py index d839c00b62..6a86a1543f 100644 --- a/lib/spack/spack/test/cmd/uninstall.py +++ b/lib/spack/spack/test/cmd/uninstall.py @@ -23,7 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import spack.test.mock_database -import spack.install_area +import spack.store from spack.cmd.uninstall import uninstall @@ -51,7 +51,7 @@ class TestUninstall(spack.test.mock_database.MockDatabase): args = MockArgs(['callpath'], all=True, dependents=True) uninstall(parser, args) - all_specs = spack.install_area.layout.all_specs() + all_specs = spack.store.layout.all_specs() self.assertEqual(len(all_specs), 7) # query specs with multiple configurations mpileaks_specs = [s for s in all_specs if s.satisfies('mpileaks')] diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index c0bb12ce10..1a1281e10e 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -30,7 +30,7 @@ import multiprocessing import os.path import spack -import spack.install_area +import spack.store from llnl.util.filesystem import join_path from llnl.util.tty.colify import colify from spack.test.mock_database import MockDatabase @@ -41,16 +41,16 @@ def _print_ref_counts(): recs = [] def add_rec(spec): - cspecs = spack.install_area.db.query(spec, installed=any) + cspecs = spack.store.db.query(spec, installed=any) if not cspecs: recs.append("[ %-7s ] %-20s-" % ('', spec)) else: key = cspecs[0].dag_hash() - rec = spack.install_area.db.get_record(cspecs[0]) + rec = spack.store.db.get_record(cspecs[0]) recs.append("[ %-7s ] %-20s%d" % (key[:7], spec, rec.ref_count)) - with spack.install_area.db.read_transaction(): + with spack.store.db.read_transaction(): add_rec('mpileaks ^mpich') add_rec('callpath ^mpich') add_rec('mpich') @@ -83,7 +83,7 @@ class DatabaseTest(MockDatabase): def test_010_all_install_sanity(self): """Ensure that the install layout reflects what we think it does.""" - all_specs = spack.install_area.layout.all_specs() + all_specs = spack.store.layout.all_specs() self.assertEqual(len(all_specs), 13) # query specs with multiple configurations @@ -114,12 +114,12 @@ class DatabaseTest(MockDatabase): def test_015_write_and_read(self): # write and read DB - with spack.install_area.db.write_transaction(): - specs = spack.install_area.db.query() - recs = [spack.install_area.db.get_record(s) for s in specs] + with spack.store.db.write_transaction(): + specs = spack.store.db.query() + recs = [spack.store.db.get_record(s) for s in specs] for spec, rec in zip(specs, recs): - new_rec = spack.install_area.db.get_record(spec) + new_rec = spack.store.db.get_record(spec) self.assertEqual(new_rec.ref_count, rec.ref_count) self.assertEqual(new_rec.spec, rec.spec) self.assertEqual(new_rec.path, rec.path) @@ -127,7 +127,7 @@ class DatabaseTest(MockDatabase): def _check_merkleiness(self): """Ensure the spack database is a valid merkle graph.""" - all_specs = spack.install_area.db.query(installed=any) + all_specs = spack.store.db.query(installed=any) seen = {} for spec in all_specs: @@ -140,7 +140,7 @@ class DatabaseTest(MockDatabase): def _check_db_sanity(self): """Utiilty function to check db against install layout.""" - expected = sorted(spack.install_area.layout.all_specs()) + expected = sorted(spack.store.layout.all_specs()) actual = sorted(self.install_db.query()) self.assertEqual(len(expected), len(actual)) @@ -155,7 +155,7 @@ class DatabaseTest(MockDatabase): def test_025_reindex(self): """Make sure reindex works and ref counts are valid.""" - spack.install_area.db.reindex(spack.install_area.layout) + spack.store.db.reindex(spack.store.layout) self._check_db_sanity() def test_030_db_sanity_from_another_process(self): @@ -179,7 +179,7 @@ class DatabaseTest(MockDatabase): def test_050_basic_query(self): """Ensure querying database is consistent with what is installed.""" # query everything - self.assertEqual(len(spack.install_area.db.query()), 13) + self.assertEqual(len(spack.store.db.query()), 13) # query specs with multiple configurations mpileaks_specs = self.install_db.query('mpileaks') @@ -224,7 +224,7 @@ class DatabaseTest(MockDatabase): self.assertTrue(concrete_spec not in remaining) # add it back and make sure everything is ok. - self.install_db.add(concrete_spec, spack.install_area.layout) + self.install_db.add(concrete_spec, spack.store.layout) installed = self.install_db.query() self.assertTrue(concrete_spec in installed) self.assertEqual(installed, original) @@ -255,7 +255,7 @@ class DatabaseTest(MockDatabase): self.assertEqual(self.install_db.get_record('mpich').ref_count, 1) # Put the spec back - self.install_db.add(rec.spec, spack.install_area.layout) + self.install_db.add(rec.spec, spack.store.layout) # record is present again self.assertEqual( diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index eb051e9301..0524c14cfd 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -26,7 +26,7 @@ import shutil import tempfile import spack -import spack.install_area +import spack.store from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout from spack.database import Database @@ -50,11 +50,11 @@ class InstallTest(MockPackagesTest): # Use a fake install directory to avoid conflicts bt/w # installed pkgs and mock packages. self.tmpdir = tempfile.mkdtemp() - self.orig_layout = spack.install_area.layout - self.orig_db = spack.install_area.db + self.orig_layout = spack.store.layout + self.orig_db = spack.store.db - spack.install_area.layout = YamlDirectoryLayout(self.tmpdir) - spack.install_area.db = Database(self.tmpdir) + spack.store.layout = YamlDirectoryLayout(self.tmpdir) + spack.store.db = Database(self.tmpdir) def tearDown(self): super(InstallTest, self).tearDown() @@ -64,8 +64,8 @@ class InstallTest(MockPackagesTest): spack.do_checksum = True # restore spack's layout. - spack.install_area.layout = self.orig_layout - spack.install_area.db = self.orig_db + spack.store.layout = self.orig_layout + spack.store.db = self.orig_db shutil.rmtree(self.tmpdir, ignore_errors=True) def fake_fetchify(self, pkg): @@ -92,7 +92,7 @@ class InstallTest(MockPackagesTest): pkg.remove_prefix() raise - def test_install_area(self): + def test_store(self): spec = Spec('cmake-client').concretized() for s in spec.traverse(): diff --git a/lib/spack/spack/test/mock_database.py b/lib/spack/spack/test/mock_database.py index d1f9fb000b..1eaec2b598 100644 --- a/lib/spack/spack/test/mock_database.py +++ b/lib/spack/spack/test/mock_database.py @@ -26,7 +26,7 @@ import shutil import tempfile import spack -import spack.install_area +import spack.store from spack.spec import Spec from spack.database import Database from spack.directory_layout import YamlDirectoryLayout @@ -42,7 +42,7 @@ class MockDatabase(MockPackagesTest): pkg.do_install(fake=True) def _mock_remove(self, spec): - specs = spack.install_area.db.query(spec) + specs = spack.store.db.query(spec) assert len(specs) == 1 spec = specs[0] spec.package.do_uninstall(spec) @@ -55,17 +55,17 @@ class MockDatabase(MockPackagesTest): # Make a fake install directory self.install_path = tempfile.mkdtemp() - self.spack_install_path = spack.install_area.path - spack.install_area.path = self.install_path + self.spack_install_path = spack.store.root + spack.store.root = self.install_path self.install_layout = YamlDirectoryLayout(self.install_path) - self.spack_install_layout = spack.install_area.layout - spack.install_area.layout = self.install_layout + self.spack_install_layout = spack.store.layout + spack.store.layout = self.install_layout # Make fake database and fake install directory. self.install_db = Database(self.install_path) - self.spack_install_db = spack.install_area.db - spack.install_area.db = self.install_db + self.spack_install_db = spack.store.db + spack.store.db = self.install_db # make a mock database with some packages installed note that # the ref count for dyninst here will be 3, as it's recycled @@ -91,18 +91,18 @@ class MockDatabase(MockPackagesTest): # # Transaction used to avoid repeated writes. - with spack.install_area.db.write_transaction(): + with spack.store.db.write_transaction(): self._mock_install('mpileaks ^mpich') self._mock_install('mpileaks ^mpich2') self._mock_install('mpileaks ^zmpi') def tearDown(self): - with spack.install_area.db.write_transaction(): - for spec in spack.install_area.db.query(): + with spack.store.db.write_transaction(): + for spec in spack.store.db.query(): spec.package.do_uninstall(spec) super(MockDatabase, self).tearDown() shutil.rmtree(self.install_path) - spack.install_area.path = self.spack_install_path - spack.install_area.layout = self.spack_install_layout - spack.install_area.db = self.spack_install_db + spack.store.root = self.spack_install_path + spack.store.layout = self.spack_install_layout + spack.store.db = self.spack_install_db -- cgit v1.2.3-70-g09d2