From 973caa3a07a886e1c4fd7bb36363fc738a72da00 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 20 Jun 2016 11:10:13 -0400 Subject: modules.yaml: use the right envvar for pkg-config Fixes #1072. --- etc/spack/modules.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'etc') diff --git a/etc/spack/modules.yaml b/etc/spack/modules.yaml index 99be5e7b6d..5f31cdd5c0 100644 --- a/etc/spack/modules.yaml +++ b/etc/spack/modules.yaml @@ -22,8 +22,8 @@ modules: include: - CPATH lib/pkgconfig: - - PKGCONFIG + - PKG_CONFIG_PATH lib64/pkgconfig: - - PKGCONFIG + - PKG_CONFIG_PATH '': - CMAKE_PREFIX_PATH -- cgit v1.2.3-60-g2f50 From 6d36ec3d76f285ad3d5c9f4a3b3ce1994e7319f9 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 11 Jul 2016 13:35:22 -0500 Subject: Add share/man to prefix inspections. The share/man path should be in prefix_inspections for MANPATH as share/man is a common location for manual pages. --- etc/spack/modules.yaml | 2 ++ 1 file changed, 2 insertions(+) (limited to 'etc') diff --git a/etc/spack/modules.yaml b/etc/spack/modules.yaml index 5f31cdd5c0..9ae54a2d09 100644 --- a/etc/spack/modules.yaml +++ b/etc/spack/modules.yaml @@ -13,6 +13,8 @@ modules: - PATH man: - MANPATH + share/man: + - MANPATH lib: - LIBRARY_PATH - LD_LIBRARY_PATH -- cgit v1.2.3-60-g2f50 From 26480f14f901da087efbac263835c8767eaea23a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 19 Jul 2016 17:10:17 -0700 Subject: Add "default" configuration scope. - Default scope is versioned with spack and can be overridden by site or user config. - Default scope provides sensible default concretization preferences for all of Spack. - per-platform concretization scope can be added later (to force a particular MPI on, e.g., Cray systems) --- .gitignore | 2 +- etc/spack/defaults/modules.yaml | 40 ++++++++++++++++++++++++++++++++++++++++ etc/spack/defaults/packages.yaml | 21 +++++++++++++++++++++ etc/spack/defaults/repos.yaml | 14 ++++++++++++++ etc/spack/modules.yaml | 31 ------------------------------- etc/spack/repos.yaml | 8 -------- lib/spack/spack/config.py | 9 ++++++++- 7 files changed, 84 insertions(+), 41 deletions(-) create mode 100644 etc/spack/defaults/modules.yaml create mode 100644 etc/spack/defaults/packages.yaml create mode 100644 etc/spack/defaults/repos.yaml delete mode 100644 etc/spack/modules.yaml delete mode 100644 etc/spack/repos.yaml (limited to 'etc') diff --git a/.gitignore b/.gitignore index bfc6172a4e..1d4d24aa52 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ *~ .DS_Store .idea -/etc/spack/* +/etc/spack/*.yaml /etc/spackconfig /share/spack/dotkit /share/spack/modules diff --git a/etc/spack/defaults/modules.yaml b/etc/spack/defaults/modules.yaml new file mode 100644 index 0000000000..8864a76547 --- /dev/null +++ b/etc/spack/defaults/modules.yaml @@ -0,0 +1,40 @@ +# ------------------------------------------------------------------------- +# This is the default configuration for Spack's module file generation. +# +# Settings here are versioned with Spack and are intended to provide +# sensible defaults out of the box. Spack maintainers should edit this +# file to keep it current. +# +# Users can override these settings by editing the following files. +# +# Per-spack-instance settings (overrides defaults): +# $SPACK_ROOT/etc/spack/modules.yaml +# +# Per-user settings (overrides default and site settings): +# ~/.spack/modules.yaml +# ------------------------------------------------------------------------- +modules: + enable: + - tcl + - dotkit + prefix_inspections: + bin: + - PATH + man: + - MANPATH + share/man: + - MANPATH + lib: + - LIBRARY_PATH + - LD_LIBRARY_PATH + lib64: + - LIBRARY_PATH + - LD_LIBRARY_PATH + include: + - CPATH + lib/pkgconfig: + - PKG_CONFIG_PATH + lib64/pkgconfig: + - PKG_CONFIG_PATH + '': + - CMAKE_PREFIX_PATH diff --git a/etc/spack/defaults/packages.yaml b/etc/spack/defaults/packages.yaml new file mode 100644 index 0000000000..83f9eb7ece --- /dev/null +++ b/etc/spack/defaults/packages.yaml @@ -0,0 +1,21 @@ +# ------------------------------------------------------------------------- +# This file controls default concretization preferences for Spack. +# +# Settings here are versioned with Spack and are intended to provide +# sensible defaults out of the box. Spack maintainers should edit this +# file to keep it current. +# +# Users can override these settings by editing the following files. +# +# Per-spack-instance settings (overrides defaults): +# $SPACK_ROOT/etc/spack/packages.yaml +# +# Per-user settings (overrides default and site settings): +# ~/.spack/packages.yaml +# ------------------------------------------------------------------------- +packages: + all: + providers: + mpi: [openmpi, mpich] + blas: [openblas] + lapack: [openblas] diff --git a/etc/spack/defaults/repos.yaml b/etc/spack/defaults/repos.yaml new file mode 100644 index 0000000000..f3e00653eb --- /dev/null +++ b/etc/spack/defaults/repos.yaml @@ -0,0 +1,14 @@ +# ------------------------------------------------------------------------- +# This is the default spack repository configuration. It includes the +# builtin spack package repository. +# +# Users can override these settings by editing the following files. +# +# Per-spack-instance settings (overrides defaults): +# $SPACK_ROOT/etc/spack/repos.yaml +# +# Per-user settings (overrides default and site settings): +# ~/.spack/repos.yaml +# ------------------------------------------------------------------------- +repos: + - $spack/var/spack/repos/builtin diff --git a/etc/spack/modules.yaml b/etc/spack/modules.yaml deleted file mode 100644 index 9ae54a2d09..0000000000 --- a/etc/spack/modules.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# ------------------------------------------------------------------------- -# This is the default spack module files generation configuration. -# -# Changes to this file will affect all users of this spack install, -# although users can override these settings in their ~/.spack/modules.yaml. -# ------------------------------------------------------------------------- -modules: - enable: - - tcl - - dotkit - prefix_inspections: - bin: - - PATH - man: - - MANPATH - share/man: - - MANPATH - lib: - - LIBRARY_PATH - - LD_LIBRARY_PATH - lib64: - - LIBRARY_PATH - - LD_LIBRARY_PATH - include: - - CPATH - lib/pkgconfig: - - PKG_CONFIG_PATH - lib64/pkgconfig: - - PKG_CONFIG_PATH - '': - - CMAKE_PREFIX_PATH diff --git a/etc/spack/repos.yaml b/etc/spack/repos.yaml deleted file mode 100644 index 2d4ff54ce6..0000000000 --- a/etc/spack/repos.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# ------------------------------------------------------------------------- -# This is the default spack repository configuration. -# -# Changes to this file will affect all users of this spack install, -# although users can override these settings in their ~/.spack/repos.yaml. -# ------------------------------------------------------------------------- -repos: - - $spack/var/spack/repos/builtin diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3a66e9f2a6..8b5e96f97d 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -497,8 +497,15 @@ class ConfigScope(object): """Empty cached config information.""" self.sections = {} +"""Default configuration scope is the lowest-level scope. These are + versioned with Spack and can be overridden by sites or users.""" +ConfigScope('defaults', os.path.join(spack.etc_path, 'spack', 'defaults')) -ConfigScope('site', os.path.join(spack.etc_path, 'spack')), +"""Site configuration is per spack instance, for sites or projects. + No site-level configs should be checked into spack by default.""" +ConfigScope('site', os.path.join(spack.etc_path, 'spack')) + +"""User configuration can override both spack defaults and site config.""" ConfigScope('user', os.path.expanduser('~/.spack')) -- cgit v1.2.3-60-g2f50 From 3352889d73ef247f75a8a2d76e19e55b956de968 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Wed, 28 Sep 2016 16:51:41 -0400 Subject: py-pil: Protect against building with Python3. (#1868) * py-pil: Does not build with Python3. * Set py-pillow to be the default pil provider * Update package.py * Change to comments requested by adamjstewart * Remove version constraint from extends(), avoid a Spack bug. --- etc/spack/defaults/packages.yaml | 1 + var/spack/repos/builtin/packages/py-pil/package.py | 3 +++ 2 files changed, 4 insertions(+) (limited to 'etc') diff --git a/etc/spack/defaults/packages.yaml b/etc/spack/defaults/packages.yaml index 83f9eb7ece..a6b361d908 100644 --- a/etc/spack/defaults/packages.yaml +++ b/etc/spack/defaults/packages.yaml @@ -19,3 +19,4 @@ packages: mpi: [openmpi, mpich] blas: [openblas] lapack: [openblas] + pil: [py-pillow] diff --git a/var/spack/repos/builtin/packages/py-pil/package.py b/var/spack/repos/builtin/packages/py-pil/package.py index 2b3cce26d3..29a86d1e85 100644 --- a/var/spack/repos/builtin/packages/py-pil/package.py +++ b/var/spack/repos/builtin/packages/py-pil/package.py @@ -37,7 +37,10 @@ class PyPil(Package): provides('pil') + # py-pil currently only works with Python2. + # If you are using Python 3, try using py-pillow instead. extends('python') + depends_on('python@1.5.2:2.8') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) -- cgit v1.2.3-60-g2f50 From c557e765653d6ed3e81c5fe05bcb3042c5551caf Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Fri, 6 May 2016 11:10:02 +0200 Subject: refactor settings for install area and make them config options --- etc/spack/install.yaml | 8 ++ lib/spack/spack/__init__.py | 14 ---- lib/spack/spack/build_environment.py | 3 +- lib/spack/spack/cmd/__init__.py | 10 ++- lib/spack/spack/cmd/common/arguments.py | 3 +- lib/spack/spack/cmd/deactivate.py | 3 +- lib/spack/spack/cmd/extensions.py | 8 +- lib/spack/spack/cmd/find.py | 5 +- lib/spack/spack/cmd/reindex.py | 4 +- lib/spack/spack/cmd/uninstall.py | 3 +- lib/spack/spack/config.py | 6 +- lib/spack/spack/database.py | 20 ++--- lib/spack/spack/directory_layout.py | 4 +- lib/spack/spack/install_area.py | 36 +++++++++ lib/spack/spack/package.py | 59 ++++++++------- lib/spack/spack/repository.py | 2 +- lib/spack/spack/schema/install.py | 43 +++++++++++ lib/spack/spack/spec.py | 7 +- lib/spack/spack/test/cmd/uninstall.py | 4 +- lib/spack/spack/test/database.py | 129 ++++++++++++++++---------------- lib/spack/spack/test/install.py | 15 ++-- lib/spack/spack/test/mock_database.py | 29 +++---- 22 files changed, 254 insertions(+), 161 deletions(-) create mode 100644 etc/spack/install.yaml create mode 100644 lib/spack/spack/install_area.py create mode 100644 lib/spack/spack/schema/install.py (limited to 'etc') diff --git a/etc/spack/install.yaml b/etc/spack/install.yaml new file mode 100644 index 0000000000..553d09f13e --- /dev/null +++ b/etc/spack/install.yaml @@ -0,0 +1,8 @@ +# ------------------------------------------------------------------------- +# This is the default spack install setup configuration. +# +# Changes to this file will affect all users of this spack install +# ------------------------------------------------------------------------- +install: + path: $spack/opt/spack + layout : YamlDirectoryLayout \ No newline at end of file diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index cadfa95426..8a0733871f 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -63,7 +63,6 @@ user_cache = FileCache(user_cache_path) prefix = spack_root opt_path = join_path(prefix, "opt") -install_path = join_path(opt_path, "spack") etc_path = join_path(prefix, "etc") # @@ -76,25 +75,12 @@ try: except spack.error.SpackError, e: tty.die('while initializing Spack RepoPath:', e.message) -# -# Set up the installed packages database -# -from spack.database import Database -installed_db = Database(install_path) - # # Paths to built-in Spack repositories. # packages_path = join_path(repos_path, "builtin") mock_packages_path = join_path(repos_path, "builtin.mock") -# -# This controls how spack lays out install prefixes and -# stage directories. -# -from spack.directory_layout import YamlDirectoryLayout -install_layout = YamlDirectoryLayout(install_path) - # # This controls how packages are sorted when trying to choose # the most preferred package. More preferred packages are sorted diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index abbc385a39..942b525bf6 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -58,6 +58,7 @@ import sys import llnl.util.tty as tty import spack +import spack.install_area from llnl.util.filesystem import * from spack.environment import EnvironmentModifications, validate from spack.util.environment import * @@ -276,7 +277,7 @@ def set_build_environment_variables(pkg, env, dirty=False): env.set(SPACK_PREFIX, pkg.prefix) # Install root prefix - env.set(SPACK_INSTALL, spack.install_path) + env.set(SPACK_INSTALL, spack.install_area.path) # Stuff in here sanitizes the build environemnt to eliminate # anything the user has set that may interfere. diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 6b1561b7fc..0da542e930 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -27,13 +27,15 @@ import re import sys import llnl.util.tty as tty -import spack -import spack.config -import spack.spec from llnl.util.lang import * from llnl.util.tty.colify import * from llnl.util.tty.color import * +import spack +import spack.config +import spack.spec +import spack.install_area + # # Settings for commands that modify configuration # @@ -135,7 +137,7 @@ def elide_list(line_list, max_num=10): def disambiguate_spec(spec): - matching_specs = spack.installed_db.query(spec) + matching_specs = spack.install_area.db.query(spec) if not matching_specs: tty.die("Spec '%s' matches no installed packages." % spec) diff --git a/lib/spack/spack/cmd/common/arguments.py b/lib/spack/spack/cmd/common/arguments.py index afcba33714..d7d2f9770c 100644 --- a/lib/spack/spack/cmd/common/arguments.py +++ b/lib/spack/spack/cmd/common/arguments.py @@ -25,6 +25,7 @@ import argparse +import spack.install_area import spack.modules from spack.util.pattern import Bunch __all__ = ['add_common_arguments'] @@ -53,7 +54,7 @@ class ConstraintAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): # Query specs from command line d = self.qualifiers.get(namespace.subparser_name, {}) - specs = [s for s in spack.installed_db.query(**d)] + specs = [s for s in spack.install_area.db.query(**d)] values = ' '.join(values) if values: specs = [x for x in specs if x.satisfies(values, strict=True)] diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index 2b15a0331e..859025034a 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -27,6 +27,7 @@ import llnl.util.tty as tty import spack import spack.cmd +import spack.install_area from spack.graph import topological_sort description = "Deactivate a package extension." @@ -56,7 +57,7 @@ def deactivate(parser, args): if args.all: if pkg.extendable: tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec) - ext_pkgs = spack.installed_db.installed_extensions_for(spec) + ext_pkgs = spack.install_area.db.installed_extensions_for(spec) for ext_pkg in ext_pkgs: ext_pkg.spec.normalize() diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index b5c484305f..49572263c5 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -30,6 +30,7 @@ from llnl.util.tty.colify import colify import spack import spack.cmd import spack.cmd.find +import spack.install_area description = "List extensions for package." @@ -86,8 +87,9 @@ def extensions(parser, args): # # List specs of installed extensions. # - installed = [ - s.spec for s in spack.installed_db.installed_extensions_for(spec)] + installed = [s.spec + for s in spack.install_area.db.installed_extensions_for(spec)] + print if not installed: tty.msg("None installed.") @@ -98,7 +100,7 @@ def extensions(parser, args): # # List specs of activated extensions. # - activated = spack.install_layout.extension_map(spec) + activated = spack.install_area.layout.extension_map(spec) print if not activated: tty.msg("None activated.") diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index d3ea38c573..9e631d6e24 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -28,6 +28,7 @@ import sys import llnl.util.tty as tty import spack import spack.spec +import spack.install_area from llnl.util.lang import * from llnl.util.tty.colify import * from llnl.util.tty.color import * @@ -145,9 +146,9 @@ def find(parser, args): # Get all the specs the user asked for if not query_specs: - specs = set(spack.installed_db.query(**q_args)) + specs = set(spack.install_area.db.query(**q_args)) else: - results = [set(spack.installed_db.query(qs, **q_args)) + results = [set(spack.install_area.db.query(qs, **q_args)) for qs in query_specs] specs = set.union(*results) diff --git a/lib/spack/spack/cmd/reindex.py b/lib/spack/spack/cmd/reindex.py index e37eebbd92..10c5ea4c05 100644 --- a/lib/spack/spack/cmd/reindex.py +++ b/lib/spack/spack/cmd/reindex.py @@ -23,9 +23,9 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import spack - +import spack.install_area description = "Rebuild Spack's package database." def reindex(parser, args): - spack.installed_db.reindex(spack.install_layout) + spack.install_area.db.reindex(spack.install_area.layout) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index bbcd2e787c..26d6080ffb 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -29,6 +29,7 @@ import argparse import llnl.util.tty as tty import spack import spack.cmd +import spack.install_area import spack.repository description = "Remove an installed package" @@ -89,7 +90,7 @@ def concretize_specs(specs, allow_multiple_matches=False, force=False): specs_from_cli = [] has_errors = False for spec in specs: - matching = spack.installed_db.query(spec) + matching = spack.install_area.db.query(spec) # For each spec provided, make sure it refers to only one package. # Fail and ask user to be unambiguous if it doesn't if not allow_multiple_matches and len(matching) > 1: diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index de5f55775c..89e02e33df 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -171,7 +171,7 @@ import spack.schema # Hacked yaml for configuration files preserves line numbers. import spack.util.spack_yaml as syaml -from spack.build_environment import get_path_from_module + """Dict from section names -> schema for that section.""" section_schemas = { @@ -180,6 +180,7 @@ section_schemas = { 'repos': spack.schema.repos.schema, 'packages': spack.schema.packages.schema, 'modules': spack.schema.modules.schema, + 'install': spack.schema.install.schema, } """OrderedDict of config scopes keyed by name. @@ -510,6 +511,9 @@ def print_section(section): def spec_externals(spec): """Return a list of external specs (with external directory path filled in), one for each known external installation.""" + # break circular import. + from spack.build_environment import get_path_from_module + allpkgs = get_config('packages') name = spec.name diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 80ebf6a2cf..e0db8c3167 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -48,14 +48,15 @@ import llnl.util.tty as tty from llnl.util.filesystem import * from llnl.util.lock import * -import spack.spec +import spack.install_area +import spack.repository from spack.directory_layout import DirectoryLayoutError from spack.version import Version -from spack.spec import * +import spack.spec from spack.error import SpackError -from spack.repository import UnknownPackageError import spack.util.spack_yaml as syaml + # DB goes in this directory underneath the root _db_dirname = '.spack-db' @@ -66,7 +67,7 @@ _db_version = Version('0.9.2') _db_lock_timeout = 60 # Types of dependencies tracked by the database -_tracked_deps = nobuild +_tracked_deps = 'nobuild' def _autospec(function): @@ -218,7 +219,7 @@ class Database(object): spec_dict[name]['hash'] = hash_key # Build spec from dict first. - spec = Spec.from_node_dict(spec_dict) + spec = spack.spec.Spec.from_node_dict(spec_dict) return spec def _assign_dependencies(self, hash_key, installs, data): @@ -229,7 +230,8 @@ class Database(object): if 'dependencies' in spec_dict[spec.name]: yaml_deps = spec_dict[spec.name]['dependencies'] - for dname, dhash, dtypes in Spec.read_yaml_dep_specs(yaml_deps): + for dname, dhash, dtypes in spack.spec.Spec.read_yaml_dep_specs( + yaml_deps): if dhash not in data: tty.warn("Missing dependency not in database: ", "%s needs %s-%s" % ( @@ -278,7 +280,7 @@ class Database(object): if version > _db_version: raise InvalidDatabaseVersionError(_db_version, version) elif version < _db_version: - self.reindex(spack.install_layout) + self.reindex(spack.install_area.layout) installs = dict((k, v.to_dict()) for k, v in self._data.items()) def invalid_record(hash_key, error): @@ -444,7 +446,7 @@ class Database(object): else: # The file doesn't exist, try to traverse the directory. # reindex() takes its own write lock, so no lock here. - self.reindex(spack.install_layout) + self.reindex(spack.install_area.layout) def _add(self, spec, directory_layout=None, explicit=False): """Add an install record for this spec to the database. @@ -587,7 +589,7 @@ class Database(object): try: if s.package.extends(extendee_spec): yield s.package - except UnknownPackageError: + except spack.repository.UnknownPackageError: continue # skips unknown packages # TODO: conditional way to do this instead of catching exceptions diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 8ef7d3c480..28e6584fb2 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -32,7 +32,7 @@ import yaml from llnl.util.filesystem import join_path, mkdirp import spack -from spack.spec import Spec +import spack.spec from spack.error import SpackError @@ -210,7 +210,7 @@ class YamlDirectoryLayout(DirectoryLayout): """Read the contents of a file and parse them as a spec""" try: with open(path) as f: - spec = Spec.from_yaml(f) + spec = spack.spec.Spec.from_yaml(f) except Exception as e: if spack.debug: raise diff --git a/lib/spack/spack/install_area.py b/lib/spack/spack/install_area.py new file mode 100644 index 0000000000..b99df4c570 --- /dev/null +++ b/lib/spack/spack/install_area.py @@ -0,0 +1,36 @@ +__author__ = "Benedikt Hegner (CERN)" + +import re + +import llnl.util.tty as tty +from llnl.util.filesystem import * + +# +# Read in the config +# +import spack.config +config = spack.config.get_config("install") + +# +# Set up the install path +# +path = re.sub(r'^\$spack', spack.prefix, config['path']) + +# +# Set up the installed packages database +# +from spack.database import Database +db = Database(path) + +# +# This controls how spack lays out install prefixes and +# stage directories. +# +import spack.directory_layout + +try: + layout_name = config["layout"] + layout_class = getattr(spack.directory_layout,layout_name) + layout = layout_class(path) +except: + tty.die("Invalid install directory layout %s chosen." %layout_name) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 3a3028885f..d9145508e3 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -47,6 +47,7 @@ from StringIO import StringIO import llnl.util.lock import llnl.util.tty as tty import spack +import spack.install_area import spack.compilers import spack.directives import spack.error @@ -831,7 +832,7 @@ class PackageBase(object): if not self.is_extension: raise ValueError( "is_extension called on package that is not an extension.") - exts = spack.install_layout.extension_map(self.extendee_spec) + exts = spack.install_area.layout.extension_map(self.extendee_spec) return (self.name in exts) and (exts[self.name] == self.spec) def provides(self, vpkg_name): @@ -852,7 +853,7 @@ class PackageBase(object): TODO: move this method to database.py? """ dependents = [] - for spec in spack.installed_db.query(): + for spec in spack.install_area.db.query(): if self.name == spec.name: continue # XXX(deptype): Should build dependencies not count here? @@ -866,7 +867,7 @@ class PackageBase(object): def prefix_lock(self): """Prefix lock is a byte range lock on the nth byte of a file. - The lock file is ``spack.installed_db.prefix_lock`` -- the DB + The lock file is ``spack.install_area.db.prefix_lock`` -- the DB tells us what to call it and it lives alongside the install DB. n is the sys.maxsize-bit prefix of the DAG hash. This makes @@ -878,7 +879,7 @@ class PackageBase(object): prefix = self.spec.prefix if prefix not in Package.prefix_locks: Package.prefix_locks[prefix] = llnl.util.lock.Lock( - spack.installed_db.prefix_lock_path, + spack.install_area.db.prefix_lock_path, self.spec.dag_hash_bit_prefix(bit_length(sys.maxsize)), 1) self._prefix_lock = Package.prefix_locks[prefix] @@ -915,7 +916,7 @@ class PackageBase(object): Removes the prefix for a package along with any empty parent directories """ - spack.install_layout.remove_install_directory(self.spec) + spack.install_area.layout.remove_install_directory(self.spec) def do_fetch(self, mirror_only=False): """ @@ -1152,15 +1153,15 @@ class PackageBase(object): return # Ensure package is not already installed - layout = spack.install_layout + layout = spack.install_area.layout with self._prefix_read_lock(): if layout.check_installed(self.spec): tty.msg( "%s is already installed in %s" % (self.name, self.prefix)) - rec = spack.installed_db.get_record(self.spec) + rec = spack.install_area.db.get_record(self.spec) if (not rec.explicit) and explicit: - with spack.installed_db.write_transaction(): - rec = spack.installed_db.get_record(self.spec) + with spack.install_area.db.write_transaction(): + rec = spack.install_area.db.get_record(self.spec) rec.explicit = True return @@ -1265,15 +1266,15 @@ class PackageBase(object): try: # Create the install prefix and fork the build process. - spack.install_layout.create_install_directory(self.spec) + spack.install_area.layout.create_install_directory(self.spec) # Fork a child to do the actual installation spack.build_environment.fork(self, build_process, dirty=dirty) # If we installed then we should keep the prefix keep_prefix = True if self.last_phase is None else keep_prefix # note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. - spack.installed_db.add( - self.spec, spack.install_layout, explicit=explicit + spack.install_area.db.add( + self.spec, spack.install_area.layout, explicit=explicit ) except directory_layout.InstallDirectoryAlreadyExistsError: # Abort install if install directory exists. @@ -1306,11 +1307,11 @@ class PackageBase(object): def log(self): # Copy provenance into the install directory on success - log_install_path = spack.install_layout.build_log_path( + log_install_path = spack.install_area.layout.build_log_path( self.spec) - env_install_path = spack.install_layout.build_env_path( + env_install_path = spack.install_area.layout.build_env_path( self.spec) - packages_dir = spack.install_layout.build_packages_path( + packages_dir = spack.install_area.layout.build_packages_path( self.spec) # Remove first if we're overwriting another build @@ -1344,7 +1345,7 @@ class PackageBase(object): check_paths(self.sanity_check_is_dir, 'directory', os.path.isdir) installed = set(os.listdir(self.prefix)) - installed.difference_update(spack.install_layout.hidden_file_paths) + installed.difference_update(spack.install_area.layout.hidden_file_paths) if not installed: raise InstallError( "Install failed for %s. Nothing was installed!" % self.name) @@ -1352,7 +1353,7 @@ class PackageBase(object): @property def build_log_path(self): if self.installed: - return spack.install_layout.build_log_path(self.spec) + return spack.install_area.layout.build_log_path(self.spec) else: return join_path(self.stage.source_path, 'spack-build.out') @@ -1482,9 +1483,9 @@ class PackageBase(object): if not self.installed: # prefix may not exist, but DB may be inconsistent. Try to fix by # removing, but omit hooks. - specs = spack.installed_db.query(self.spec, installed=True) + specs = spack.install_area.db.query(self.spec, installed=True) if specs: - spack.installed_db.remove(specs[0]) + spack.install_area.db.remove(specs[0]) tty.msg("Removed stale DB entry for %s" % self.spec.short_spec) return else: @@ -1501,7 +1502,7 @@ class PackageBase(object): # Uninstalling in Spack only requires removing the prefix. self.remove_prefix() # - spack.installed_db.remove(self.spec) + spack.install_area.db.remove(self.spec) tty.msg("Successfully uninstalled %s" % self.spec.short_spec) # Once everything else is done, run post install hooks @@ -1535,7 +1536,7 @@ class PackageBase(object): """ self._sanity_check_extension() - spack.install_layout.check_extension_conflict(self.extendee_spec, + spack.install_area.layout.check_extension_conflict(self.extendee_spec, self.spec) # Activate any package dependencies that are also extensions. @@ -1547,7 +1548,7 @@ class PackageBase(object): self.extendee_spec.package.activate(self, **self.extendee_args) - spack.install_layout.add_extension(self.extendee_spec, self.spec) + spack.install_area.layout.add_extension(self.extendee_spec, self.spec) tty.msg("Activated extension %s for %s" % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) @@ -1562,7 +1563,7 @@ class PackageBase(object): """ def ignore(filename): - return (filename in spack.install_layout.hidden_file_paths or + return (filename in spack.install_area.layout.hidden_file_paths or kwargs.get('ignore', lambda f: False)(filename)) tree = LinkTree(extension.prefix) @@ -1580,9 +1581,9 @@ class PackageBase(object): # Allow a force deactivate to happen. This can unlink # spurious files if something was corrupted. if not force: - spack.install_layout.check_activated(self.extendee_spec, self.spec) + spack.install_area.layout.check_activated(self.extendee_spec, self.spec) - activated = spack.install_layout.extension_map(self.extendee_spec) + activated = spack.install_area.layout.extension_map(self.extendee_spec) for name, aspec in activated.items(): if aspec == self.spec: continue @@ -1598,7 +1599,7 @@ class PackageBase(object): # redundant activation check -- makes SURE the spec is not # still activated even if something was wrong above. if self.activated: - spack.install_layout.remove_extension(self.extendee_spec, + spack.install_area.layout.remove_extension(self.extendee_spec, self.spec) tty.msg("Deactivated extension %s for %s" % @@ -1615,7 +1616,7 @@ class PackageBase(object): """ def ignore(filename): - return (filename in spack.install_layout.hidden_file_paths or + return (filename in spack.install_area.layout.hidden_file_paths or kwargs.get('ignore', lambda f: False)(filename)) tree = LinkTree(extension.prefix) @@ -1716,7 +1717,7 @@ def flatten_dependencies(spec, flat_dir): for dep in spec.traverse(root=False): name = dep.name - dep_path = spack.install_layout.path_for_spec(dep) + dep_path = spack.install_area.layout.path_for_spec(dep) dep_files = LinkTree(dep_path) os.mkdir(flat_dir + '/' + name) @@ -1745,7 +1746,7 @@ def dump_packages(spec, path): if node is not spec: # Locate the dependency package in the install tree and find # its provenance information. - source = spack.install_layout.build_packages_path(node) + source = spack.install_area.layout.build_packages_path(node) source_repo_root = join_path(source, node.namespace) # There's no provenance installed for the source package. Skip it. diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 1d4cddaa0f..4696d1d5bc 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -42,7 +42,6 @@ from llnl.util.filesystem import * import spack import spack.error -import spack.config import spack.spec from spack.provider_index import ProviderIndex from spack.util.naming import * @@ -128,6 +127,7 @@ class RepoPath(object): # If repo_dirs is empty, just use the configuration if not repo_dirs: + import spack.config repo_dirs = spack.config.get_config('repos') if not repo_dirs: raise NoRepoConfiguredError( diff --git a/lib/spack/spack/schema/install.py b/lib/spack/spack/schema/install.py new file mode 100644 index 0000000000..2c43ad1adf --- /dev/null +++ b/lib/spack/spack/schema/install.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Schema for packages.yaml configuration files.""" + +schema = { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack module file configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + 'install': { + 'type': 'object', + 'default' : {}, + 'additionalProperties': False, + 'properties': { + 'path' : { 'type': 'string' }, + 'layout' : { 'type': 'string' } + } + }, + }, +} diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index b1e3e2ed67..1d9fd8d97a 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -111,6 +111,7 @@ from llnl.util.tty.color import * import spack import spack.architecture +import spack.install_area import spack.compilers as compilers import spack.error import spack.parse @@ -964,7 +965,7 @@ class Spec(object): @property def prefix(self): - return Prefix(spack.install_layout.path_for_spec(self)) + return Prefix(spack.install_area.layout.path_for_spec(self)) def dag_hash(self, length=None): """Return a hash of the entire spec DAG, including connectivity.""" @@ -2328,7 +2329,7 @@ class Spec(object): elif named_str == 'SPACK_ROOT': out.write(fmt % spack.prefix) elif named_str == 'SPACK_INSTALL': - out.write(fmt % spack.install_path) + out.write(fmt % spack.install_area.path) elif named_str == 'PREFIX': out.write(fmt % self.prefix) elif named_str.startswith('HASH'): @@ -2532,7 +2533,7 @@ class SpecParser(spack.parse.Parser): def spec_by_hash(self): self.expect(ID) - specs = spack.installed_db.query() + specs = spack.install_area.db.query() matches = [spec for spec in specs if spec.dag_hash()[:len(self.token.value)] == self.token.value] diff --git a/lib/spack/spack/test/cmd/uninstall.py b/lib/spack/spack/test/cmd/uninstall.py index 4ccb9ddbf4..d839c00b62 100644 --- a/lib/spack/spack/test/cmd/uninstall.py +++ b/lib/spack/spack/test/cmd/uninstall.py @@ -23,7 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import spack.test.mock_database - +import spack.install_area from spack.cmd.uninstall import uninstall @@ -51,7 +51,7 @@ class TestUninstall(spack.test.mock_database.MockDatabase): args = MockArgs(['callpath'], all=True, dependents=True) uninstall(parser, args) - all_specs = spack.install_layout.all_specs() + all_specs = spack.install_area.layout.all_specs() self.assertEqual(len(all_specs), 7) # query specs with multiple configurations mpileaks_specs = [s for s in all_specs if s.satisfies('mpileaks')] diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 4395f17f97..c0bb12ce10 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -30,6 +30,7 @@ import multiprocessing import os.path import spack +import spack.install_area from llnl.util.filesystem import join_path from llnl.util.tty.colify import colify from spack.test.mock_database import MockDatabase @@ -40,16 +41,16 @@ def _print_ref_counts(): recs = [] def add_rec(spec): - cspecs = spack.installed_db.query(spec, installed=any) + cspecs = spack.install_area.db.query(spec, installed=any) if not cspecs: recs.append("[ %-7s ] %-20s-" % ('', spec)) else: key = cspecs[0].dag_hash() - rec = spack.installed_db.get_record(cspecs[0]) + rec = spack.install_area.db.get_record(cspecs[0]) recs.append("[ %-7s ] %-20s%d" % (key[:7], spec, rec.ref_count)) - with spack.installed_db.read_transaction(): + with spack.install_area.db.read_transaction(): add_rec('mpileaks ^mpich') add_rec('callpath ^mpich') add_rec('mpich') @@ -82,7 +83,7 @@ class DatabaseTest(MockDatabase): def test_010_all_install_sanity(self): """Ensure that the install layout reflects what we think it does.""" - all_specs = spack.install_layout.all_specs() + all_specs = spack.install_area.layout.all_specs() self.assertEqual(len(all_specs), 13) # query specs with multiple configurations @@ -113,12 +114,12 @@ class DatabaseTest(MockDatabase): def test_015_write_and_read(self): # write and read DB - with spack.installed_db.write_transaction(): - specs = spack.installed_db.query() - recs = [spack.installed_db.get_record(s) for s in specs] + with spack.install_area.db.write_transaction(): + specs = spack.install_area.db.query() + recs = [spack.install_area.db.get_record(s) for s in specs] for spec, rec in zip(specs, recs): - new_rec = spack.installed_db.get_record(spec) + new_rec = spack.install_area.db.get_record(spec) self.assertEqual(new_rec.ref_count, rec.ref_count) self.assertEqual(new_rec.spec, rec.spec) self.assertEqual(new_rec.path, rec.path) @@ -126,7 +127,7 @@ class DatabaseTest(MockDatabase): def _check_merkleiness(self): """Ensure the spack database is a valid merkle graph.""" - all_specs = spack.installed_db.query(installed=any) + all_specs = spack.install_area.db.query(installed=any) seen = {} for spec in all_specs: @@ -139,8 +140,8 @@ class DatabaseTest(MockDatabase): def _check_db_sanity(self): """Utiilty function to check db against install layout.""" - expected = sorted(spack.install_layout.all_specs()) - actual = sorted(self.installed_db.query()) + expected = sorted(spack.install_area.layout.all_specs()) + actual = sorted(self.install_db.query()) self.assertEqual(len(expected), len(actual)) for e, a in zip(expected, actual): @@ -154,13 +155,13 @@ class DatabaseTest(MockDatabase): def test_025_reindex(self): """Make sure reindex works and ref counts are valid.""" - spack.installed_db.reindex(spack.install_layout) + spack.install_area.db.reindex(spack.install_area.layout) self._check_db_sanity() def test_030_db_sanity_from_another_process(self): def read_and_modify(): self._check_db_sanity() # check that other process can read DB - with self.installed_db.write_transaction(): + with self.install_db.write_transaction(): self._mock_remove('mpileaks ^zmpi') p = multiprocessing.Process(target=read_and_modify, args=()) @@ -168,40 +169,40 @@ class DatabaseTest(MockDatabase): p.join() # ensure child process change is visible in parent process - with self.installed_db.read_transaction(): - self.assertEqual(len(self.installed_db.query('mpileaks ^zmpi')), 0) + with self.install_db.read_transaction(): + self.assertEqual(len(self.install_db.query('mpileaks ^zmpi')), 0) def test_040_ref_counts(self): """Ensure that we got ref counts right when we read the DB.""" - self.installed_db._check_ref_counts() + self.install_db._check_ref_counts() def test_050_basic_query(self): """Ensure querying database is consistent with what is installed.""" # query everything - self.assertEqual(len(spack.installed_db.query()), 13) + self.assertEqual(len(spack.install_area.db.query()), 13) # query specs with multiple configurations - mpileaks_specs = self.installed_db.query('mpileaks') - callpath_specs = self.installed_db.query('callpath') - mpi_specs = self.installed_db.query('mpi') + mpileaks_specs = self.install_db.query('mpileaks') + callpath_specs = self.install_db.query('callpath') + mpi_specs = self.install_db.query('mpi') self.assertEqual(len(mpileaks_specs), 3) self.assertEqual(len(callpath_specs), 3) self.assertEqual(len(mpi_specs), 3) # query specs with single configurations - dyninst_specs = self.installed_db.query('dyninst') - libdwarf_specs = self.installed_db.query('libdwarf') - libelf_specs = self.installed_db.query('libelf') + dyninst_specs = self.install_db.query('dyninst') + libdwarf_specs = self.install_db.query('libdwarf') + libelf_specs = self.install_db.query('libelf') self.assertEqual(len(dyninst_specs), 1) self.assertEqual(len(libdwarf_specs), 1) self.assertEqual(len(libelf_specs), 1) # Query by dependency - self.assertEqual(len(self.installed_db.query('mpileaks ^mpich')), 1) - self.assertEqual(len(self.installed_db.query('mpileaks ^mpich2')), 1) - self.assertEqual(len(self.installed_db.query('mpileaks ^zmpi')), 1) + self.assertEqual(len(self.install_db.query('mpileaks ^mpich')), 1) + self.assertEqual(len(self.install_db.query('mpileaks ^mpich2')), 1) + self.assertEqual(len(self.install_db.query('mpileaks ^zmpi')), 1) def _check_remove_and_add_package(self, spec): """Remove a spec from the DB, then add it and make sure everything's @@ -209,13 +210,13 @@ class DatabaseTest(MockDatabase): removed, that it's back when added again, and that ref counts are consistent. """ - original = self.installed_db.query() - self.installed_db._check_ref_counts() + original = self.install_db.query() + self.install_db._check_ref_counts() # Remove spec - concrete_spec = self.installed_db.remove(spec) - self.installed_db._check_ref_counts() - remaining = self.installed_db.query() + concrete_spec = self.install_db.remove(spec) + self.install_db._check_ref_counts() + remaining = self.install_db.query() # ensure spec we removed is gone self.assertEqual(len(original) - 1, len(remaining)) @@ -223,14 +224,14 @@ class DatabaseTest(MockDatabase): self.assertTrue(concrete_spec not in remaining) # add it back and make sure everything is ok. - self.installed_db.add(concrete_spec, spack.install_layout) - installed = self.installed_db.query() + self.install_db.add(concrete_spec, spack.install_area.layout) + installed = self.install_db.query() self.assertTrue(concrete_spec in installed) self.assertEqual(installed, original) # sanity check against direcory layout and check ref counts. self._check_db_sanity() - self.installed_db._check_ref_counts() + self.install_db._check_ref_counts() def test_060_remove_and_add_root_package(self): self._check_remove_and_add_package('mpileaks ^mpich') @@ -239,95 +240,95 @@ class DatabaseTest(MockDatabase): self._check_remove_and_add_package('dyninst') def test_080_root_ref_counts(self): - rec = self.installed_db.get_record('mpileaks ^mpich') + rec = self.install_db.get_record('mpileaks ^mpich') # Remove a top-level spec from the DB - self.installed_db.remove('mpileaks ^mpich') + self.install_db.remove('mpileaks ^mpich') # record no longer in DB self.assertEqual( - self.installed_db.query('mpileaks ^mpich', installed=any), []) + self.install_db.query('mpileaks ^mpich', installed=any), []) # record's deps have updated ref_counts self.assertEqual( - self.installed_db.get_record('callpath ^mpich').ref_count, 0) - self.assertEqual(self.installed_db.get_record('mpich').ref_count, 1) + self.install_db.get_record('callpath ^mpich').ref_count, 0) + self.assertEqual(self.install_db.get_record('mpich').ref_count, 1) # Put the spec back - self.installed_db.add(rec.spec, spack.install_layout) + self.install_db.add(rec.spec, spack.install_area.layout) # record is present again self.assertEqual( - len(self.installed_db.query('mpileaks ^mpich', installed=any)), 1) + len(self.install_db.query('mpileaks ^mpich', installed=any)), 1) # dependencies have ref counts updated self.assertEqual( - self.installed_db.get_record('callpath ^mpich').ref_count, 1) - self.assertEqual(self.installed_db.get_record('mpich').ref_count, 2) + self.install_db.get_record('callpath ^mpich').ref_count, 1) + self.assertEqual(self.install_db.get_record('mpich').ref_count, 2) def test_090_non_root_ref_counts(self): - self.installed_db.get_record('mpileaks ^mpich') - self.installed_db.get_record('callpath ^mpich') + self.install_db.get_record('mpileaks ^mpich') + self.install_db.get_record('callpath ^mpich') # "force remove" a non-root spec from the DB - self.installed_db.remove('callpath ^mpich') + self.install_db.remove('callpath ^mpich') # record still in DB but marked uninstalled self.assertEqual( - self.installed_db.query('callpath ^mpich', installed=True), []) + self.install_db.query('callpath ^mpich', installed=True), []) self.assertEqual( - len(self.installed_db.query('callpath ^mpich', installed=any)), 1) + len(self.install_db.query('callpath ^mpich', installed=any)), 1) # record and its deps have same ref_counts - self.assertEqual(self.installed_db.get_record( + self.assertEqual(self.install_db.get_record( 'callpath ^mpich', installed=any).ref_count, 1) - self.assertEqual(self.installed_db.get_record('mpich').ref_count, 2) + self.assertEqual(self.install_db.get_record('mpich').ref_count, 2) # remove only dependent of uninstalled callpath record - self.installed_db.remove('mpileaks ^mpich') + self.install_db.remove('mpileaks ^mpich') # record and parent are completely gone. self.assertEqual( - self.installed_db.query('mpileaks ^mpich', installed=any), []) + self.install_db.query('mpileaks ^mpich', installed=any), []) self.assertEqual( - self.installed_db.query('callpath ^mpich', installed=any), []) + self.install_db.query('callpath ^mpich', installed=any), []) # mpich ref count updated properly. - mpich_rec = self.installed_db.get_record('mpich') + mpich_rec = self.install_db.get_record('mpich') self.assertEqual(mpich_rec.ref_count, 0) def test_100_no_write_with_exception_on_remove(self): def fail_while_writing(): - with self.installed_db.write_transaction(): + with self.install_db.write_transaction(): self._mock_remove('mpileaks ^zmpi') raise Exception() - with self.installed_db.read_transaction(): + with self.install_db.read_transaction(): self.assertEqual( - len(self.installed_db.query('mpileaks ^zmpi', installed=any)), + len(self.install_db.query('mpileaks ^zmpi', installed=any)), 1) self.assertRaises(Exception, fail_while_writing) # reload DB and make sure zmpi is still there. - with self.installed_db.read_transaction(): + with self.install_db.read_transaction(): self.assertEqual( - len(self.installed_db.query('mpileaks ^zmpi', installed=any)), + len(self.install_db.query('mpileaks ^zmpi', installed=any)), 1) def test_110_no_write_with_exception_on_install(self): def fail_while_writing(): - with self.installed_db.write_transaction(): + with self.install_db.write_transaction(): self._mock_install('cmake') raise Exception() - with self.installed_db.read_transaction(): + with self.install_db.read_transaction(): self.assertEqual( - self.installed_db.query('cmake', installed=any), []) + self.install_db.query('cmake', installed=any), []) self.assertRaises(Exception, fail_while_writing) # reload DB and make sure cmake was not written. - with self.installed_db.read_transaction(): + with self.install_db.read_transaction(): self.assertEqual( - self.installed_db.query('cmake', installed=any), []) + self.install_db.query('cmake', installed=any), []) diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 232d5aeeaf..eb051e9301 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -26,6 +26,7 @@ import shutil import tempfile import spack +import spack.install_area from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout from spack.database import Database @@ -49,11 +50,11 @@ class InstallTest(MockPackagesTest): # Use a fake install directory to avoid conflicts bt/w # installed pkgs and mock packages. self.tmpdir = tempfile.mkdtemp() - self.orig_layout = spack.install_layout - self.orig_db = spack.installed_db + self.orig_layout = spack.install_area.layout + self.orig_db = spack.install_area.db - spack.install_layout = YamlDirectoryLayout(self.tmpdir) - spack.installed_db = Database(self.tmpdir) + spack.install_area.layout = YamlDirectoryLayout(self.tmpdir) + spack.install_area.db = Database(self.tmpdir) def tearDown(self): super(InstallTest, self).tearDown() @@ -63,8 +64,8 @@ class InstallTest(MockPackagesTest): spack.do_checksum = True # restore spack's layout. - spack.install_layout = self.orig_layout - spack.installed_db = self.orig_db + spack.install_area.layout = self.orig_layout + spack.install_area.db = self.orig_db shutil.rmtree(self.tmpdir, ignore_errors=True) def fake_fetchify(self, pkg): @@ -91,7 +92,7 @@ class InstallTest(MockPackagesTest): pkg.remove_prefix() raise - def test_install_environment(self): + def test_install_area(self): spec = Spec('cmake-client').concretized() for s in spec.traverse(): diff --git a/lib/spack/spack/test/mock_database.py b/lib/spack/spack/test/mock_database.py index d5867f06ec..d1f9fb000b 100644 --- a/lib/spack/spack/test/mock_database.py +++ b/lib/spack/spack/test/mock_database.py @@ -26,6 +26,7 @@ import shutil import tempfile import spack +import spack.install_area from spack.spec import Spec from spack.database import Database from spack.directory_layout import YamlDirectoryLayout @@ -41,7 +42,7 @@ class MockDatabase(MockPackagesTest): pkg.do_install(fake=True) def _mock_remove(self, spec): - specs = spack.installed_db.query(spec) + specs = spack.install_area.db.query(spec) assert len(specs) == 1 spec = specs[0] spec.package.do_uninstall(spec) @@ -54,17 +55,17 @@ class MockDatabase(MockPackagesTest): # Make a fake install directory self.install_path = tempfile.mkdtemp() - self.spack_install_path = spack.install_path - spack.install_path = self.install_path + self.spack_install_path = spack.install_area.path + spack.install_area.path = self.install_path self.install_layout = YamlDirectoryLayout(self.install_path) - self.spack_install_layout = spack.install_layout - spack.install_layout = self.install_layout + self.spack_install_layout = spack.install_area.layout + spack.install_area.layout = self.install_layout # Make fake database and fake install directory. - self.installed_db = Database(self.install_path) - self.spack_installed_db = spack.installed_db - spack.installed_db = self.installed_db + self.install_db = Database(self.install_path) + self.spack_install_db = spack.install_area.db + spack.install_area.db = self.install_db # make a mock database with some packages installed note that # the ref count for dyninst here will be 3, as it's recycled @@ -90,18 +91,18 @@ class MockDatabase(MockPackagesTest): # # Transaction used to avoid repeated writes. - with spack.installed_db.write_transaction(): + with spack.install_area.db.write_transaction(): self._mock_install('mpileaks ^mpich') self._mock_install('mpileaks ^mpich2') self._mock_install('mpileaks ^zmpi') def tearDown(self): - with spack.installed_db.write_transaction(): - for spec in spack.installed_db.query(): + with spack.install_area.db.write_transaction(): + for spec in spack.install_area.db.query(): spec.package.do_uninstall(spec) super(MockDatabase, self).tearDown() shutil.rmtree(self.install_path) - spack.install_path = self.spack_install_path - spack.install_layout = self.spack_install_layout - spack.installed_db = self.spack_installed_db + spack.install_area.path = self.spack_install_path + spack.install_area.layout = self.spack_install_layout + spack.install_area.db = self.spack_install_db -- cgit v1.2.3-60-g2f50 From 9347f86939f3dea9d8d8ea723c238cd4f8c518a6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 27 Oct 2016 10:57:27 -0700 Subject: Rename install.yaml -> config.yaml, install_area -> "store" - Added a schema for config.yaml - Moved install tree configuration to config.yaml - Moved etc/spack/install.yaml to etc/spack/defaults/config.yaml - renamed install_area to "store", to use a term in common with guix/nix. - in `config.yaml` file, it's called the `install_tree` to be more intuitive to users. - `install_tree` might've worked in the code, but `install_tree` is already a global function in the spack namespace, from llnl.util.filesystem. --- etc/spack/defaults/config.yaml | 64 ++++++++++++++++++++++++++++ etc/spack/install.yaml | 8 ---- lib/spack/docs/configuration.rst | 4 +- lib/spack/spack/__init__.py | 16 ------- lib/spack/spack/build_environment.py | 4 +- lib/spack/spack/cmd/__init__.py | 4 +- lib/spack/spack/cmd/common/arguments.py | 4 +- lib/spack/spack/cmd/deactivate.py | 4 +- lib/spack/spack/cmd/extensions.py | 6 +-- lib/spack/spack/cmd/find.py | 6 +-- lib/spack/spack/cmd/reindex.py | 4 +- lib/spack/spack/cmd/uninstall.py | 4 +- lib/spack/spack/config.py | 2 +- lib/spack/spack/database.py | 6 +-- lib/spack/spack/install_area.py | 36 ---------------- lib/spack/spack/package.py | 70 +++++++++++++++--------------- lib/spack/spack/schema/config.py | 62 +++++++++++++++++++++++++++ lib/spack/spack/schema/install.py | 43 ------------------- lib/spack/spack/spec.py | 8 ++-- lib/spack/spack/store.py | 75 +++++++++++++++++++++++++++++++++ lib/spack/spack/test/cmd/uninstall.py | 4 +- lib/spack/spack/test/database.py | 30 ++++++------- lib/spack/spack/test/install.py | 16 +++---- lib/spack/spack/test/mock_database.py | 28 ++++++------ 24 files changed, 305 insertions(+), 203 deletions(-) create mode 100644 etc/spack/defaults/config.yaml delete mode 100644 etc/spack/install.yaml delete mode 100644 lib/spack/spack/install_area.py create mode 100644 lib/spack/spack/schema/config.py delete mode 100644 lib/spack/spack/schema/install.py create mode 100644 lib/spack/spack/store.py (limited to 'etc') diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml new file mode 100644 index 0000000000..5ce60049ce --- /dev/null +++ b/etc/spack/defaults/config.yaml @@ -0,0 +1,64 @@ +# ------------------------------------------------------------------------- +# This is the default spack configuration file. +# +# Settings here are versioned with Spack and are intended to provide +# sensible defaults out of the box. Spack maintainers should edit this +# file to keep it current. +# +# Users can override these settings by editing the following files. +# +# Per-spack-instance settings (overrides defaults): +# $SPACK_ROOT/etc/spack/config.yaml +# +# Per-user settings (overrides default and site settings): +# ~/.spack/config.yaml +# ------------------------------------------------------------------------- +config: + + # Precedence of configuration scopes, high to low. + # The user can override this paradoxically. + # scopes: [user, spack, default] + + + # This is the path to the root of the Spack install tree. + # You can use $spack here to refer to the root of the spack instance. + install_tree: $spack/opt/spack + + + # Temporary locations Spack can try to use for builds. + # + # Spack will use the first one it finds that exists and is writable. + # You can use $tempdir to refer to the system default temp directory + # (as returned by tempfile.gettempdir()). + # + # A value of $local indicates that Spack should run builds directly + # inside its install directory without staging them in temporary space. + build_stage: + - /usr/workspace/*/%u + - $tempdir + - /nfs/tmp2/%u + - $local + + + # Cache directory already downloaded source tarballs and archived + # repositories. This can be purged with spack purge + source_cache: $spack/var/spack/cache + + + # Cache directory for miscellaneous files, like the package index. + misc_cache: ~/.spack/cache + + + # If this is false, tools like curl that use SSL will not verify + # certifiates. (e.g., curl will use use the -k option) + verify_ssl: true + + + # If set to true, Spack will always check checksums after downloading + # archives. If false, Spack skips the checksum step. + checksum: true + + + # If set to true, `spack install` and friends will NOT clean + # potentially harmful variables from the build environment. Use wisely. + dirty: false diff --git a/etc/spack/install.yaml b/etc/spack/install.yaml deleted file mode 100644 index 553d09f13e..0000000000 --- a/etc/spack/install.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# ------------------------------------------------------------------------- -# This is the default spack install setup configuration. -# -# Changes to this file will affect all users of this spack install -# ------------------------------------------------------------------------- -install: - path: $spack/opt/spack - layout : YamlDirectoryLayout \ No newline at end of file diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index c797e7e948..1244a27d2f 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -10,8 +10,8 @@ Install options ---------------------------- By default, Spack will install software into ``opt/spack``. -To set a custom install directory, the option ``path`` in -``install.yaml`` can be used. This file can be found +To set a custom install directory, the option ``install_tree`` in +``config.yaml`` can be used. This file can be found in a Spack installation's ``etc/spack/`` or a user's ``~/.spack/`` directory. diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 8a0733871f..ab03c0c848 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -145,22 +145,6 @@ for path in _tmp_candidates: # for. do_checksum = True -# -# SYS_TYPE to use for the spack installation. -# Value of this determines what platform spack thinks it is by -# default. You can assign three types of values: -# 1. None -# Spack will try to determine the sys_type automatically. -# -# 2. A string -# Spack will assume that the sys_type is hardcoded to the value. -# -# 3. A function that returns a string: -# Spack will use this function to determine the sys_type. -# -sys_type = None - - # # When packages call 'from spack import *', this extra stuff is brought in. # diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 942b525bf6..f626bab1af 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -58,7 +58,7 @@ import sys import llnl.util.tty as tty import spack -import spack.install_area +import spack.store from llnl.util.filesystem import * from spack.environment import EnvironmentModifications, validate from spack.util.environment import * @@ -277,7 +277,7 @@ def set_build_environment_variables(pkg, env, dirty=False): env.set(SPACK_PREFIX, pkg.prefix) # Install root prefix - env.set(SPACK_INSTALL, spack.install_area.path) + env.set(SPACK_INSTALL, spack.store.root) # Stuff in here sanitizes the build environemnt to eliminate # anything the user has set that may interfere. diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 0da542e930..8000c9f1f2 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -34,7 +34,7 @@ from llnl.util.tty.color import * import spack import spack.config import spack.spec -import spack.install_area +import spack.store # # Settings for commands that modify configuration @@ -137,7 +137,7 @@ def elide_list(line_list, max_num=10): def disambiguate_spec(spec): - matching_specs = spack.install_area.db.query(spec) + matching_specs = spack.store.db.query(spec) if not matching_specs: tty.die("Spec '%s' matches no installed packages." % spec) diff --git a/lib/spack/spack/cmd/common/arguments.py b/lib/spack/spack/cmd/common/arguments.py index d7d2f9770c..5ab54bdb61 100644 --- a/lib/spack/spack/cmd/common/arguments.py +++ b/lib/spack/spack/cmd/common/arguments.py @@ -25,7 +25,7 @@ import argparse -import spack.install_area +import spack.store import spack.modules from spack.util.pattern import Bunch __all__ = ['add_common_arguments'] @@ -54,7 +54,7 @@ class ConstraintAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): # Query specs from command line d = self.qualifiers.get(namespace.subparser_name, {}) - specs = [s for s in spack.install_area.db.query(**d)] + specs = [s for s in spack.store.db.query(**d)] values = ' '.join(values) if values: specs = [x for x in specs if x.satisfies(values, strict=True)] diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index 859025034a..fedd078972 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -27,7 +27,7 @@ import llnl.util.tty as tty import spack import spack.cmd -import spack.install_area +import spack.store from spack.graph import topological_sort description = "Deactivate a package extension." @@ -57,7 +57,7 @@ def deactivate(parser, args): if args.all: if pkg.extendable: tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec) - ext_pkgs = spack.install_area.db.installed_extensions_for(spec) + ext_pkgs = spack.store.db.installed_extensions_for(spec) for ext_pkg in ext_pkgs: ext_pkg.spec.normalize() diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index 49572263c5..bd149044ca 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -30,7 +30,7 @@ from llnl.util.tty.colify import colify import spack import spack.cmd import spack.cmd.find -import spack.install_area +import spack.store description = "List extensions for package." @@ -88,7 +88,7 @@ def extensions(parser, args): # List specs of installed extensions. # installed = [s.spec - for s in spack.install_area.db.installed_extensions_for(spec)] + for s in spack.store.db.installed_extensions_for(spec)] print if not installed: @@ -100,7 +100,7 @@ def extensions(parser, args): # # List specs of activated extensions. # - activated = spack.install_area.layout.extension_map(spec) + activated = spack.store.layout.extension_map(spec) print if not activated: tty.msg("None activated.") diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 9e631d6e24..50e6112486 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -28,7 +28,7 @@ import sys import llnl.util.tty as tty import spack import spack.spec -import spack.install_area +import spack.store from llnl.util.lang import * from llnl.util.tty.colify import * from llnl.util.tty.color import * @@ -146,9 +146,9 @@ def find(parser, args): # Get all the specs the user asked for if not query_specs: - specs = set(spack.install_area.db.query(**q_args)) + specs = set(spack.store.db.query(**q_args)) else: - results = [set(spack.install_area.db.query(qs, **q_args)) + results = [set(spack.store.db.query(qs, **q_args)) for qs in query_specs] specs = set.union(*results) diff --git a/lib/spack/spack/cmd/reindex.py b/lib/spack/spack/cmd/reindex.py index 10c5ea4c05..7dddda2ffb 100644 --- a/lib/spack/spack/cmd/reindex.py +++ b/lib/spack/spack/cmd/reindex.py @@ -23,9 +23,9 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import spack -import spack.install_area +import spack.store description = "Rebuild Spack's package database." def reindex(parser, args): - spack.install_area.db.reindex(spack.install_area.layout) + spack.store.db.reindex(spack.store.layout) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 26d6080ffb..f48b28ff73 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -29,7 +29,7 @@ import argparse import llnl.util.tty as tty import spack import spack.cmd -import spack.install_area +import spack.store import spack.repository description = "Remove an installed package" @@ -90,7 +90,7 @@ def concretize_specs(specs, allow_multiple_matches=False, force=False): specs_from_cli = [] has_errors = False for spec in specs: - matching = spack.install_area.db.query(spec) + matching = spack.store.db.query(spec) # For each spec provided, make sure it refers to only one package. # Fail and ask user to be unambiguous if it doesn't if not allow_multiple_matches and len(matching) > 1: diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 89e02e33df..1bd1689150 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -180,7 +180,7 @@ section_schemas = { 'repos': spack.schema.repos.schema, 'packages': spack.schema.packages.schema, 'modules': spack.schema.modules.schema, - 'install': spack.schema.install.schema, + 'config': spack.schema.config.schema, } """OrderedDict of config scopes keyed by name. diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index e0db8c3167..e8902ec024 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -48,7 +48,7 @@ import llnl.util.tty as tty from llnl.util.filesystem import * from llnl.util.lock import * -import spack.install_area +import spack.store import spack.repository from spack.directory_layout import DirectoryLayoutError from spack.version import Version @@ -280,7 +280,7 @@ class Database(object): if version > _db_version: raise InvalidDatabaseVersionError(_db_version, version) elif version < _db_version: - self.reindex(spack.install_area.layout) + self.reindex(spack.store.layout) installs = dict((k, v.to_dict()) for k, v in self._data.items()) def invalid_record(hash_key, error): @@ -446,7 +446,7 @@ class Database(object): else: # The file doesn't exist, try to traverse the directory. # reindex() takes its own write lock, so no lock here. - self.reindex(spack.install_area.layout) + self.reindex(spack.store.layout) def _add(self, spec, directory_layout=None, explicit=False): """Add an install record for this spec to the database. diff --git a/lib/spack/spack/install_area.py b/lib/spack/spack/install_area.py deleted file mode 100644 index b99df4c570..0000000000 --- a/lib/spack/spack/install_area.py +++ /dev/null @@ -1,36 +0,0 @@ -__author__ = "Benedikt Hegner (CERN)" - -import re - -import llnl.util.tty as tty -from llnl.util.filesystem import * - -# -# Read in the config -# -import spack.config -config = spack.config.get_config("install") - -# -# Set up the install path -# -path = re.sub(r'^\$spack', spack.prefix, config['path']) - -# -# Set up the installed packages database -# -from spack.database import Database -db = Database(path) - -# -# This controls how spack lays out install prefixes and -# stage directories. -# -import spack.directory_layout - -try: - layout_name = config["layout"] - layout_class = getattr(spack.directory_layout,layout_name) - layout = layout_class(path) -except: - tty.die("Invalid install directory layout %s chosen." %layout_name) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index d9145508e3..6bc3362639 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -47,7 +47,7 @@ from StringIO import StringIO import llnl.util.lock import llnl.util.tty as tty import spack -import spack.install_area +import spack.store import spack.compilers import spack.directives import spack.error @@ -832,7 +832,7 @@ class PackageBase(object): if not self.is_extension: raise ValueError( "is_extension called on package that is not an extension.") - exts = spack.install_area.layout.extension_map(self.extendee_spec) + exts = spack.store.layout.extension_map(self.extendee_spec) return (self.name in exts) and (exts[self.name] == self.spec) def provides(self, vpkg_name): @@ -853,7 +853,7 @@ class PackageBase(object): TODO: move this method to database.py? """ dependents = [] - for spec in spack.install_area.db.query(): + for spec in spack.store.db.query(): if self.name == spec.name: continue # XXX(deptype): Should build dependencies not count here? @@ -867,7 +867,7 @@ class PackageBase(object): def prefix_lock(self): """Prefix lock is a byte range lock on the nth byte of a file. - The lock file is ``spack.install_area.db.prefix_lock`` -- the DB + The lock file is ``spack.store.db.prefix_lock`` -- the DB tells us what to call it and it lives alongside the install DB. n is the sys.maxsize-bit prefix of the DAG hash. This makes @@ -879,7 +879,7 @@ class PackageBase(object): prefix = self.spec.prefix if prefix not in Package.prefix_locks: Package.prefix_locks[prefix] = llnl.util.lock.Lock( - spack.install_area.db.prefix_lock_path, + spack.store.db.prefix_lock_path, self.spec.dag_hash_bit_prefix(bit_length(sys.maxsize)), 1) self._prefix_lock = Package.prefix_locks[prefix] @@ -916,7 +916,7 @@ class PackageBase(object): Removes the prefix for a package along with any empty parent directories """ - spack.install_area.layout.remove_install_directory(self.spec) + spack.store.layout.remove_install_directory(self.spec) def do_fetch(self, mirror_only=False): """ @@ -1153,15 +1153,15 @@ class PackageBase(object): return # Ensure package is not already installed - layout = spack.install_area.layout + layout = spack.store.layout with self._prefix_read_lock(): if layout.check_installed(self.spec): tty.msg( "%s is already installed in %s" % (self.name, self.prefix)) - rec = spack.install_area.db.get_record(self.spec) + rec = spack.store.db.get_record(self.spec) if (not rec.explicit) and explicit: - with spack.install_area.db.write_transaction(): - rec = spack.install_area.db.get_record(self.spec) + with spack.store.db.write_transaction(): + rec = spack.store.db.get_record(self.spec) rec.explicit = True return @@ -1266,15 +1266,15 @@ class PackageBase(object): try: # Create the install prefix and fork the build process. - spack.install_area.layout.create_install_directory(self.spec) + spack.store.layout.create_install_directory(self.spec) # Fork a child to do the actual installation spack.build_environment.fork(self, build_process, dirty=dirty) # If we installed then we should keep the prefix keep_prefix = True if self.last_phase is None else keep_prefix # note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. - spack.install_area.db.add( - self.spec, spack.install_area.layout, explicit=explicit + spack.store.db.add( + self.spec, spack.store.layout, explicit=explicit ) except directory_layout.InstallDirectoryAlreadyExistsError: # Abort install if install directory exists. @@ -1303,15 +1303,16 @@ class PackageBase(object): """ self.last_phase = kwargs.pop('stop_at', None) if self.last_phase is not None and self.last_phase not in self.phases: - tty.die('\'{0.last_phase}\' is not among the allowed phases for package {0.name}'.format(self)) # NOQA: ignore=E501 + tty.die('\'{0}\' is not an allowed phase for package {1}' + .format(self.last_phase, self.name)) def log(self): # Copy provenance into the install directory on success - log_install_path = spack.install_area.layout.build_log_path( + log_install_path = spack.store.layout.build_log_path( self.spec) - env_install_path = spack.install_area.layout.build_env_path( + env_install_path = spack.store.layout.build_env_path( self.spec) - packages_dir = spack.install_area.layout.build_packages_path( + packages_dir = spack.store.layout.build_packages_path( self.spec) # Remove first if we're overwriting another build @@ -1345,7 +1346,8 @@ class PackageBase(object): check_paths(self.sanity_check_is_dir, 'directory', os.path.isdir) installed = set(os.listdir(self.prefix)) - installed.difference_update(spack.install_area.layout.hidden_file_paths) + installed.difference_update( + spack.store.layout.hidden_file_paths) if not installed: raise InstallError( "Install failed for %s. Nothing was installed!" % self.name) @@ -1353,7 +1355,7 @@ class PackageBase(object): @property def build_log_path(self): if self.installed: - return spack.install_area.layout.build_log_path(self.spec) + return spack.store.layout.build_log_path(self.spec) else: return join_path(self.stage.source_path, 'spack-build.out') @@ -1483,9 +1485,9 @@ class PackageBase(object): if not self.installed: # prefix may not exist, but DB may be inconsistent. Try to fix by # removing, but omit hooks. - specs = spack.install_area.db.query(self.spec, installed=True) + specs = spack.store.db.query(self.spec, installed=True) if specs: - spack.install_area.db.remove(specs[0]) + spack.store.db.remove(specs[0]) tty.msg("Removed stale DB entry for %s" % self.spec.short_spec) return else: @@ -1502,7 +1504,7 @@ class PackageBase(object): # Uninstalling in Spack only requires removing the prefix. self.remove_prefix() # - spack.install_area.db.remove(self.spec) + spack.store.db.remove(self.spec) tty.msg("Successfully uninstalled %s" % self.spec.short_spec) # Once everything else is done, run post install hooks @@ -1536,8 +1538,8 @@ class PackageBase(object): """ self._sanity_check_extension() - spack.install_area.layout.check_extension_conflict(self.extendee_spec, - self.spec) + spack.store.layout.check_extension_conflict( + self.extendee_spec, self.spec) # Activate any package dependencies that are also extensions. if not force: @@ -1548,7 +1550,7 @@ class PackageBase(object): self.extendee_spec.package.activate(self, **self.extendee_args) - spack.install_area.layout.add_extension(self.extendee_spec, self.spec) + spack.store.layout.add_extension(self.extendee_spec, self.spec) tty.msg("Activated extension %s for %s" % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) @@ -1563,7 +1565,7 @@ class PackageBase(object): """ def ignore(filename): - return (filename in spack.install_area.layout.hidden_file_paths or + return (filename in spack.store.layout.hidden_file_paths or kwargs.get('ignore', lambda f: False)(filename)) tree = LinkTree(extension.prefix) @@ -1581,9 +1583,11 @@ class PackageBase(object): # Allow a force deactivate to happen. This can unlink # spurious files if something was corrupted. if not force: - spack.install_area.layout.check_activated(self.extendee_spec, self.spec) + spack.store.layout.check_activated( + self.extendee_spec, self.spec) - activated = spack.install_area.layout.extension_map(self.extendee_spec) + activated = spack.store.layout.extension_map( + self.extendee_spec) for name, aspec in activated.items(): if aspec == self.spec: continue @@ -1599,8 +1603,8 @@ class PackageBase(object): # redundant activation check -- makes SURE the spec is not # still activated even if something was wrong above. if self.activated: - spack.install_area.layout.remove_extension(self.extendee_spec, - self.spec) + spack.store.layout.remove_extension( + self.extendee_spec, self.spec) tty.msg("Deactivated extension %s for %s" % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) @@ -1616,7 +1620,7 @@ class PackageBase(object): """ def ignore(filename): - return (filename in spack.install_area.layout.hidden_file_paths or + return (filename in spack.store.layout.hidden_file_paths or kwargs.get('ignore', lambda f: False)(filename)) tree = LinkTree(extension.prefix) @@ -1717,7 +1721,7 @@ def flatten_dependencies(spec, flat_dir): for dep in spec.traverse(root=False): name = dep.name - dep_path = spack.install_area.layout.path_for_spec(dep) + dep_path = spack.store.layout.path_for_spec(dep) dep_files = LinkTree(dep_path) os.mkdir(flat_dir + '/' + name) @@ -1746,7 +1750,7 @@ def dump_packages(spec, path): if node is not spec: # Locate the dependency package in the install tree and find # its provenance information. - source = spack.install_area.layout.build_packages_path(node) + source = spack.store.layout.build_packages_path(node) source_repo_root = join_path(source, node.namespace) # There's no provenance installed for the source package. Skip it. diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py new file mode 100644 index 0000000000..7ae3b75c3e --- /dev/null +++ b/lib/spack/spack/schema/config.py @@ -0,0 +1,62 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Schema for packages.yaml configuration files.""" + +schema = { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack module file configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + 'config': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'properties': { + 'install_tree': {'type': 'string'}, + 'build_stage': { + 'oneOf': [ + {'type': 'string'}, + {'type': 'array', + 'items': {'type': 'string'}}], + }, + 'source_cache': {'type': 'string'}, + 'misc_cache': {'type': 'string'}, + 'verify_ssl': { + 'type': 'boolean', + 'default': True, + }, + 'checksum': { + 'type': 'boolean', + 'default': True, + }, + 'dirty': { + 'type': 'boolean', + 'default': False, + }, + } + }, + }, +} diff --git a/lib/spack/spack/schema/install.py b/lib/spack/spack/schema/install.py deleted file mode 100644 index 2c43ad1adf..0000000000 --- a/lib/spack/spack/schema/install.py +++ /dev/null @@ -1,43 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -"""Schema for packages.yaml configuration files.""" - -schema = { - '$schema': 'http://json-schema.org/schema#', - 'title': 'Spack module file configuration file schema', - 'type': 'object', - 'additionalProperties': False, - 'patternProperties': { - 'install': { - 'type': 'object', - 'default' : {}, - 'additionalProperties': False, - 'properties': { - 'path' : { 'type': 'string' }, - 'layout' : { 'type': 'string' } - } - }, - }, -} diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 1d9fd8d97a..17f206d81b 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -111,7 +111,7 @@ from llnl.util.tty.color import * import spack import spack.architecture -import spack.install_area +import spack.store import spack.compilers as compilers import spack.error import spack.parse @@ -965,7 +965,7 @@ class Spec(object): @property def prefix(self): - return Prefix(spack.install_area.layout.path_for_spec(self)) + return Prefix(spack.store.layout.path_for_spec(self)) def dag_hash(self, length=None): """Return a hash of the entire spec DAG, including connectivity.""" @@ -2329,7 +2329,7 @@ class Spec(object): elif named_str == 'SPACK_ROOT': out.write(fmt % spack.prefix) elif named_str == 'SPACK_INSTALL': - out.write(fmt % spack.install_area.path) + out.write(fmt % spack.store.root) elif named_str == 'PREFIX': out.write(fmt % self.prefix) elif named_str.startswith('HASH'): @@ -2533,7 +2533,7 @@ class SpecParser(spack.parse.Parser): def spec_by_hash(self): self.expect(ID) - specs = spack.install_area.db.query() + specs = spack.store.db.query() matches = [spec for spec in specs if spec.dag_hash()[:len(self.token.value)] == self.token.value] diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py new file mode 100644 index 0000000000..3f559315d2 --- /dev/null +++ b/lib/spack/spack/store.py @@ -0,0 +1,75 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Components that manage Spack's installation tree. + +An install tree, or "build store" consists of two parts: + + 1. A package database that tracks what is installed. + 2. A directory layout that determines how the installations + are laid out. + +The store contains all the install prefixes for packages installed by +Spack. The simplest store could just contain prefixes named by DAG hash, +but we use a fancier directory layout to make browsing the store and +debugging easier. + +The directory layout is currently hard-coded to be a YAMLDirectoryLayout, +so called because it stores build metadata within each prefix, in +`spec.yaml` files. In future versions of Spack we may consider allowing +install trees to define their own layouts with some per-tree +configuration. + +""" +import os +import spack +import spack.config +from spack.util.path import canonicalize_path +from spack.database import Database +from spack.directory_layout import YamlDirectoryLayout + +__author__ = "Benedikt Hegner (CERN)" +__all__ = ['db', 'layout', 'root'] + +# +# Read in the config +# +config = spack.config.get_config("config") + +# +# Set up the install path +# +root = canonicalize_path( + config.get('install_tree', os.path.join(spack.opt_path, 'spack'))) + +# +# Set up the installed packages database +# +db = Database(root) + +# +# This controls how spack lays out install prefixes and +# stage directories. +# +layout = YamlDirectoryLayout(root) diff --git a/lib/spack/spack/test/cmd/uninstall.py b/lib/spack/spack/test/cmd/uninstall.py index d839c00b62..6a86a1543f 100644 --- a/lib/spack/spack/test/cmd/uninstall.py +++ b/lib/spack/spack/test/cmd/uninstall.py @@ -23,7 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import spack.test.mock_database -import spack.install_area +import spack.store from spack.cmd.uninstall import uninstall @@ -51,7 +51,7 @@ class TestUninstall(spack.test.mock_database.MockDatabase): args = MockArgs(['callpath'], all=True, dependents=True) uninstall(parser, args) - all_specs = spack.install_area.layout.all_specs() + all_specs = spack.store.layout.all_specs() self.assertEqual(len(all_specs), 7) # query specs with multiple configurations mpileaks_specs = [s for s in all_specs if s.satisfies('mpileaks')] diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index c0bb12ce10..1a1281e10e 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -30,7 +30,7 @@ import multiprocessing import os.path import spack -import spack.install_area +import spack.store from llnl.util.filesystem import join_path from llnl.util.tty.colify import colify from spack.test.mock_database import MockDatabase @@ -41,16 +41,16 @@ def _print_ref_counts(): recs = [] def add_rec(spec): - cspecs = spack.install_area.db.query(spec, installed=any) + cspecs = spack.store.db.query(spec, installed=any) if not cspecs: recs.append("[ %-7s ] %-20s-" % ('', spec)) else: key = cspecs[0].dag_hash() - rec = spack.install_area.db.get_record(cspecs[0]) + rec = spack.store.db.get_record(cspecs[0]) recs.append("[ %-7s ] %-20s%d" % (key[:7], spec, rec.ref_count)) - with spack.install_area.db.read_transaction(): + with spack.store.db.read_transaction(): add_rec('mpileaks ^mpich') add_rec('callpath ^mpich') add_rec('mpich') @@ -83,7 +83,7 @@ class DatabaseTest(MockDatabase): def test_010_all_install_sanity(self): """Ensure that the install layout reflects what we think it does.""" - all_specs = spack.install_area.layout.all_specs() + all_specs = spack.store.layout.all_specs() self.assertEqual(len(all_specs), 13) # query specs with multiple configurations @@ -114,12 +114,12 @@ class DatabaseTest(MockDatabase): def test_015_write_and_read(self): # write and read DB - with spack.install_area.db.write_transaction(): - specs = spack.install_area.db.query() - recs = [spack.install_area.db.get_record(s) for s in specs] + with spack.store.db.write_transaction(): + specs = spack.store.db.query() + recs = [spack.store.db.get_record(s) for s in specs] for spec, rec in zip(specs, recs): - new_rec = spack.install_area.db.get_record(spec) + new_rec = spack.store.db.get_record(spec) self.assertEqual(new_rec.ref_count, rec.ref_count) self.assertEqual(new_rec.spec, rec.spec) self.assertEqual(new_rec.path, rec.path) @@ -127,7 +127,7 @@ class DatabaseTest(MockDatabase): def _check_merkleiness(self): """Ensure the spack database is a valid merkle graph.""" - all_specs = spack.install_area.db.query(installed=any) + all_specs = spack.store.db.query(installed=any) seen = {} for spec in all_specs: @@ -140,7 +140,7 @@ class DatabaseTest(MockDatabase): def _check_db_sanity(self): """Utiilty function to check db against install layout.""" - expected = sorted(spack.install_area.layout.all_specs()) + expected = sorted(spack.store.layout.all_specs()) actual = sorted(self.install_db.query()) self.assertEqual(len(expected), len(actual)) @@ -155,7 +155,7 @@ class DatabaseTest(MockDatabase): def test_025_reindex(self): """Make sure reindex works and ref counts are valid.""" - spack.install_area.db.reindex(spack.install_area.layout) + spack.store.db.reindex(spack.store.layout) self._check_db_sanity() def test_030_db_sanity_from_another_process(self): @@ -179,7 +179,7 @@ class DatabaseTest(MockDatabase): def test_050_basic_query(self): """Ensure querying database is consistent with what is installed.""" # query everything - self.assertEqual(len(spack.install_area.db.query()), 13) + self.assertEqual(len(spack.store.db.query()), 13) # query specs with multiple configurations mpileaks_specs = self.install_db.query('mpileaks') @@ -224,7 +224,7 @@ class DatabaseTest(MockDatabase): self.assertTrue(concrete_spec not in remaining) # add it back and make sure everything is ok. - self.install_db.add(concrete_spec, spack.install_area.layout) + self.install_db.add(concrete_spec, spack.store.layout) installed = self.install_db.query() self.assertTrue(concrete_spec in installed) self.assertEqual(installed, original) @@ -255,7 +255,7 @@ class DatabaseTest(MockDatabase): self.assertEqual(self.install_db.get_record('mpich').ref_count, 1) # Put the spec back - self.install_db.add(rec.spec, spack.install_area.layout) + self.install_db.add(rec.spec, spack.store.layout) # record is present again self.assertEqual( diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index eb051e9301..0524c14cfd 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -26,7 +26,7 @@ import shutil import tempfile import spack -import spack.install_area +import spack.store from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout from spack.database import Database @@ -50,11 +50,11 @@ class InstallTest(MockPackagesTest): # Use a fake install directory to avoid conflicts bt/w # installed pkgs and mock packages. self.tmpdir = tempfile.mkdtemp() - self.orig_layout = spack.install_area.layout - self.orig_db = spack.install_area.db + self.orig_layout = spack.store.layout + self.orig_db = spack.store.db - spack.install_area.layout = YamlDirectoryLayout(self.tmpdir) - spack.install_area.db = Database(self.tmpdir) + spack.store.layout = YamlDirectoryLayout(self.tmpdir) + spack.store.db = Database(self.tmpdir) def tearDown(self): super(InstallTest, self).tearDown() @@ -64,8 +64,8 @@ class InstallTest(MockPackagesTest): spack.do_checksum = True # restore spack's layout. - spack.install_area.layout = self.orig_layout - spack.install_area.db = self.orig_db + spack.store.layout = self.orig_layout + spack.store.db = self.orig_db shutil.rmtree(self.tmpdir, ignore_errors=True) def fake_fetchify(self, pkg): @@ -92,7 +92,7 @@ class InstallTest(MockPackagesTest): pkg.remove_prefix() raise - def test_install_area(self): + def test_store(self): spec = Spec('cmake-client').concretized() for s in spec.traverse(): diff --git a/lib/spack/spack/test/mock_database.py b/lib/spack/spack/test/mock_database.py index d1f9fb000b..1eaec2b598 100644 --- a/lib/spack/spack/test/mock_database.py +++ b/lib/spack/spack/test/mock_database.py @@ -26,7 +26,7 @@ import shutil import tempfile import spack -import spack.install_area +import spack.store from spack.spec import Spec from spack.database import Database from spack.directory_layout import YamlDirectoryLayout @@ -42,7 +42,7 @@ class MockDatabase(MockPackagesTest): pkg.do_install(fake=True) def _mock_remove(self, spec): - specs = spack.install_area.db.query(spec) + specs = spack.store.db.query(spec) assert len(specs) == 1 spec = specs[0] spec.package.do_uninstall(spec) @@ -55,17 +55,17 @@ class MockDatabase(MockPackagesTest): # Make a fake install directory self.install_path = tempfile.mkdtemp() - self.spack_install_path = spack.install_area.path - spack.install_area.path = self.install_path + self.spack_install_path = spack.store.root + spack.store.root = self.install_path self.install_layout = YamlDirectoryLayout(self.install_path) - self.spack_install_layout = spack.install_area.layout - spack.install_area.layout = self.install_layout + self.spack_install_layout = spack.store.layout + spack.store.layout = self.install_layout # Make fake database and fake install directory. self.install_db = Database(self.install_path) - self.spack_install_db = spack.install_area.db - spack.install_area.db = self.install_db + self.spack_install_db = spack.store.db + spack.store.db = self.install_db # make a mock database with some packages installed note that # the ref count for dyninst here will be 3, as it's recycled @@ -91,18 +91,18 @@ class MockDatabase(MockPackagesTest): # # Transaction used to avoid repeated writes. - with spack.install_area.db.write_transaction(): + with spack.store.db.write_transaction(): self._mock_install('mpileaks ^mpich') self._mock_install('mpileaks ^mpich2') self._mock_install('mpileaks ^zmpi') def tearDown(self): - with spack.install_area.db.write_transaction(): - for spec in spack.install_area.db.query(): + with spack.store.db.write_transaction(): + for spec in spack.store.db.query(): spec.package.do_uninstall(spec) super(MockDatabase, self).tearDown() shutil.rmtree(self.install_path) - spack.install_area.path = self.spack_install_path - spack.install_area.layout = self.spack_install_layout - spack.install_area.db = self.spack_install_db + spack.store.root = self.spack_install_path + spack.store.layout = self.spack_install_layout + spack.store.db = self.spack_install_db -- cgit v1.2.3-60-g2f50 From 0da639298c8d2d956c76d8beb9f08c54c87cfec7 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 27 Oct 2016 22:36:59 -0700 Subject: Move temp directory configuration to config.yaml - Moved temp finding logic to spack.stage - Updated stage tests - Added tests for new path substaitution of $user, $spack, $tempdir --- etc/spack/defaults/config.yaml | 10 +- lib/spack/llnl/util/filesystem.py | 12 -- lib/spack/spack/__init__.py | 21 ---- lib/spack/spack/repository.py | 14 +-- lib/spack/spack/stage.py | 117 ++++++++++++------ lib/spack/spack/test/config.py | 52 ++++++++ lib/spack/spack/test/mock_packages_test.py | 28 ++++- lib/spack/spack/test/stage.py | 184 +++++++++++++++-------------- lib/spack/spack/util/path.py | 68 +++++++++++ 9 files changed, 330 insertions(+), 176 deletions(-) create mode 100644 lib/spack/spack/util/path.py (limited to 'etc') diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 5ce60049ce..6a749b5f6d 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -31,13 +31,13 @@ config: # You can use $tempdir to refer to the system default temp directory # (as returned by tempfile.gettempdir()). # - # A value of $local indicates that Spack should run builds directly - # inside its install directory without staging them in temporary space. + # A value of $spack/var/spack/stage indicates that Spack should run + # builds directly inside its install directory without staging them in + # temporary space. build_stage: - - /usr/workspace/*/%u - $tempdir - - /nfs/tmp2/%u - - $local + - /nfs/tmp2/$user + - $spack/var/spack/stage # Cache directory already downloaded source tarballs and archived diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index e522fdda6d..31e09f2fe6 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,6 @@ import collections import errno import fileinput -import getpass import glob import numbers import os @@ -46,7 +45,6 @@ __all__ = [ 'can_access', 'change_sed_delimiter', 'copy_mode', - 'expand_user', 'filter_file', 'find_libraries', 'fix_darwin_install_name', @@ -229,16 +227,6 @@ def is_exe(path): return os.path.isfile(path) and os.access(path, os.X_OK) -def expand_user(path): - """Find instances of '%u' in a path and replace with the current user's - username.""" - username = getpass.getuser() - if not username and '%u' in path: - tty.die("Couldn't get username to complete path '%s'" % path) - - return path.replace('%u', username) - - def mkdirp(*paths): """Creates a directory, as well as parent directories if needed.""" for path in paths: diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index ab03c0c848..782a9b8a9f 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -119,27 +119,6 @@ editor = Executable(os.environ.get("EDITOR", "vi")) # certifiates. e.g., curl should use the -k option. insecure = False -# Whether to build in tmp space or directly in the stage_path. -# If this is true, then spack will make stage directories in -# a tmp filesystem, and it will symlink them into stage_path. -use_tmp_stage = True - -# Locations to use for staging and building, in order of preference -# Use a %u to add a username to the stage paths here, in case this -# is a shared filesystem. Spack will use the first of these paths -# that it can create. -tmp_dirs = [] -_default_tmp = tempfile.gettempdir() -_tmp_user = getpass.getuser() - -_tmp_candidates = (_default_tmp, '/nfs/tmp2', '/tmp', '/var/tmp') -for path in _tmp_candidates: - # don't add a second username if it's already unique by user. - if _tmp_user not in path: - tmp_dirs.append(join_path(path, '%u', 'spack-stage')) - else: - tmp_dirs.append(join_path(path, 'spack-stage')) - # Whether spack should allow installation of unsafe versions of # software. "Unsafe" versions are ones it doesn't have a checksum # for. diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 4696d1d5bc..47d6df85b3 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -44,6 +44,7 @@ import spack import spack.error import spack.spec from spack.provider_index import ProviderIndex +from spack.util.path import canonicalize_path from spack.util.naming import * # @@ -93,19 +94,6 @@ class SpackNamespace(ModuleType): return getattr(self, name) -def substitute_spack_prefix(path): - """Replaces instances of $spack with Spack's prefix.""" - return re.sub(r'^\$spack', spack.prefix, path) - - -def canonicalize_path(path): - """Substitute $spack, expand user home, take abspath.""" - path = substitute_spack_prefix(path) - path = os.path.expanduser(path) - path = os.path.abspath(path) - return path - - class RepoPath(object): """A RepoPath is a list of repos that function as one. diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 7e6b543799..ff10a38ca8 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -28,6 +28,7 @@ import errno import hashlib import shutil import tempfile +import getpass from urlparse import urljoin import llnl.util.tty as tty @@ -41,9 +42,72 @@ import spack.config import spack.fetch_strategy as fs import spack.error from spack.version import * +from spack.util.path import canonicalize_path from spack.util.crypto import prefix_bits, bit_length -STAGE_PREFIX = 'spack-stage-' +_stage_prefix = 'spack-stage-' + + +def _first_accessible_path(paths): + """Find a tmp dir that exists that we can access.""" + for path in paths: + try: + # try to create the path if it doesn't exist. + path = canonicalize_path(path) + mkdirp(path) + + # ensure accessible + if not can_access(path): + continue + + # return it if successful. + return path + + except OSError: + tty.debug('OSError while checking temporary path: %s' % path) + continue + + return None + + +# cached temporary root +_tmp_root = None +_use_tmp_stage = True + + +def get_tmp_root(): + global _tmp_root, _use_tmp_stage + + if not _use_tmp_stage: + return None + + if _tmp_root is None: + config = spack.config.get_config('config') + candidates = config['build_stage'] + if isinstance(candidates, basestring): + candidates = [candidates] + + path = _first_accessible_path(candidates) + if not path: + raise StageError("No accessible stage paths in %s", candidates) + + # Return None to indicate we're using a local staging area. + if path == canonicalize_path(spack.stage_path): + _use_tmp_stage = False + return None + + # ensure that any temp path is unique per user, so users don't + # fight over shared temporary space. + user = getpass.getuser() + if user not in path: + path = os.path.join(path, user, 'spack-stage') + else: + path = os.path.join(path, 'spack-stage') + + mkdirp(path) + _tmp_root = path + + return _tmp_root class Stage(object): @@ -141,9 +205,8 @@ class Stage(object): # TODO : won't be the same as the temporary stage area in tmp_root self.name = name if name is None: - self.name = STAGE_PREFIX + next(tempfile._get_candidate_names()) + self.name = _stage_prefix + next(tempfile._get_candidate_names()) self.mirror_path = mirror_path - self.tmp_root = find_tmp_root() # Try to construct here a temporary name for the stage directory # If this is a named stage, then construct a named path. @@ -217,10 +280,11 @@ class Stage(object): # Path looks ok, but need to check the target of the link. if os.path.islink(self.path): - real_path = os.path.realpath(self.path) - real_tmp = os.path.realpath(self.tmp_root) + tmp_root = get_tmp_root() + if tmp_root is not None: + real_path = os.path.realpath(self.path) + real_tmp = os.path.realpath(tmp_root) - if spack.use_tmp_stage: # If we're using a tmp dir, it's a link, and it points at the # right spot, then keep it. if (real_path.startswith(real_tmp) and @@ -416,11 +480,11 @@ class Stage(object): """ path = self.source_path if not path: - tty.die("Attempt to chdir before expanding archive.") + raise StageError("Attempt to chdir before expanding archive.") else: os.chdir(path) if not os.listdir(path): - tty.die("Archive was empty for %s" % self.name) + raise StageError("Archive was empty for %s" % self.name) def restage(self): """Removes the expanded archive path if it exists, then re-expands @@ -429,17 +493,17 @@ class Stage(object): self.fetcher.reset() def create(self): - """ - Creates the stage directory + """Creates the stage directory. - If self.tmp_root evaluates to False, the stage directory is - created directly under spack.stage_path, otherwise this will - attempt to create a stage in a temporary directory and link it - into spack.stage_path. + If get_tmp_root() is None, the stage directory is created + directly under spack.stage_path, otherwise this will attempt to + create a stage in a temporary directory and link it into + spack.stage_path. Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location in tmp_dirs, fall back to making the stage inside spack.stage_path. + """ # Create the top-level stage directory mkdirp(spack.stage_path) @@ -448,8 +512,10 @@ class Stage(object): # If a tmp_root exists then create a directory there and then link it # in the stage area, otherwise create the stage directory in self.path if self._need_to_create_path(): - if self.tmp_root: - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + tmp_root = get_tmp_root() + if tmp_root is not None: + tmp_dir = tempfile.mkdtemp('', _stage_prefix, tmp_root) + tty.debug('link %s -> %s' % (self.path, tmp_dir)) os.symlink(tmp_dir, self.path) else: mkdirp(self.path) @@ -614,25 +680,6 @@ def purge(): remove_linked_tree(stage_path) -def find_tmp_root(): - if spack.use_tmp_stage: - for tmp in spack.tmp_dirs: - try: - # Replace %u with username - expanded = expand_user(tmp) - - # try to create a directory for spack stuff - mkdirp(expanded) - - # return it if successful. - return expanded - - except OSError: - continue - - return None - - class StageError(spack.error.SpackError): """"Superclass for all errors encountered during staging.""" diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 02f53a5261..de6cd79594 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -24,10 +24,12 @@ ############################################################################## import os import shutil +import getpass from tempfile import mkdtemp import spack import spack.config +from spack.util.path import canonicalize_path from ordereddict_backport import OrderedDict from spack.test.mock_packages_test import * @@ -217,3 +219,53 @@ class ConfigTest(MockPackagesTest): # Same check again, to ensure consistency. self.check_config(a_comps, *self.a_comp_specs) self.check_config(b_comps, *self.b_comp_specs) + + def check_canonical(self, var, expected): + """ensure things are substituted properly and canonicalized.""" + path = '/foo/bar/baz' + + self.assertEqual(canonicalize_path(var + path), + expected + path) + + self.assertEqual(canonicalize_path(path + var), + path + '/' + expected) + + self.assertEqual(canonicalize_path(path + var + path), + expected + path) + + def test_substitute_config_variables(self): + prefix = spack.prefix.lstrip('/') + + self.assertEqual(os.path.join('/foo/bar/baz', prefix), + canonicalize_path('/foo/bar/baz/$spack')) + + self.assertEqual(os.path.join(spack.prefix, 'foo/bar/baz'), + canonicalize_path('$spack/foo/bar/baz/')) + + self.assertEqual(os.path.join('/foo/bar/baz', prefix, 'foo/bar/baz'), + canonicalize_path('/foo/bar/baz/$spack/foo/bar/baz/')) + + self.assertEqual(os.path.join('/foo/bar/baz', prefix), + canonicalize_path('/foo/bar/baz/${spack}')) + + self.assertEqual(os.path.join(spack.prefix, 'foo/bar/baz'), + canonicalize_path('${spack}/foo/bar/baz/')) + + self.assertEqual( + os.path.join('/foo/bar/baz', prefix, 'foo/bar/baz'), + canonicalize_path('/foo/bar/baz/${spack}/foo/bar/baz/')) + + self.assertNotEqual( + os.path.join('/foo/bar/baz', prefix, 'foo/bar/baz'), + canonicalize_path('/foo/bar/baz/${spack/foo/bar/baz/')) + + def test_substitute_user(self): + user = getpass.getuser() + self.assertEqual('/foo/bar/' + user + '/baz', + canonicalize_path('/foo/bar/$user/baz')) + + def test_substitute_tempdir(self): + tempdir = tempfile.gettempdir() + self.assertEqual(tempdir, canonicalize_path('$tempdir')) + self.assertEqual(tempdir + '/foo/bar/baz', + canonicalize_path('$tempdir/foo/bar/baz')) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index cba9c81e9d..4e1f243c82 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -180,6 +180,27 @@ packages: externalmodule@1.0%gcc@4.5.0: external-module """ +mock_config = """\ +config: + install_tree: $spack/opt/spack + build_stage: + - $tempdir + - /nfs/tmp2/$user + - $spack/var/spack/stage + source_cache: $spack/var/spack/cache + misc_cache: ~/.spack/cache + verify_ssl: true + checksum: true + dirty: false +""" + +# these are written out to mock config files. +mock_configs = { + 'config.yaml': mock_config, + 'compilers.yaml': mock_compiler_config, + 'packages.yaml': mock_packages_config, +} + class MockPackagesTest(unittest.TestCase): @@ -199,11 +220,10 @@ class MockPackagesTest(unittest.TestCase): self.mock_user_config = os.path.join(self.temp_config, 'user') mkdirp(self.mock_site_config) mkdirp(self.mock_user_config) - for confs in [('compilers.yaml', mock_compiler_config), - ('packages.yaml', mock_packages_config)]: - conf_yaml = os.path.join(self.mock_site_config, confs[0]) + for filename, data in mock_configs.items(): + conf_yaml = os.path.join(self.mock_site_config, filename) with open(conf_yaml, 'w') as f: - f.write(confs[1]) + f.write(data) # TODO: Mocking this up is kind of brittle b/c ConfigScope # TODO: constructor modifies config_scopes. Make it cleaner. diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index ec661bfe50..a21142c2cb 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -27,64 +27,71 @@ Test that the Stage class works correctly. """ import os import shutil -import unittest +import tempfile from contextlib import * import spack +import spack.stage from llnl.util.filesystem import * from spack.stage import Stage from spack.util.executable import which +from spack.test.mock_packages_test import * -test_files_dir = os.path.realpath(join_path(spack.stage_path, '.test')) -test_tmp_path = os.path.realpath(join_path(test_files_dir, 'tmp')) - -archive_dir = 'test-files' -archive_name = archive_dir + '.tar.gz' -archive_dir_path = join_path(test_files_dir, archive_dir) -archive_url = 'file://' + join_path(test_files_dir, archive_name) -readme_name = 'README.txt' -test_readme = join_path(archive_dir_path, readme_name) -readme_text = "hello world!\n" - -stage_name = 'spack-test-stage' +_test_tmp_path = None @contextmanager def use_tmp(use_tmp): - """Allow some test code to be executed with spack.use_tmp_stage - set to a certain value. Context manager makes sure it's reset - on failure. + """Allow some test code to be executed such that spack will either use or + not use temporary space for stages. """ - old_tmp = spack.use_tmp_stage - spack.use_tmp_stage = use_tmp + # mock up config + path = _test_tmp_path if use_tmp else spack.stage_path + spack.config.update_config( + 'config', {'build_stage': [path]}, scope='user') yield - spack.use_tmp_stage = old_tmp -class StageTest(unittest.TestCase): +class StageTest(MockPackagesTest): def setUp(self): """This sets up a mock archive to fetch, and a mock temp space for use by the Stage class. It doesn't actually create the Stage -- that is done by individual tests. """ - if os.path.exists(test_files_dir): - shutil.rmtree(test_files_dir) + global _test_tmp_path + + self.test_files_dir = tempfile.mkdtemp() + self.test_tmp_path = os.path.realpath( + os.path.join(self.test_files_dir, 'tmp')) + _test_tmp_path = self.test_tmp_path + + self.archive_dir = 'test-files' + self.archive_name = self.archive_dir + '.tar.gz' + archive_dir_path = os.path.join(self.test_files_dir, + self.archive_dir) + self.archive_url = 'file://' + os.path.join(self.test_files_dir, + self.archive_name) + test_readme = join_path(archive_dir_path, 'README.txt') + self.readme_text = "hello world!\n" + + self.stage_name = 'spack-test-stage' - mkdirp(test_files_dir) mkdirp(archive_dir_path) - mkdirp(test_tmp_path) + mkdirp(self.test_tmp_path) with open(test_readme, 'w') as readme: - readme.write(readme_text) + readme.write(self.readme_text) - with working_dir(test_files_dir): - tar = which('tar') - tar('czf', archive_name, archive_dir) + with working_dir(self.test_files_dir): + tar = which('tar', required=True) + tar('czf', self.archive_name, self.archive_dir) # Make spack use the test environment for tmp stuff. - self.old_tmp_dirs = spack.tmp_dirs - spack.tmp_dirs = [test_tmp_path] + self._old_tmp_root = spack.stage._tmp_root + self._old_use_tmp_stage = spack.stage._use_tmp_stage + spack.stage._tmp_root = None + spack.stage._use_tmp_stage = True # record this since this test changes to directories that will # be removed. @@ -92,13 +99,14 @@ class StageTest(unittest.TestCase): def tearDown(self): """Blows away the test environment directory.""" - shutil.rmtree(test_files_dir) + shutil.rmtree(self.test_files_dir, ignore_errors=True) # chdir back to original working dir os.chdir(self.working_dir) # restore spack's original tmp environment - spack.tmp_dirs = self.old_tmp_dirs + spack.stage._tmp_root = self._old_tmp_root + spack.stage._use_tmp_stage = self._old_use_tmp_stage def get_stage_path(self, stage, stage_name): """Figure out where a stage should be living. This depends on @@ -120,7 +128,7 @@ class StageTest(unittest.TestCase): # Ensure stage was created in the spack stage directory self.assertTrue(os.path.isdir(stage_path)) - if spack.use_tmp_stage: + if spack.stage.get_tmp_root(): # Check that the stage dir is really a symlink. self.assertTrue(os.path.islink(stage_path)) @@ -131,7 +139,7 @@ class StageTest(unittest.TestCase): # Make sure the directory is in the place we asked it to # be (see setUp and tearDown) - self.assertTrue(target.startswith(test_tmp_path)) + self.assertTrue(target.startswith(self.test_tmp_path)) else: # Make sure the stage path is NOT a link for a non-tmp stage @@ -139,24 +147,24 @@ class StageTest(unittest.TestCase): def check_fetch(self, stage, stage_name): stage_path = self.get_stage_path(stage, stage_name) - self.assertTrue(archive_name in os.listdir(stage_path)) - self.assertEqual(join_path(stage_path, archive_name), + self.assertTrue(self.archive_name in os.listdir(stage_path)) + self.assertEqual(join_path(stage_path, self.archive_name), stage.fetcher.archive_file) def check_expand_archive(self, stage, stage_name): stage_path = self.get_stage_path(stage, stage_name) - self.assertTrue(archive_name in os.listdir(stage_path)) - self.assertTrue(archive_dir in os.listdir(stage_path)) + self.assertTrue(self.archive_name in os.listdir(stage_path)) + self.assertTrue(self.archive_dir in os.listdir(stage_path)) self.assertEqual( - join_path(stage_path, archive_dir), + join_path(stage_path, self.archive_dir), stage.source_path) - readme = join_path(stage_path, archive_dir, readme_name) + readme = join_path(stage_path, self.archive_dir, 'README.txt') self.assertTrue(os.path.isfile(readme)) with open(readme) as file: - self.assertEqual(readme_text, file.read()) + self.assertEqual(self.readme_text, file.read()) def check_chdir(self, stage, stage_name): stage_path = self.get_stage_path(stage, stage_name) @@ -165,7 +173,7 @@ class StageTest(unittest.TestCase): def check_chdir_to_source(self, stage, stage_name): stage_path = self.get_stage_path(stage, stage_name) self.assertEqual( - join_path(os.path.realpath(stage_path), archive_dir), + join_path(os.path.realpath(stage_path), self.archive_dir), os.getcwd()) def check_destroy(self, stage, stage_name): @@ -176,76 +184,76 @@ class StageTest(unittest.TestCase): self.assertFalse(os.path.exists(stage_path)) # tmp stage needs to remove tmp dir too. - if spack.use_tmp_stage: + if spack.stage._use_tmp_stage: target = os.path.realpath(stage_path) self.assertFalse(os.path.exists(target)) def test_setup_and_destroy_name_with_tmp(self): with use_tmp(True): - with Stage(archive_url, name=stage_name) as stage: - self.check_setup(stage, stage_name) - self.check_destroy(stage, stage_name) + with Stage(self.archive_url, name=self.stage_name) as stage: + self.check_setup(stage, self.stage_name) + self.check_destroy(stage, self.stage_name) def test_setup_and_destroy_name_without_tmp(self): with use_tmp(False): - with Stage(archive_url, name=stage_name) as stage: - self.check_setup(stage, stage_name) - self.check_destroy(stage, stage_name) + with Stage(self.archive_url, name=self.stage_name) as stage: + self.check_setup(stage, self.stage_name) + self.check_destroy(stage, self.stage_name) def test_setup_and_destroy_no_name_with_tmp(self): with use_tmp(True): - with Stage(archive_url) as stage: + with Stage(self.archive_url) as stage: self.check_setup(stage, None) self.check_destroy(stage, None) def test_setup_and_destroy_no_name_without_tmp(self): with use_tmp(False): - with Stage(archive_url) as stage: + with Stage(self.archive_url) as stage: self.check_setup(stage, None) self.check_destroy(stage, None) def test_chdir(self): - with Stage(archive_url, name=stage_name) as stage: + with Stage(self.archive_url, name=self.stage_name) as stage: stage.chdir() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - self.check_destroy(stage, stage_name) + self.check_setup(stage, self.stage_name) + self.check_chdir(stage, self.stage_name) + self.check_destroy(stage, self.stage_name) def test_fetch(self): - with Stage(archive_url, name=stage_name) as stage: + with Stage(self.archive_url, name=self.stage_name) as stage: stage.fetch() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - self.check_destroy(stage, stage_name) + self.check_setup(stage, self.stage_name) + self.check_chdir(stage, self.stage_name) + self.check_fetch(stage, self.stage_name) + self.check_destroy(stage, self.stage_name) def test_expand_archive(self): - with Stage(archive_url, name=stage_name) as stage: + with Stage(self.archive_url, name=self.stage_name) as stage: stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) + self.check_setup(stage, self.stage_name) + self.check_fetch(stage, self.stage_name) stage.expand_archive() - self.check_expand_archive(stage, stage_name) - self.check_destroy(stage, stage_name) + self.check_expand_archive(stage, self.stage_name) + self.check_destroy(stage, self.stage_name) def test_expand_archive_with_chdir(self): - with Stage(archive_url, name=stage_name) as stage: + with Stage(self.archive_url, name=self.stage_name) as stage: stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) + self.check_setup(stage, self.stage_name) + self.check_fetch(stage, self.stage_name) stage.expand_archive() stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) - self.check_destroy(stage, stage_name) + self.check_expand_archive(stage, self.stage_name) + self.check_chdir_to_source(stage, self.stage_name) + self.check_destroy(stage, self.stage_name) def test_restage(self): - with Stage(archive_url, name=stage_name) as stage: + with Stage(self.archive_url, name=self.stage_name) as stage: stage.fetch() stage.expand_archive() stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) + self.check_expand_archive(stage, self.stage_name) + self.check_chdir_to_source(stage, self.stage_name) # Try to make a file in the old archive dir with open('foobar', 'w') as file: @@ -255,40 +263,44 @@ class StageTest(unittest.TestCase): # Make sure the file is not there after restage. stage.restage() - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) + self.check_chdir(stage, self.stage_name) + self.check_fetch(stage, self.stage_name) stage.chdir_to_source() - self.check_chdir_to_source(stage, stage_name) + self.check_chdir_to_source(stage, self.stage_name) self.assertFalse('foobar' in os.listdir(stage.source_path)) - self.check_destroy(stage, stage_name) + self.check_destroy(stage, self.stage_name) def test_no_keep_without_exceptions(self): - with Stage(archive_url, name=stage_name, keep=False) as stage: + with Stage(self.archive_url, + name=self.stage_name, keep=False) as stage: pass - self.check_destroy(stage, stage_name) + self.check_destroy(stage, self.stage_name) def test_keep_without_exceptions(self): - with Stage(archive_url, name=stage_name, keep=True) as stage: + with Stage(self.archive_url, + name=self.stage_name, keep=True) as stage: pass - path = self.get_stage_path(stage, stage_name) + path = self.get_stage_path(stage, self.stage_name) self.assertTrue(os.path.isdir(path)) def test_no_keep_with_exceptions(self): try: - with Stage(archive_url, name=stage_name, keep=False) as stage: + with Stage(self.archive_url, + name=self.stage_name, keep=False) as stage: raise Exception() - path = self.get_stage_path(stage, stage_name) + path = self.get_stage_path(stage, self.stage_name) self.assertTrue(os.path.isdir(path)) except: pass # ignore here. def test_keep_exceptions(self): try: - with Stage(archive_url, name=stage_name, keep=True) as stage: + with Stage(self.archive_url, + name=self.stage_name, keep=True) as stage: raise Exception() - path = self.get_stage_path(stage, stage_name) + path = self.get_stage_path(stage, self.stage_name) self.assertTrue(os.path.isdir(path)) except: pass # ignore here. diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py new file mode 100644 index 0000000000..5332115ae9 --- /dev/null +++ b/lib/spack/spack/util/path.py @@ -0,0 +1,68 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Utilities for managing paths in Spack. +""" +import os +import re +import spack +import getpass +import tempfile + +__all__ = [ + 'substitute_config_variables', + 'canonicalize_path'] + +# Substitutions to perform +replacements = { + 'spack': spack.prefix, + 'user': getpass.getuser(), + 'tempdir': tempfile.gettempdir(), +} + + +def substitute_config_variables(path): + """Substitute placeholders into paths. + + Spack allows paths in configs to have some placeholders, as follows: + + - $spack The Spack instance's prefix + - $user The current user's username + - $tempdir Default temporary directory returned by tempfile.gettempdir() + """ + # Look up replacements for re.sub in the replacements dict. + def repl(match): + m = match.group(0).strip('${}') + return replacements.get(m, match.group(0)) + + # Replace $var or ${var}. + return re.sub(r'(\$\w+\b|\$\{\w+\})', repl, path) + + +def canonicalize_path(path): + """Substitute $spack, expand user home, take abspath.""" + path = substitute_config_variables(path) + path = os.path.expanduser(path) + path = os.path.abspath(path) + return path -- cgit v1.2.3-60-g2f50 From 22b14e0f235864a636919911f134aee9fe4def8b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 27 Oct 2016 23:32:42 -0700 Subject: Add module_roots to config.yaml --- etc/spack/defaults/config.yaml | 13 ++++++----- lib/spack/spack/cmd/module.py | 3 ++- lib/spack/spack/modules.py | 36 +++++++++++++++++++++--------- lib/spack/spack/schema/config.py | 10 +++++++++ lib/spack/spack/test/modules.py | 48 ++++++++++++++++++++-------------------- 5 files changed, 69 insertions(+), 41 deletions(-) (limited to 'etc') diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 6a749b5f6d..85019ede61 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -14,17 +14,18 @@ # ~/.spack/config.yaml # ------------------------------------------------------------------------- config: - - # Precedence of configuration scopes, high to low. - # The user can override this paradoxically. - # scopes: [user, spack, default] - - # This is the path to the root of the Spack install tree. # You can use $spack here to refer to the root of the spack instance. install_tree: $spack/opt/spack + # Locations where different types of modules should be installed. + module_roots: + tcl: $spack/share/spack/modules + lmod: $spack/share/spack/lmod + dotkit: $spack/share/spack/dotkit + + # Temporary locations Spack can try to use for builds. # # Spack will use the first one it finds that exists and is writable. diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index c6fa84109e..d7abe0fa87 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -210,7 +210,8 @@ def refresh(mtype, specs, args): cls = module_types[mtype] # Detect name clashes - writers = [cls(spec) for spec in specs] + writers = [cls(spec) for spec in specs + if spack.repo.exists(spec.name)] # skip unknown packages. file2writer = collections.defaultdict(list) for item in writers: file2writer[item.file_name].append(item) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index cf67ca905d..2c2804aea1 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -48,10 +48,12 @@ import string import textwrap import llnl.util.tty as tty +from llnl.util.filesystem import join_path, mkdirp + import spack import spack.compilers # Needed by LmodModules import spack.config -from llnl.util.filesystem import join_path, mkdirp +from spack.util.path import canonicalize_path from spack.build_environment import parent_class_modules from spack.build_environment import set_module_variables_for_package from spack.environment import * @@ -62,7 +64,11 @@ __all__ = ['EnvModule', 'Dotkit', 'TclModule'] metaclass.""" module_types = {} -CONFIGURATION = spack.config.get_config('modules') +"""Module install roots are in config.yaml.""" +_roots = spack.config.get_config('config').get('module_roots', {}) + +"""Specifics about modules are in modules.yaml""" +_module_config = spack.config.get_config('modules') def print_help(): @@ -92,7 +98,7 @@ def inspect_path(prefix): """ env = EnvironmentModifications() # Inspect the prefix to check for the existence of common directories - prefix_inspections = CONFIGURATION.get('prefix_inspections', {}) + prefix_inspections = _module_config.get('prefix_inspections', {}) for relative_path, variables in prefix_inspections.items(): expected = join_path(prefix, relative_path) if os.path.isdir(expected): @@ -168,7 +174,7 @@ def parse_config_options(module_generator): module file """ # Get the configuration for this kind of generator - module_configuration = copy.deepcopy(CONFIGURATION.get( + module_configuration = copy.deepcopy(_module_config.get( module_generator.name, {})) ##### @@ -248,7 +254,7 @@ class EnvModule(object): def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) - if cls.name != 'env_module' and cls.name in CONFIGURATION[ + if cls.name != 'env_module' and cls.name in _module_config[ 'enable']: module_types[cls.name] = cls @@ -271,7 +277,7 @@ class EnvModule(object): @property def naming_scheme(self): try: - naming_scheme = CONFIGURATION[self.name]['naming_scheme'] + naming_scheme = _module_config[self.name]['naming_scheme'] except KeyError: naming_scheme = self.default_naming_format return naming_scheme @@ -314,7 +320,7 @@ class EnvModule(object): @property def blacklisted(self): - configuration = CONFIGURATION.get(self.name, {}) + configuration = _module_config.get(self.name, {}) whitelist_matches = [x for x in configuration.get('whitelist', []) if self.spec.satisfies(x)] @@ -473,7 +479,9 @@ class EnvModule(object): class Dotkit(EnvModule): name = 'dotkit' - path = join_path(spack.share_path, 'dotkit') + path = canonicalize_path( + _roots.get(name, join_path(spack.share_path, name))) + environment_modifications_formats = { PrependPath: 'dk_alter {name} {value}\n', RemovePath: 'dk_unalter {name} {value}\n', @@ -516,7 +524,8 @@ class Dotkit(EnvModule): class TclModule(EnvModule): name = 'tcl' - path = join_path(spack.share_path, "modules") + path = canonicalize_path( + _roots.get(name, join_path(spack.share_path, 'modules'))) autoload_format = ('if ![ is-loaded {module_file} ] {{\n' ' puts stderr "Autoloading {module_file}"\n' @@ -635,7 +644,8 @@ class TclModule(EnvModule): class LmodModule(EnvModule): name = 'lmod' - path = join_path(spack.share_path, "lmod") + path = canonicalize_path( + _roots.get(name, join_path(spack.share_path, name))) environment_modifications_formats = { PrependPath: 'prepend_path("{name}", "{value}")\n', @@ -656,6 +666,12 @@ class LmodModule(EnvModule): path_part_without_hash = join_path('{token.name}', '{token.version}') + # TODO : Check that extra tokens specified in configuration file + # TODO : are actually virtual dependencies + configuration = _module_config.get('lmod', {}) + hierarchy_tokens = configuration.get('hierarchical_scheme', []) + hierarchy_tokens = hierarchy_tokens + ['mpi', 'compiler'] + def __init__(self, spec=None): super(LmodModule, self).__init__(spec) diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index 7ae3b75c3e..a8057f303c 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -42,6 +42,16 @@ schema = { {'type': 'array', 'items': {'type': 'string'}}], }, + 'module_roots': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'properties': { + 'tcl': {'type': 'string'}, + 'lmod': {'type': 'string'}, + 'dotkit': {'type': 'string'}, + }, + }, 'source_cache': {'type': 'string'}, 'misc_cache': {'type': 'string'}, 'verify_ssl': { diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index cdaeef3339..cb3a26e62b 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -102,18 +102,18 @@ class ModuleFileGeneratorTests(MockPackagesTest): def setUp(self): super(ModuleFileGeneratorTests, self).setUp() - self.configuration_instance = spack.modules.CONFIGURATION + self.configuration_instance = spack.modules._module_config self.module_types_instance = spack.modules.module_types spack.modules.open = mock_open spack.modules.mkdirp = lambda x: None # Make sure that a non-mocked configuration will trigger an error - spack.modules.CONFIGURATION = None + spack.modules._module_config = None spack.modules.module_types = {self.factory.name: self.factory} def tearDown(self): del spack.modules.open spack.modules.module_types = self.module_types_instance - spack.modules.CONFIGURATION = self.configuration_instance + spack.modules._module_config = self.configuration_instance spack.modules.mkdirp = llnl.util.filesystem.mkdirp super(ModuleFileGeneratorTests, self).tearDown() @@ -231,7 +231,7 @@ class TclTests(ModuleFileGeneratorTests): } def test_simple_case(self): - spack.modules.CONFIGURATION = self.configuration_autoload_direct + spack.modules._module_config = self.configuration_autoload_direct spec = spack.spec.Spec(mpich_spec_string) content = self.get_modulefile_content(spec) self.assertTrue('module-whatis "mpich @3.0.4"' in content) @@ -239,13 +239,13 @@ class TclTests(ModuleFileGeneratorTests): spec, 'non-existing-tag') def test_autoload(self): - spack.modules.CONFIGURATION = self.configuration_autoload_direct + spack.modules._module_config = self.configuration_autoload_direct spec = spack.spec.Spec(mpileaks_spec_string) content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2) self.assertEqual(len([x for x in content if 'module load ' in x]), 2) - spack.modules.CONFIGURATION = self.configuration_autoload_all + spack.modules._module_config = self.configuration_autoload_all spec = spack.spec.Spec(mpileaks_spec_string) content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5) @@ -256,7 +256,7 @@ class TclTests(ModuleFileGeneratorTests): # - 1 ('build','link') dependency # - 1 ('build',) dependency # Just make sure the 'build' dependency is not there - spack.modules.CONFIGURATION = self.configuration_autoload_direct + spack.modules._module_config = self.configuration_autoload_direct spec = spack.spec.Spec('dtbuild1') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2) @@ -267,25 +267,25 @@ class TclTests(ModuleFileGeneratorTests): # - 1 ('build','link') dependency # - 1 ('build',) dependency # Just make sure the 'build' dependency is not there - spack.modules.CONFIGURATION = self.configuration_autoload_all + spack.modules._module_config = self.configuration_autoload_all spec = spack.spec.Spec('dtbuild1') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2) self.assertEqual(len([x for x in content if 'module load ' in x]), 2) def test_prerequisites(self): - spack.modules.CONFIGURATION = self.configuration_prerequisites_direct + spack.modules._module_config = self.configuration_prerequisites_direct spec = spack.spec.Spec('mpileaks arch=x86-linux') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'prereq' in x]), 2) - spack.modules.CONFIGURATION = self.configuration_prerequisites_all + spack.modules._module_config = self.configuration_prerequisites_all spec = spack.spec.Spec('mpileaks arch=x86-linux') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'prereq' in x]), 5) def test_alter_environment(self): - spack.modules.CONFIGURATION = self.configuration_alter_environment + spack.modules._module_config = self.configuration_alter_environment spec = spack.spec.Spec('mpileaks platform=test target=x86_64') content = self.get_modulefile_content(spec) self.assertEqual( @@ -315,7 +315,7 @@ class TclTests(ModuleFileGeneratorTests): len([x for x in content if 'setenv LIBDWARF_ROOT' in x]), 1) def test_blacklist(self): - spack.modules.CONFIGURATION = self.configuration_blacklist + spack.modules._module_config = self.configuration_blacklist spec = spack.spec.Spec('mpileaks ^zmpi') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) @@ -329,7 +329,7 @@ class TclTests(ModuleFileGeneratorTests): self.assertEqual(len([x for x in content if 'module load ' in x]), 1) def test_conflicts(self): - spack.modules.CONFIGURATION = self.configuration_conflicts + spack.modules._module_config = self.configuration_conflicts spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual( @@ -339,11 +339,11 @@ class TclTests(ModuleFileGeneratorTests): self.assertEqual( len([x for x in content if x == 'conflict intel/14.0.1']), 1) - spack.modules.CONFIGURATION = self.configuration_wrong_conflicts + spack.modules._module_config = self.configuration_wrong_conflicts self.assertRaises(SystemExit, self.get_modulefile_content, spec) def test_suffixes(self): - spack.modules.CONFIGURATION = self.configuration_suffix + spack.modules._module_config = self.configuration_suffix spec = spack.spec.Spec('mpileaks+debug arch=x86-linux') spec.concretize() generator = spack.modules.TclModule(spec) @@ -412,7 +412,7 @@ class LmodTests(ModuleFileGeneratorTests): } def test_simple_case(self): - spack.modules.CONFIGURATION = self.configuration_autoload_direct + spack.modules._module_config = self.configuration_autoload_direct spec = spack.spec.Spec(mpich_spec_string) content = self.get_modulefile_content(spec) self.assertTrue('-- -*- lua -*-' in content) @@ -420,14 +420,14 @@ class LmodTests(ModuleFileGeneratorTests): self.assertTrue('whatis([[Version : 3.0.4]])' in content) def test_autoload(self): - spack.modules.CONFIGURATION = self.configuration_autoload_direct + spack.modules._module_config = self.configuration_autoload_direct spec = spack.spec.Spec(mpileaks_spec_string) content = self.get_modulefile_content(spec) self.assertEqual( len([x for x in content if 'if not isloaded(' in x]), 2) self.assertEqual(len([x for x in content if 'load(' in x]), 2) - spack.modules.CONFIGURATION = self.configuration_autoload_all + spack.modules._module_config = self.configuration_autoload_all spec = spack.spec.Spec(mpileaks_spec_string) content = self.get_modulefile_content(spec) self.assertEqual( @@ -435,7 +435,7 @@ class LmodTests(ModuleFileGeneratorTests): self.assertEqual(len([x for x in content if 'load(' in x]), 5) def test_alter_environment(self): - spack.modules.CONFIGURATION = self.configuration_alter_environment + spack.modules._module_config = self.configuration_alter_environment spec = spack.spec.Spec('mpileaks platform=test target=x86_64') content = self.get_modulefile_content(spec) self.assertEqual( @@ -460,7 +460,7 @@ class LmodTests(ModuleFileGeneratorTests): len([x for x in content if 'unsetenv("BAR")' in x]), 0) def test_blacklist(self): - spack.modules.CONFIGURATION = self.configuration_blacklist + spack.modules._module_config = self.configuration_blacklist spec = spack.spec.Spec(mpileaks_spec_string) content = self.get_modulefile_content(spec) self.assertEqual( @@ -499,14 +499,14 @@ class DotkitTests(MockPackagesTest): def setUp(self): super(DotkitTests, self).setUp() - self.configuration_obj = spack.modules.CONFIGURATION + self.configuration_obj = spack.modules._module_config spack.modules.open = mock_open # Make sure that a non-mocked configuration will trigger an error - spack.modules.CONFIGURATION = None + spack.modules._module_config = None def tearDown(self): del spack.modules.open - spack.modules.CONFIGURATION = self.configuration_obj + spack.modules._module_config = self.configuration_obj super(DotkitTests, self).tearDown() def get_modulefile_content(self, spec): @@ -517,7 +517,7 @@ class DotkitTests(MockPackagesTest): return content def test_dotkit(self): - spack.modules.CONFIGURATION = self.configuration_dotkit + spack.modules._module_config = self.configuration_dotkit spec = spack.spec.Spec('mpileaks arch=x86-linux') content = self.get_modulefile_content(spec) self.assertTrue('#c spack' in content) -- cgit v1.2.3-60-g2f50 From da760a66de8370ad73eadfe7a38a7cb6782c99bc Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 28 Oct 2016 00:57:51 -0700 Subject: source_cache, misc_cache, verify_ssl, checksum, & dirty in config.yaml - Added new preferences to config.yaml: - source_cache - misc_cache - verify_ssl - checksum - dirty --- etc/spack/defaults/config.yaml | 5 +- lib/spack/spack/__init__.py | 115 ++++++++++++++++++----------- lib/spack/spack/cmd/common/arguments.py | 18 +++++ lib/spack/spack/cmd/diy.py | 7 +- lib/spack/spack/cmd/install.py | 8 +- lib/spack/spack/cmd/purge.py | 10 +-- lib/spack/spack/cmd/setup.py | 7 +- lib/spack/spack/config.py | 3 + lib/spack/spack/file_cache.py | 2 +- lib/spack/spack/modules.py | 2 +- lib/spack/spack/package.py | 6 +- lib/spack/spack/repository.py | 8 +- lib/spack/spack/test/config.py | 3 +- lib/spack/spack/test/mock_packages_test.py | 9 ++- lib/spack/spack/test/modules.py | 2 +- lib/spack/spack/test/stage.py | 28 ++++++- lib/spack/spack/util/path.py | 8 +- 17 files changed, 164 insertions(+), 77 deletions(-) (limited to 'etc') diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 85019ede61..4e02d0973d 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -35,6 +35,8 @@ config: # A value of $spack/var/spack/stage indicates that Spack should run # builds directly inside its install directory without staging them in # temporary space. + # + # The build stage can be purged with `spack purge --stage`. build_stage: - $tempdir - /nfs/tmp2/$user @@ -42,11 +44,12 @@ config: # Cache directory already downloaded source tarballs and archived - # repositories. This can be purged with spack purge + # repositories. This can be purged with `spack purge --downloads`. source_cache: $spack/var/spack/cache # Cache directory for miscellaneous files, like the package index. + # This can be purged with `spack purge --misc-cache` misc_cache: ~/.spack/cache diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 782a9b8a9f..0646f5cb32 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -30,7 +30,10 @@ import getpass from llnl.util.filesystem import * import llnl.util.tty as tty -# This lives in $prefix/lib/spack/spack/__file__ +#----------------------------------------------------------------------------- +# Variables describing how Spack is laid out in its prefix. +#----------------------------------------------------------------------------- +# This file lives in $prefix/lib/spack/spack/__file__ spack_root = ancestor(__file__, 4) # The spack script itself @@ -49,82 +52,100 @@ var_path = join_path(spack_root, "var", "spack") stage_path = join_path(var_path, "stage") repos_path = join_path(var_path, "repos") share_path = join_path(spack_root, "share", "spack") -cache_path = join_path(var_path, "cache") + +# Paths to built-in Spack repositories. +packages_path = join_path(repos_path, "builtin") +mock_packages_path = join_path(repos_path, "builtin.mock") # User configuration location user_config_path = os.path.expanduser('~/.spack') -import spack.fetch_strategy -fetch_cache = spack.fetch_strategy.FsCache(cache_path) - -from spack.file_cache import FileCache -user_cache_path = join_path(user_config_path, 'cache') -user_cache = FileCache(user_cache_path) - prefix = spack_root opt_path = join_path(prefix, "opt") etc_path = join_path(prefix, "etc") -# -# Set up the default packages database. -# + +#----------------------------------------------------------------------------- +# Initial imports (only for use in this file -- see __all__ below.) +#----------------------------------------------------------------------------- +# These imports depend on the paths above, or on each other +# Group them here so it's easy to understand the order. +# TODO: refactor this stuff to be more init order agnostic. import spack.repository +import spack.error +import spack.config +import spack.fetch_strategy +from spack.file_cache import FileCache +from spack.preferred_packages import PreferredPackages +from spack.abi import ABI +from spack.concretize import DefaultConcretizer +from spack.version import Version +from spack.util.path import canonicalize_path + + +#----------------------------------------------------------------------------- +# Initialize various data structures & objects at the core of Spack. +#----------------------------------------------------------------------------- +# Version information +spack_version = Version("0.9.1") + + +# Set up the default packages database. try: repo = spack.repository.RepoPath() sys.meta_path.append(repo) except spack.error.SpackError, e: tty.die('while initializing Spack RepoPath:', e.message) -# -# Paths to built-in Spack repositories. -# -packages_path = join_path(repos_path, "builtin") -mock_packages_path = join_path(repos_path, "builtin.mock") -# -# This controls how packages are sorted when trying to choose -# the most preferred package. More preferred packages are sorted -# first. -# -from spack.preferred_packages import PreferredPackages +# PreferredPackages controls preference sort order during concretization. +# More preferred packages are sorted first. pkgsort = PreferredPackages() -# -# This tests ABI compatibility between packages -# -from spack.abi import ABI + +# Tests ABI compatibility between packages abi = ABI() -# + # This controls how things are concretized in spack. # Replace it with a subclass if you want different # policies. -# -from spack.concretize import DefaultConcretizer concretizer = DefaultConcretizer() -# Version information -from spack.version import Version -spack_version = Version("0.9.1") +#----------------------------------------------------------------------------- +# config.yaml options +#----------------------------------------------------------------------------- +_config = spack.config.get_config('config') -# -# Executables used by Spack -# -from spack.util.executable import Executable, which -# User's editor from the environment -editor = Executable(os.environ.get("EDITOR", "vi")) +# Path where downloaded source code is cached +cache_path = canonicalize_path( + _config.get('source_cache', join_path(var_path, "cache"))) +fetch_cache = spack.fetch_strategy.FsCache(cache_path) + + +# cache for miscellaneous stuff. +misc_cache_path = canonicalize_path( + _config.get('misc_cache', join_path(user_config_path, 'cache'))) +misc_cache = FileCache(misc_cache_path) + # If this is enabled, tools that use SSL should not verify # certifiates. e.g., curl should use the -k option. -insecure = False +insecure = not _config.get('verify_ssl', True) -# Whether spack should allow installation of unsafe versions of -# software. "Unsafe" versions are ones it doesn't have a checksum -# for. -do_checksum = True -# +# Whether spack should allow installation of unsafe versions of software. +# "Unsafe" versions are ones it doesn't have a checksum for. +do_checksum = _config.get('checksum', True) + + +# If this is True, spack will not clean the environment to remove +# potentially harmful variables before builds. +dirty = _config.get('dirty', False) + + +#----------------------------------------------------------------------------- # When packages call 'from spack import *', this extra stuff is brought in. # # Spack internal code should call 'import spack' and accesses other @@ -135,6 +156,7 @@ do_checksum = True # packages should live. This file is overloaded for spack core vs. # for packages. # +#----------------------------------------------------------------------------- __all__ = ['PackageBase', 'Package', 'CMakePackage', @@ -165,6 +187,9 @@ import spack.util.executable from spack.util.executable import * __all__ += spack.util.executable.__all__ +# User's editor from the environment +editor = Executable(os.environ.get("EDITOR", "vi")) + from spack.package import \ install_dependency_symlinks, flatten_dependencies, \ DependencyConflictError, InstallError, ExternalPackageError diff --git a/lib/spack/spack/cmd/common/arguments.py b/lib/spack/spack/cmd/common/arguments.py index 5ab54bdb61..8cb4a4b871 100644 --- a/lib/spack/spack/cmd/common/arguments.py +++ b/lib/spack/spack/cmd/common/arguments.py @@ -95,3 +95,21 @@ parms = Bunch( 'help': 'Recursively traverse spec dependencies' }) _arguments['recurse_dependencies'] = parms + +parms = Bunch( + flags=('--clean',), + kwargs={ + 'action': 'store_false', + 'dest': 'dirty', + 'help': 'Clean environment before installing package.' + }) +_arguments['clean'] = parms + +parms = Bunch( + flags=('--dirty',), + kwargs={ + 'action': 'store_true', + 'dest': 'dirty', + 'help': 'Do NOT clean environment before installing.' + }) +_arguments['dirty'] = parms diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 08386cac07..22966a26eb 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -30,6 +30,7 @@ import llnl.util.tty as tty import spack import spack.cmd +import spack.cmd.common.arguments as arguments from spack.cmd.edit import edit_package from spack.stage import DIYStage @@ -52,9 +53,9 @@ def setup_parser(subparser): subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="specs to use for install. Must contain package AND version.") - subparser.add_argument( - '--dirty', action='store_true', dest='dirty', - help="Install a package *without* cleaning the environment.") + + cd_group = subparser.add_mutually_exclusive_group() + arguments.add_common_arguments(cd_group, ['clean', 'dirty']) def diy(self, args): diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 417e07e9c4..79af35d3a0 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -34,6 +34,7 @@ import llnl.util.filesystem as fs import llnl.util.tty as tty import spack import spack.cmd +import spack.cmd.common.arguments as arguments from spack.build_environment import InstallError from spack.fetch_strategy import FetchError from spack.package import PackageBase @@ -70,9 +71,10 @@ the dependencies.""" subparser.add_argument( '--fake', action='store_true', dest='fake', help="Fake install. Just remove prefix and create a fake file.") - subparser.add_argument( - '--dirty', action='store_true', dest='dirty', - help="Install a package *without* cleaning the environment.") + + cd_group = subparser.add_mutually_exclusive_group() + arguments.add_common_arguments(cd_group, ['clean', 'dirty']) + subparser.add_argument( 'package', nargs=argparse.REMAINDER, diff --git a/lib/spack/spack/cmd/purge.py b/lib/spack/spack/cmd/purge.py index 26d160635c..66cfc2af29 100644 --- a/lib/spack/spack/cmd/purge.py +++ b/lib/spack/spack/cmd/purge.py @@ -36,8 +36,8 @@ def setup_parser(subparser): '-d', '--downloads', action='store_true', help="Remove cached downloads.") subparser.add_argument( - '-u', '--user-cache', action='store_true', - help="Remove caches in user home directory. Includes virtual indices.") + '-m', '--misc-cache', action='store_true', + help="Remove long-lived caches, like the virtual package index.") subparser.add_argument( '-a', '--all', action='store_true', help="Remove all of the above.") @@ -45,7 +45,7 @@ def setup_parser(subparser): def purge(parser, args): # Special case: no flags. - if not any((args.stage, args.downloads, args.user_cache, args.all)): + if not any((args.stage, args.downloads, args.misc_cache, args.all)): stage.purge() return @@ -54,5 +54,5 @@ def purge(parser, args): stage.purge() if args.downloads or args.all: spack.fetch_cache.destroy() - if args.user_cache or args.all: - spack.user_cache.destroy() + if args.misc_cache or args.all: + spack.misc_cache.destroy() diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py index 6509228a98..953906975e 100644 --- a/lib/spack/spack/cmd/setup.py +++ b/lib/spack/spack/cmd/setup.py @@ -32,6 +32,7 @@ import llnl.util.tty as tty import spack import spack.cmd import spack.cmd.install as install +import spack.cmd.common.arguments as arguments from llnl.util.filesystem import set_executable from spack import which from spack.cmd.edit import edit_package @@ -50,9 +51,9 @@ def setup_parser(subparser): subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="specs to use for install. Must contain package AND version.") - subparser.add_argument( - '--dirty', action='store_true', dest='dirty', - help="Install a package *without* cleaning the environment.") + + cd_group = subparser.add_mutually_exclusive_group() + arguments.add_common_arguments(cd_group, ['clean', 'dirty']) def spack_transitive_include_path(): diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 1bd1689150..904195e47e 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -298,6 +298,9 @@ class ConfigScope(object): """Empty cached config information.""" self.sections = {} + def __repr__(self): + return '' % (self.name, self.path) + # # Below are configuration scopes. # diff --git a/lib/spack/spack/file_cache.py b/lib/spack/spack/file_cache.py index 31ae009836..e37f77d68d 100644 --- a/lib/spack/spack/file_cache.py +++ b/lib/spack/spack/file_cache.py @@ -113,7 +113,7 @@ class FileCache(object): Returns a ReadTransaction context manager and opens the cache file for reading. You can use it like this: - with spack.user_cache.read_transaction(key) as cache_file: + with file_cache_object.read_transaction(key) as cache_file: cache_file.read() """ diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 2c2804aea1..5e2a840e14 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -675,7 +675,7 @@ class LmodModule(EnvModule): def __init__(self, spec=None): super(LmodModule, self).__init__(spec) - self.configuration = CONFIGURATION.get('lmod', {}) + self.configuration = _module_config.get('lmod', {}) hierarchy_tokens = self.configuration.get('hierarchical_scheme', []) # TODO : Check that the extra hierarchy tokens specified in the # TODO : configuration file are actually virtual dependencies diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 6bc3362639..491e21bf95 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1118,7 +1118,7 @@ class PackageBase(object): run_tests=False, fake=False, explicit=False, - dirty=False, + dirty=None, **kwargs): """Called by commands to install a package and its dependencies. @@ -1165,6 +1165,10 @@ class PackageBase(object): rec.explicit = True return + # Dirty argument takes precedence over dirty config setting. + if dirty is None: + dirty = spack.dirty + self._do_install_pop_kwargs(kwargs) tty.msg("Installing %s" % self.name) diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 47d6df85b3..94b79accdb 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -620,12 +620,12 @@ class Repo(object): # Read the old ProviderIndex, or make a new one. key = self._cache_file - index_existed = spack.user_cache.init_entry(key) + index_existed = spack.misc_cache.init_entry(key) if index_existed and not self._needs_update: - with spack.user_cache.read_transaction(key) as f: + with spack.misc_cache.read_transaction(key) as f: self._provider_index = ProviderIndex.from_yaml(f) else: - with spack.user_cache.write_transaction(key) as (old, new): + with spack.misc_cache.write_transaction(key) as (old, new): if old: self._provider_index = ProviderIndex.from_yaml(old) else: @@ -701,7 +701,7 @@ class Repo(object): self._all_package_names = [] # Get index modification time. - index_mtime = spack.user_cache.mtime(self._cache_file) + index_mtime = spack.misc_cache.mtime(self._cache_file) for pkg_name in os.listdir(self.packages_path): # Skip non-directories in the package root. diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index de6cd79594..d5e1791b40 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -221,7 +221,8 @@ class ConfigTest(MockPackagesTest): self.check_config(b_comps, *self.b_comp_specs) def check_canonical(self, var, expected): - """ensure things are substituted properly and canonicalized.""" + """Ensure that is substituted properly for in strings + containing in various positions.""" path = '/foo/bar/baz' self.assertEqual(canonicalize_path(var + path), diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 4e1f243c82..a1919cd437 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -191,7 +191,7 @@ config: misc_cache: ~/.spack/cache verify_ssl: true checksum: true - dirty: false + dirty: True """ # these are written out to mock config files. @@ -211,9 +211,6 @@ class MockPackagesTest(unittest.TestCase): self.db = RepoPath(spack.mock_packages_path) spack.repo.swap(self.db) - spack.config.clear_config_caches() - self.real_scopes = spack.config.config_scopes - # Mock up temporary configuration directories self.temp_config = tempfile.mkdtemp() self.mock_site_config = os.path.join(self.temp_config, 'site') @@ -227,6 +224,9 @@ class MockPackagesTest(unittest.TestCase): # TODO: Mocking this up is kind of brittle b/c ConfigScope # TODO: constructor modifies config_scopes. Make it cleaner. + spack.config.clear_config_caches() + self.real_scopes = spack.config.config_scopes + spack.config.config_scopes = OrderedDict() spack.config.ConfigScope('site', self.mock_site_config) spack.config.ConfigScope('user', self.mock_user_config) @@ -261,6 +261,7 @@ class MockPackagesTest(unittest.TestCase): """Restore the real packages path after any test.""" spack.repo.swap(self.db) spack.config.config_scopes = self.real_scopes + shutil.rmtree(self.temp_config, ignore_errors=True) spack.config.clear_config_caches() diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index cb3a26e62b..42f072debb 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -471,7 +471,7 @@ class LmodTests(ModuleFileGeneratorTests): # Make sure that virtual providers (in the hierarchy) always # include a hash. Make sure that the module file for the spec # does not include a hash if hash_length is 0. - spack.modules.CONFIGURATION = self.configuration_no_hash + spack.modules._module_config = self.configuration_no_hash spec = spack.spec.Spec(mpileaks_spec_string) spec.concretize() module = spack.modules.LmodModule(spec) diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index a21142c2cb..64cfa222db 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -46,9 +46,16 @@ def use_tmp(use_tmp): not use temporary space for stages. """ # mock up config - path = _test_tmp_path if use_tmp else spack.stage_path + assert(_test_tmp_path is not None) + + if use_tmp: + path = _test_tmp_path # use temporary stage + else: + path = spack.stage_path # Use Spack's stage dir (no links) + spack.config.update_config( 'config', {'build_stage': [path]}, scope='user') + yield @@ -59,13 +66,28 @@ class StageTest(MockPackagesTest): by the Stage class. It doesn't actually create the Stage -- that is done by individual tests. """ + super(StageTest, self).setUp() + global _test_tmp_path + # + # Mock up a stage area that looks like this: + # + # TMPDIR/ test_files_dir + # tmp/ test_tmp_path (where stage should be) + # test-files/ archive_dir_path + # README.txt test_readme (contains "hello world!\n") + # test-files.tar.gz archive_url = file:///path/to/this + # self.test_files_dir = tempfile.mkdtemp() self.test_tmp_path = os.path.realpath( os.path.join(self.test_files_dir, 'tmp')) _test_tmp_path = self.test_tmp_path + # set _test_tmp_path as the default test directory to use for stages. + spack.config.update_config( + 'config', {'build_stage': [_test_tmp_path]}, scope='user') + self.archive_dir = 'test-files' self.archive_name = self.archive_dir + '.tar.gz' archive_dir_path = os.path.join(self.test_files_dir, @@ -99,6 +121,8 @@ class StageTest(MockPackagesTest): def tearDown(self): """Blows away the test environment directory.""" + super(StageTest, self).tearDown() + shutil.rmtree(self.test_files_dir, ignore_errors=True) # chdir back to original working dir @@ -138,7 +162,7 @@ class StageTest(MockPackagesTest): self.assertFalse(os.path.islink(target)) # Make sure the directory is in the place we asked it to - # be (see setUp and tearDown) + # be (see setUp, tearDown, and use_tmp) self.assertTrue(target.startswith(self.test_tmp_path)) else: diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 5332115ae9..7235f6b756 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -50,18 +50,22 @@ def substitute_config_variables(path): - $spack The Spack instance's prefix - $user The current user's username - $tempdir Default temporary directory returned by tempfile.gettempdir() + + These are substituted case-insensitively into the path, and users can + use either ``$var`` or ``${var}`` syntax for the variables. + """ # Look up replacements for re.sub in the replacements dict. def repl(match): m = match.group(0).strip('${}') - return replacements.get(m, match.group(0)) + return replacements.get(m.lower(), match.group(0)) # Replace $var or ${var}. return re.sub(r'(\$\w+\b|\$\{\w+\})', repl, path) def canonicalize_path(path): - """Substitute $spack, expand user home, take abspath.""" + """Substitute config vars, expand user home, take abspath.""" path = substitute_config_variables(path) path = os.path.expanduser(path) path = os.path.abspath(path) -- cgit v1.2.3-60-g2f50 From 4e6d5350588ab2435afa4393c75190ec0e9c031c Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 4 Nov 2016 19:32:25 -0500 Subject: Set Clang as the default compiler on macOS (#2225) * Set OS-specific default compilers * Fix flake8 warnings --- etc/spack/defaults/darwin/packages.yaml | 18 ++++++++++++++++++ etc/spack/defaults/packages.yaml | 1 + lib/spack/spack/compilers/__init__.py | 19 ------------------- lib/spack/spack/preferred_packages.py | 6 ------ 4 files changed, 19 insertions(+), 25 deletions(-) create mode 100644 etc/spack/defaults/darwin/packages.yaml (limited to 'etc') diff --git a/etc/spack/defaults/darwin/packages.yaml b/etc/spack/defaults/darwin/packages.yaml new file mode 100644 index 0000000000..24a08809db --- /dev/null +++ b/etc/spack/defaults/darwin/packages.yaml @@ -0,0 +1,18 @@ +# ------------------------------------------------------------------------- +# This file controls default concretization preferences for Spack. +# +# Settings here are versioned with Spack and are intended to provide +# sensible defaults out of the box. Spack maintainers should edit this +# file to keep it current. +# +# Users can override these settings by editing the following files. +# +# Per-spack-instance settings (overrides defaults): +# $SPACK_ROOT/etc/spack/packages.yaml +# +# Per-user settings (overrides default and site settings): +# ~/.spack/packages.yaml +# ------------------------------------------------------------------------- +packages: + all: + compiler: [clang, gcc, intel] diff --git a/etc/spack/defaults/packages.yaml b/etc/spack/defaults/packages.yaml index a6b361d908..eae7752eee 100644 --- a/etc/spack/defaults/packages.yaml +++ b/etc/spack/defaults/packages.yaml @@ -15,6 +15,7 @@ # ------------------------------------------------------------------------- packages: all: + compiler: [gcc, intel, pgi, clang, xl, nag] providers: mpi: [openmpi, mpich] blas: [openblas] diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 0db632a880..72f8532d3e 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -26,7 +26,6 @@ system and configuring Spack to use multiple compilers. """ import imp -import platform from llnl.util.lang import list_modules from llnl.util.filesystem import join_path @@ -44,12 +43,6 @@ _path_instance_vars = ['cc', 'cxx', 'f77', 'fc'] _other_instance_vars = ['modules', 'operating_system'] _cache_config_file = [] -# TODO: customize order in config file -if platform.system() == 'Darwin': - _default_order = ['clang', 'gcc', 'intel'] -else: - _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc', 'nag'] - def _auto_compiler_spec(function): def converter(cspec_like, *args, **kwargs): @@ -169,18 +162,6 @@ def all_compilers(scope=None, init_config=True): for s in all_compilers_config(scope, init_config)] -def default_compiler(): - versions = [] - for name in _default_order: - versions = find(name) - if versions: - break - else: - raise NoCompilersError() - - return sorted(versions)[-1] - - def find_compilers(*paths): """Return a list of compilers found in the suppied paths. This invokes the find_compilers() method for each operating diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index 45a41c8e2b..08f9c3cfa8 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -28,9 +28,6 @@ from spack.version import * class PreferredPackages(object): - # Arbitrary, but consistent - _default_order = {'compiler': ['gcc', 'intel', 'clang', 'pgi', 'xlc']} - def __init__(self): self.preferred = spack.config.get_config('packages') self._spec_for_pkgname_cache = {} @@ -128,9 +125,6 @@ class PreferredPackages(object): key = (pkgname, component, second_key) if key not in self._spec_for_pkgname_cache: pkglist = self._order_for_package(pkgname, component, second_key) - if not pkglist: - if component in self._default_order: - pkglist = self._default_order[component] if component == 'compiler': self._spec_for_pkgname_cache[key] = \ [spack.spec.CompilerSpec(s) for s in pkglist] -- cgit v1.2.3-60-g2f50 From e0f5b12475810419863c23de16661caa78823823 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 7 Dec 2016 13:53:57 -0800 Subject: add ACLOCAL_PATH updates for packages that register m4 macros (#2518) --- etc/spack/defaults/modules.yaml | 2 ++ 1 file changed, 2 insertions(+) (limited to 'etc') diff --git a/etc/spack/defaults/modules.yaml b/etc/spack/defaults/modules.yaml index 8864a76547..25fe2088e7 100644 --- a/etc/spack/defaults/modules.yaml +++ b/etc/spack/defaults/modules.yaml @@ -24,6 +24,8 @@ modules: - MANPATH share/man: - MANPATH + share/aclocal: + - ACLOCAL_PATH lib: - LIBRARY_PATH - LD_LIBRARY_PATH -- cgit v1.2.3-60-g2f50