diff options
| -rw-r--r-- | lib/spack/llnl/util/filesystem.py | 82 | ||||
| -rw-r--r-- | lib/spack/spack/__init__.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/directory_layout.py | 55 | ||||
| -rw-r--r-- | lib/spack/spack/hooks/extensions.py | 49 | ||||
| -rw-r--r-- | lib/spack/spack/package.py | 85 | ||||
| -rw-r--r-- | lib/spack/spack/relations.py | 4 | ||||
| -rw-r--r-- | var/spack/packages/python/package.py | 3 |
7 files changed, 261 insertions, 19 deletions
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 0578415653..9fb76d3a35 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -24,7 +24,8 @@ ############################################################################## __all__ = ['set_install_permissions', 'install', 'expand_user', 'working_dir', 'touch', 'mkdirp', 'force_remove', 'join_path', 'ancestor', - 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe'] + 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe', + 'check_link_tree', 'merge_link_tree', 'unmerge_link_tree'] import os import sys @@ -222,3 +223,82 @@ def ancestor(dir, n=1): def can_access(file_name): """True if we have read/write access to the file.""" return os.access(file_name, os.R_OK|os.W_OK) + + +def traverse_link_tree(src_root, dest_root, follow_nonexisting=True, **kwargs): + # Yield directories before or after their contents. + order = kwargs.get('order', 'pre') + if order not in ('pre', 'post'): + raise ValueError("Order must be 'pre' or 'post'.") + + # List of relative paths to ignore under the src root. + ignore = kwargs.get('ignore', None) + if isinstance(ignore, basestring): + ignore = (ignore,) + + for dirpath, dirnames, filenames in os.walk(src_root): + rel_path = dirpath[len(src_root):] + rel_path = rel_path.lstrip(os.path.sep) + dest_dirpath = os.path.join(dest_root, rel_path) + + # Don't descend into ignored directories + if ignore and dest_dirpath in ignore: + return + + # Don't descend into dirs in dest that do not exist in src. + if not follow_nonexisting: + dirnames[:] = [ + d for d in dirnames + if os.path.exists(os.path.join(dest_dirpath, d))] + + # preorder yields directories before children + if order == 'pre': + yield (dirpath, dest_dirpath) + + for name in filenames: + src_file = os.path.join(dirpath, name) + dest_file = os.path.join(dest_dirpath, name) + + # Ignore particular paths inside the install root. + src_relpath = src_file[len(src_root):] + src_relpath = src_relpath.lstrip(os.path.sep) + if ignore and src_relpath in ignore: + continue + + yield (src_file, dest_file) + + # postorder yields directories after children + if order == 'post': + yield (dirpath, dest_dirpath) + + + +def check_link_tree(src_root, dest_root, **kwargs): + for src, dest in traverse_link_tree(src_root, dest_root, False, **kwargs): + if os.path.exists(dest) and not os.path.isdir(dest): + return dest + return None + + +def merge_link_tree(src_root, dest_root, **kwargs): + kwargs['order'] = 'pre' + for src, dest in traverse_link_tree(src_root, dest_root, **kwargs): + if os.path.isdir(src): + mkdirp(dest) + else: + assert(not os.path.exists(dest)) + os.symlink(src, dest) + + +def unmerge_link_tree(src_root, dest_root, **kwargs): + kwargs['order'] = 'post' + for src, dest in traverse_link_tree(src_root, dest_root, **kwargs): + if os.path.isdir(dest): + if not os.listdir(dest): + # TODO: what if empty directories were present pre-merge? + shutil.rmtree(dest, ignore_errors=True) + + elif os.path.exists(dest): + if not os.path.islink(dest): + raise ValueError("%s is not a link tree!" % dest) + os.remove(dest) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 6697e00e40..6763411f7d 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -138,7 +138,7 @@ sys_type = None # should live. This file is overloaded for spack core vs. for packages. # __all__ = ['Package', 'Version', 'when', 'ver'] -from spack.package import Package +from spack.package import Package, ExtensionConflictError from spack.version import Version, ver from spack.multimethod import when diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 4ab9a515cf..ff327ed504 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -53,6 +53,19 @@ class DirectoryLayout(object): self.root = root + @property + def hidden_file_paths(self): + """Return a list of hidden files used by the directory layout. + + Paths are relative to the root of an install directory. + + If the directory layout uses no hidden files to maintain + state, this should return an empty container, e.g. [] or (,). + + """ + raise NotImplementedError() + + def all_specs(self): """To be implemented by subclasses to traverse all specs for which there is a directory within the root. @@ -156,6 +169,11 @@ class SpecHashDirectoryLayout(DirectoryLayout): self.extension_file_name = extension_file_name + @property + def hidden_file_paths(self): + return ('.spec', '.extensions') + + def relative_path_for_spec(self, spec): _check_concrete(spec) dir_name = spec.format('$_$@$+$#') @@ -249,28 +267,32 @@ class SpecHashDirectoryLayout(DirectoryLayout): def get_extensions(self, spec): - path = self.extension_file_path(spec) + _check_concrete(spec) + path = self.extension_file_path(spec) extensions = set() if os.path.exists(path): - with closing(open(path)) as spec_file: - for line in spec_file: + with closing(open(path)) as ext_file: + for line in ext_file: try: - extensions.add(Spec(line)) - except SpecError, e: + extensions.add(Spec(line.strip())) + except spack.error.SpackError, e: raise InvalidExtensionSpecError(str(e)) return extensions - def write_extensions(self, extensions): + def write_extensions(self, spec, extensions): path = self.extension_file_path(spec) with closing(open(path, 'w')) as spec_file: for extension in sorted(extensions): - spec_file.write("%s\n" % extensions) + spec_file.write("%s\n" % extension) def add_extension(self, spec, extension_spec): - exts = get_extensions(spec) + _check_concrete(spec) + _check_concrete(extension_spec) + + exts = self.get_extensions(spec) if extension_spec in exts: raise ExtensionAlreadyInstalledError(spec, extension_spec) else: @@ -279,16 +301,19 @@ class SpecHashDirectoryLayout(DirectoryLayout): raise ExtensionConflictError(spec, extension_spec, already_installed) exts.add(extension_spec) - self.write_extensions(exts) + self.write_extensions(spec, exts) def remove_extension(self, spec, extension_spec): - exts = get_extensions(spec) + _check_concrete(spec) + _check_concrete(extension_spec) + + exts = self.get_extensions(spec) if not extension_spec in exts: raise NoSuchExtensionError(spec, extension_spec) exts.remove(extension_spec) - self.write_extensions(exts) + self.write_extensions(spec, exts) class DirectoryLayoutError(SpackError): @@ -328,7 +353,7 @@ class ExtensionAlreadyInstalledError(DirectoryLayoutError): """Raised when an extension is added to a package that already has it.""" def __init__(self, spec, extension_spec): super(ExtensionAlreadyInstalledError, self).__init__( - "%s is already installed in %s" % (extension_spec, spec)) + "%s is already installed in %s" % (extension_spec.short_spec, spec.short_spec)) class ExtensionConflictError(DirectoryLayoutError): @@ -336,12 +361,12 @@ class ExtensionConflictError(DirectoryLayoutError): def __init__(self, spec, extension_spec, conflict): super(ExtensionConflictError, self).__init__( "%s cannot be installed in %s because it conflicts with %s."% ( - extension_spec, spec, conflict)) + extension_spec.short_spec, spec.short_spec, conflict.short_spec)) class NoSuchExtensionError(DirectoryLayoutError): """Raised when an extension isn't there on remove.""" def __init__(self, spec, extension_spec): super(NoSuchExtensionError, self).__init__( - "%s cannot be removed from %s beacuse it's not installed."% ( - extension_spec, spec, conflict)) + "%s cannot be removed from %s because it's not installed."% ( + extension_spec.short_spec, spec.short_spec)) diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py new file mode 100644 index 0000000000..444472bffa --- /dev/null +++ b/lib/spack/spack/hooks/extensions.py @@ -0,0 +1,49 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import spack + + +def post_install(pkg): + assert(pkg.spec.concrete) + for name, spec in pkg.extendees.items(): + ext = pkg.spec[name] + epkg = ext.package + if epkg.installed: + epkg.do_activate(pkg) + + +def pre_uninstall(pkg): + assert(pkg.spec.concrete) + + # Need to do this b/c uninstall does not automatically do it. + # TODO: store full graph info in stored .spec file. + pkg.spec.normalize() + + for name, spec in pkg.extendees.items(): + ext = pkg.spec[name] + epkg = ext.package + if epkg.installed: + epkg.do_deactivate(pkg) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 04f0d842da..b7dae552e4 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -329,6 +329,9 @@ class Package(object): """By default we build in parallel. Subclasses can override this.""" parallel = True + """Most packages are NOT extendable. Set to True if you want extensions.""" + extendable = False + def __init__(self, spec): # this determines how the package should be built. @@ -398,6 +401,9 @@ class Package(object): self._fetch_time = 0.0 self._total_time = 0.0 + for name, spec in self.extendees.items(): + spack.db.get(spec)._check_extendable() + @property def version(self): @@ -877,6 +883,79 @@ class Package(object): spack.hooks.post_uninstall(self) + def _check_extendable(self): + if not self.extendable: + raise ValueError("Package %s is not extendable!" % self.name) + + + def _sanity_check_extension(self, extension): + self._check_extendable() + if not self.installed: + raise ValueError("Can only (de)activate extensions for installed packages.") + if not extension.installed: + raise ValueError("Extensions must first be installed.") + if not self.name in extension.extendees: + raise ValueError("%s does not extend %s!" % (extension.name, self.name)) + if not self.spec.satisfies(extension.extendees[self.name]): + raise ValueError("%s does not satisfy %s!" % (self.spec, extension.spec)) + + + def do_activate(self, extension): + self._sanity_check_extension(extension) + + self.activate(extension) + spack.install_layout.add_extension(self.spec, extension.spec) + tty.msg("Activated extension %s for %s." + % (extension.spec.short_spec, self.spec.short_spec)) + + + def activate(self, extension): + """Symlinks all files from the extension into extendee's install dir. + + Package authors can override this method to support other + extension mechanisms. Spack internals (commands, hooks, etc.) + should call do_activate() method so that proper checks are + always executed. + + """ + conflict = check_link_tree( + extension.prefix, self.prefix, + ignore=spack.install_layout.hidden_file_paths) + + if conflict: + raise ExtensionConflictError(conflict) + + merge_link_tree(extension.prefix, self.prefix, + ignore=spack.install_layout.hidden_file_paths) + + + def do_deactivate(self, extension): + self._sanity_check_extension(extension) + self.deactivate(extension) + + ext = extension.spec + if ext in spack.install_layout.get_extensions(self.spec): + spack.install_layout.remove_extension(self.spec, ext) + + tty.msg("Deactivated extension %s for %s." + % (extension.spec.short_spec, self.spec.short_spec)) + + + def deactivate(self, extension): + """Unlinks all files from extension out of extendee's install dir. + + Package authors can override this method to support other + extension mechanisms. Spack internals (commands, hooks, etc.) + should call do_deactivate() method so that proper checks are + always executed. + + """ + unmerge_link_tree(extension.prefix, self.prefix, + ignore=spack.install_layout.hidden_file_paths) + tty.msg("Deactivated %s as extension of %s." + % (extension.spec.short_spec, self.spec.short_spec)) + + def do_clean(self): if self.stage.expanded_archive_path: self.stage.chdir_to_source() @@ -1068,3 +1147,9 @@ class NoURLError(PackageError): def __init__(self, cls): super(NoURLError, self).__init__( "Package %s has no version with a URL." % cls.__name__) + + +class ExtensionConflictError(PackageError): + def __init__(self, path): + super(ExtensionConflictError, self).__init__( + "Extension blocked by file: %s" % path) diff --git a/lib/spack/spack/relations.py b/lib/spack/spack/relations.py index aaca9c199e..17bec1664f 100644 --- a/lib/spack/spack/relations.py +++ b/lib/spack/spack/relations.py @@ -68,7 +68,7 @@ provides spack install mpileaks ^mvapich spack install mpileaks ^mpich """ -__all__ = [ 'depends_on', 'provides', 'patch', 'version' ] +__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version' ] import re import inspect @@ -135,7 +135,7 @@ def extends(*specs): for string in specs: for spec in spack.spec.parse(string): if pkg == spec.name: - raise CircularReferenceError('depends_on', pkg) + raise CircularReferenceError('extends', pkg) dependencies[spec.name] = spec extendees[spec.name] = spec diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index e6c3e28820..953be69cc2 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -1,10 +1,13 @@ from spack import * + class Python(Package): """The Python programming language.""" homepage = "http://www.python.org" url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tar.xz" + extendable = True + version('2.7.8', 'd235bdfa75b8396942e360a70487ee00') depends_on("openssl") |