From 99775434785779d223997a9e41972da470214e5d Mon Sep 17 00:00:00 2001
From: Todd Gamblin <tgamblin@llnl.gov>
Date: Wed, 7 Jan 2015 11:48:21 -0800
Subject: Added feature: package extensions
- packages can be "extended" by others
- allows extension to be symlinked into extendee's prefix.
- used for python modules.
- first module: py-setuptools
---
lib/spack/llnl/util/filesystem.py | 82 ++++++++++++++++++++++++++++++++++-
lib/spack/spack/__init__.py | 2 +-
lib/spack/spack/directory_layout.py | 55 +++++++++++++++++-------
lib/spack/spack/hooks/extensions.py | 49 +++++++++++++++++++++
lib/spack/spack/package.py | 85 +++++++++++++++++++++++++++++++++++++
lib/spack/spack/relations.py | 4 +-
6 files changed, 258 insertions(+), 19 deletions(-)
create mode 100644 lib/spack/spack/hooks/extensions.py
(limited to 'lib')
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index 0578415653..9fb76d3a35 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -24,7 +24,8 @@
##############################################################################
__all__ = ['set_install_permissions', 'install', 'expand_user', 'working_dir',
'touch', 'mkdirp', 'force_remove', 'join_path', 'ancestor',
- 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe']
+ 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe',
+ 'check_link_tree', 'merge_link_tree', 'unmerge_link_tree']
import os
import sys
@@ -222,3 +223,82 @@ def ancestor(dir, n=1):
def can_access(file_name):
"""True if we have read/write access to the file."""
return os.access(file_name, os.R_OK|os.W_OK)
+
+
+def traverse_link_tree(src_root, dest_root, follow_nonexisting=True, **kwargs):
+ # Yield directories before or after their contents.
+ order = kwargs.get('order', 'pre')
+ if order not in ('pre', 'post'):
+ raise ValueError("Order must be 'pre' or 'post'.")
+
+ # List of relative paths to ignore under the src root.
+ ignore = kwargs.get('ignore', None)
+ if isinstance(ignore, basestring):
+ ignore = (ignore,)
+
+ for dirpath, dirnames, filenames in os.walk(src_root):
+ rel_path = dirpath[len(src_root):]
+ rel_path = rel_path.lstrip(os.path.sep)
+ dest_dirpath = os.path.join(dest_root, rel_path)
+
+ # Don't descend into ignored directories
+ if ignore and dest_dirpath in ignore:
+ return
+
+ # Don't descend into dirs in dest that do not exist in src.
+ if not follow_nonexisting:
+ dirnames[:] = [
+ d for d in dirnames
+ if os.path.exists(os.path.join(dest_dirpath, d))]
+
+ # preorder yields directories before children
+ if order == 'pre':
+ yield (dirpath, dest_dirpath)
+
+ for name in filenames:
+ src_file = os.path.join(dirpath, name)
+ dest_file = os.path.join(dest_dirpath, name)
+
+ # Ignore particular paths inside the install root.
+ src_relpath = src_file[len(src_root):]
+ src_relpath = src_relpath.lstrip(os.path.sep)
+ if ignore and src_relpath in ignore:
+ continue
+
+ yield (src_file, dest_file)
+
+ # postorder yields directories after children
+ if order == 'post':
+ yield (dirpath, dest_dirpath)
+
+
+
+def check_link_tree(src_root, dest_root, **kwargs):
+ for src, dest in traverse_link_tree(src_root, dest_root, False, **kwargs):
+ if os.path.exists(dest) and not os.path.isdir(dest):
+ return dest
+ return None
+
+
+def merge_link_tree(src_root, dest_root, **kwargs):
+ kwargs['order'] = 'pre'
+ for src, dest in traverse_link_tree(src_root, dest_root, **kwargs):
+ if os.path.isdir(src):
+ mkdirp(dest)
+ else:
+ assert(not os.path.exists(dest))
+ os.symlink(src, dest)
+
+
+def unmerge_link_tree(src_root, dest_root, **kwargs):
+ kwargs['order'] = 'post'
+ for src, dest in traverse_link_tree(src_root, dest_root, **kwargs):
+ if os.path.isdir(dest):
+ if not os.listdir(dest):
+ # TODO: what if empty directories were present pre-merge?
+ shutil.rmtree(dest, ignore_errors=True)
+
+ elif os.path.exists(dest):
+ if not os.path.islink(dest):
+ raise ValueError("%s is not a link tree!" % dest)
+ os.remove(dest)
diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index 6697e00e40..6763411f7d 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -138,7 +138,7 @@ sys_type = None
# should live. This file is overloaded for spack core vs. for packages.
#
__all__ = ['Package', 'Version', 'when', 'ver']
-from spack.package import Package
+from spack.package import Package, ExtensionConflictError
from spack.version import Version, ver
from spack.multimethod import when
diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py
index 4ab9a515cf..ff327ed504 100644
--- a/lib/spack/spack/directory_layout.py
+++ b/lib/spack/spack/directory_layout.py
@@ -53,6 +53,19 @@ class DirectoryLayout(object):
self.root = root
+ @property
+ def hidden_file_paths(self):
+ """Return a list of hidden files used by the directory layout.
+
+ Paths are relative to the root of an install directory.
+
+ If the directory layout uses no hidden files to maintain
+ state, this should return an empty container, e.g. [] or (,).
+
+ """
+ raise NotImplementedError()
+
+
def all_specs(self):
"""To be implemented by subclasses to traverse all specs for which there is
a directory within the root.
@@ -156,6 +169,11 @@ class SpecHashDirectoryLayout(DirectoryLayout):
self.extension_file_name = extension_file_name
+ @property
+ def hidden_file_paths(self):
+ return ('.spec', '.extensions')
+
+
def relative_path_for_spec(self, spec):
_check_concrete(spec)
dir_name = spec.format('$_$@$+$#')
@@ -249,28 +267,32 @@ class SpecHashDirectoryLayout(DirectoryLayout):
def get_extensions(self, spec):
- path = self.extension_file_path(spec)
+ _check_concrete(spec)
+ path = self.extension_file_path(spec)
extensions = set()
if os.path.exists(path):
- with closing(open(path)) as spec_file:
- for line in spec_file:
+ with closing(open(path)) as ext_file:
+ for line in ext_file:
try:
- extensions.add(Spec(line))
- except SpecError, e:
+ extensions.add(Spec(line.strip()))
+ except spack.error.SpackError, e:
raise InvalidExtensionSpecError(str(e))
return extensions
- def write_extensions(self, extensions):
+ def write_extensions(self, spec, extensions):
path = self.extension_file_path(spec)
with closing(open(path, 'w')) as spec_file:
for extension in sorted(extensions):
- spec_file.write("%s\n" % extensions)
+ spec_file.write("%s\n" % extension)
def add_extension(self, spec, extension_spec):
- exts = get_extensions(spec)
+ _check_concrete(spec)
+ _check_concrete(extension_spec)
+
+ exts = self.get_extensions(spec)
if extension_spec in exts:
raise ExtensionAlreadyInstalledError(spec, extension_spec)
else:
@@ -279,16 +301,19 @@ class SpecHashDirectoryLayout(DirectoryLayout):
raise ExtensionConflictError(spec, extension_spec, already_installed)
exts.add(extension_spec)
- self.write_extensions(exts)
+ self.write_extensions(spec, exts)
def remove_extension(self, spec, extension_spec):
- exts = get_extensions(spec)
+ _check_concrete(spec)
+ _check_concrete(extension_spec)
+
+ exts = self.get_extensions(spec)
if not extension_spec in exts:
raise NoSuchExtensionError(spec, extension_spec)
exts.remove(extension_spec)
- self.write_extensions(exts)
+ self.write_extensions(spec, exts)
class DirectoryLayoutError(SpackError):
@@ -328,7 +353,7 @@ class ExtensionAlreadyInstalledError(DirectoryLayoutError):
"""Raised when an extension is added to a package that already has it."""
def __init__(self, spec, extension_spec):
super(ExtensionAlreadyInstalledError, self).__init__(
- "%s is already installed in %s" % (extension_spec, spec))
+ "%s is already installed in %s" % (extension_spec.short_spec, spec.short_spec))
class ExtensionConflictError(DirectoryLayoutError):
@@ -336,12 +361,12 @@ class ExtensionConflictError(DirectoryLayoutError):
def __init__(self, spec, extension_spec, conflict):
super(ExtensionConflictError, self).__init__(
"%s cannot be installed in %s because it conflicts with %s."% (
- extension_spec, spec, conflict))
+ extension_spec.short_spec, spec.short_spec, conflict.short_spec))
class NoSuchExtensionError(DirectoryLayoutError):
"""Raised when an extension isn't there on remove."""
def __init__(self, spec, extension_spec):
super(NoSuchExtensionError, self).__init__(
- "%s cannot be removed from %s beacuse it's not installed."% (
- extension_spec, spec, conflict))
+ "%s cannot be removed from %s because it's not installed."% (
+ extension_spec.short_spec, spec.short_spec))
diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py
new file mode 100644
index 0000000000..444472bffa
--- /dev/null
+++ b/lib/spack/spack/hooks/extensions.py
@@ -0,0 +1,49 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+import spack
+
+
+def post_install(pkg):
+ assert(pkg.spec.concrete)
+ for name, spec in pkg.extendees.items():
+ ext = pkg.spec[name]
+ epkg = ext.package
+ if epkg.installed:
+ epkg.do_activate(pkg)
+
+
+def pre_uninstall(pkg):
+ assert(pkg.spec.concrete)
+
+ # Need to do this b/c uninstall does not automatically do it.
+ # TODO: store full graph info in stored .spec file.
+ pkg.spec.normalize()
+
+ for name, spec in pkg.extendees.items():
+ ext = pkg.spec[name]
+ epkg = ext.package
+ if epkg.installed:
+ epkg.do_deactivate(pkg)
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 04f0d842da..b7dae552e4 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -329,6 +329,9 @@ class Package(object):
"""By default we build in parallel. Subclasses can override this."""
parallel = True
+ """Most packages are NOT extendable. Set to True if you want extensions."""
+ extendable = False
+
def __init__(self, spec):
# this determines how the package should be built.
@@ -398,6 +401,9 @@ class Package(object):
self._fetch_time = 0.0
self._total_time = 0.0
+ for name, spec in self.extendees.items():
+ spack.db.get(spec)._check_extendable()
+
@property
def version(self):
@@ -877,6 +883,79 @@ class Package(object):
spack.hooks.post_uninstall(self)
+ def _check_extendable(self):
+ if not self.extendable:
+ raise ValueError("Package %s is not extendable!" % self.name)
+
+
+ def _sanity_check_extension(self, extension):
+ self._check_extendable()
+ if not self.installed:
+ raise ValueError("Can only (de)activate extensions for installed packages.")
+ if not extension.installed:
+ raise ValueError("Extensions must first be installed.")
+ if not self.name in extension.extendees:
+ raise ValueError("%s does not extend %s!" % (extension.name, self.name))
+ if not self.spec.satisfies(extension.extendees[self.name]):
+ raise ValueError("%s does not satisfy %s!" % (self.spec, extension.spec))
+
+
+ def do_activate(self, extension):
+ self._sanity_check_extension(extension)
+
+ self.activate(extension)
+ spack.install_layout.add_extension(self.spec, extension.spec)
+ tty.msg("Activated extension %s for %s."
+ % (extension.spec.short_spec, self.spec.short_spec))
+
+
+ def activate(self, extension):
+ """Symlinks all files from the extension into extendee's install dir.
+
+ Package authors can override this method to support other
+ extension mechanisms. Spack internals (commands, hooks, etc.)
+ should call do_activate() method so that proper checks are
+ always executed.
+
+ """
+ conflict = check_link_tree(
+ extension.prefix, self.prefix,
+ ignore=spack.install_layout.hidden_file_paths)
+
+ if conflict:
+ raise ExtensionConflictError(conflict)
+
+ merge_link_tree(extension.prefix, self.prefix,
+ ignore=spack.install_layout.hidden_file_paths)
+
+
+ def do_deactivate(self, extension):
+ self._sanity_check_extension(extension)
+ self.deactivate(extension)
+
+ ext = extension.spec
+ if ext in spack.install_layout.get_extensions(self.spec):
+ spack.install_layout.remove_extension(self.spec, ext)
+
+ tty.msg("Deactivated extension %s for %s."
+ % (extension.spec.short_spec, self.spec.short_spec))
+
+
+ def deactivate(self, extension):
+ """Unlinks all files from extension out of extendee's install dir.
+
+ Package authors can override this method to support other
+ extension mechanisms. Spack internals (commands, hooks, etc.)
+ should call do_deactivate() method so that proper checks are
+ always executed.
+
+ """
+ unmerge_link_tree(extension.prefix, self.prefix,
+ ignore=spack.install_layout.hidden_file_paths)
+ tty.msg("Deactivated %s as extension of %s."
+ % (extension.spec.short_spec, self.spec.short_spec))
+
+
def do_clean(self):
if self.stage.expanded_archive_path:
self.stage.chdir_to_source()
@@ -1068,3 +1147,9 @@ class NoURLError(PackageError):
def __init__(self, cls):
super(NoURLError, self).__init__(
"Package %s has no version with a URL." % cls.__name__)
+
+
+class ExtensionConflictError(PackageError):
+ def __init__(self, path):
+ super(ExtensionConflictError, self).__init__(
+ "Extension blocked by file: %s" % path)
diff --git a/lib/spack/spack/relations.py b/lib/spack/spack/relations.py
index aaca9c199e..17bec1664f 100644
--- a/lib/spack/spack/relations.py
+++ b/lib/spack/spack/relations.py
@@ -68,7 +68,7 @@ provides
spack install mpileaks ^mvapich
spack install mpileaks ^mpich
"""
-__all__ = [ 'depends_on', 'provides', 'patch', 'version' ]
+__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version' ]
import re
import inspect
@@ -135,7 +135,7 @@ def extends(*specs):
for string in specs:
for spec in spack.spec.parse(string):
if pkg == spec.name:
- raise CircularReferenceError('depends_on', pkg)
+ raise CircularReferenceError('extends', pkg)
dependencies[spec.name] = spec
extendees[spec.name] = spec
--
cgit v1.2.3-70-g09d2