From e6b4530234dc9c32fc69a0dae21cf8c28ce33804 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 22 Jan 2015 13:52:28 -0800 Subject: Add is_exe function to filesystem. --- lib/spack/llnl/util/filesystem.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 9f08832598..0578415653 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -24,7 +24,7 @@ ############################################################################## __all__ = ['set_install_permissions', 'install', 'expand_user', 'working_dir', 'touch', 'mkdirp', 'force_remove', 'join_path', 'ancestor', - 'can_access', 'filter_file', 'change_sed_delimiter'] + 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe'] import os import sys @@ -154,6 +154,11 @@ def install(src, dest): os.chmod(dest, dest_mode) +def is_exe(path): + """True if path is an executable file.""" + return os.path.isfile(path) and os.access(path, os.X_OK) + + def expand_user(path): """Find instances of '%u' in a path and replace with the current user's username.""" -- cgit v1.2.3-70-g09d2 From 3e37903ffdbc72410264d1949dde9239e81ec698 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 22 Jan 2015 23:34:39 -0800 Subject: Packages have rpath property. --- lib/spack/spack/package.py | 31 +++++++++++++++++++++++++++---- var/spack/packages/gcc/package.py | 29 +++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 1a797e88b1..fe4fc748d8 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -713,6 +713,14 @@ class Package(object): tty.msg("Patched %s" % self.name) + def do_fake_install(self): + """Make a fake install directory contaiing a 'fake' file in bin.""" + mkdirp(self.prefix.bin) + touch(join_path(self.prefix.bin, 'fake')) + mkdirp(self.prefix.lib) + mkdirp(self.prefix.man1) + + def do_install(self, **kwargs): """This class should call this version of the install method. Package implementations should override install(). @@ -758,13 +766,11 @@ class Package(object): spack.install_layout.make_path_for_spec(self.spec) # Set up process's build environment before running install. + self.stage.chdir_to_source() build_env.setup_package(self) if fake_install: - mkdirp(self.prefix.bin) - touch(join_path(self.prefix.bin, 'fake')) - mkdirp(self.prefix.lib) - mkdirp(self.prefix.man1) + self.do_fake_install() else: # Subclasses implement install() to do the real work. self.install(self.spec, self.prefix) @@ -925,6 +931,23 @@ class Package(object): e.url, e.message) + @property + def rpath(self): + """Get the rpath this package links with, as a list of paths.""" + rpaths = [self.prefix.lib, self.prefix.lib64] + rpaths.extend(d.prefix.lib for d in self.spec.traverse(root=False) + if os.path.isdir(d.prefix.lib)) + rpaths.extend(d.prefix.lib64 for d in self.spec.traverse(root=False) + if os.path.isdir(d.prefix.lib64)) + return rpaths + + + @property + def rpath_args(self): + """Get the rpath args as a string, with -Wl,-rpath= for each element.""" + return " ".join("-Wl,-rpath=%s" % p for p in self.rpath) + + def find_versions_of_archive(*archive_urls, **kwargs): list_url = kwargs.get('list_url', None) list_depth = kwargs.get('list_depth', 1) diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py index da0debd5dc..3da6c25d47 100644 --- a/var/spack/packages/gcc/package.py +++ b/var/spack/packages/gcc/package.py @@ -32,29 +32,46 @@ class Gcc(Package): Objective-C, Fortran, and Java.""" homepage = "https://gcc.gnu.org" + url = "http://open-source-box.org/gcc/gcc-4.9.2/gcc-4.9.2.tar.bz2" list_url = 'http://open-source-box.org/gcc/' list_depth = 2 - version('4.9.2', '4df8ee253b7f3863ad0b86359cd39c43', - url="http://open-source-box.org/gcc/gcc-4.9.2/gcc-4.9.2.tar.bz2") - version('4.9.1', 'fddf71348546af523353bd43d34919c1', - url="http://open-source-box.org/gcc/gcc-4.9.1/gcc-4.9.1.tar.bz2") + version('4.9.2', '4df8ee253b7f3863ad0b86359cd39c43') + version('4.9.1', 'fddf71348546af523353bd43d34919c1') + version('4.8.4', '5a84a30839b2aca22a2d723de2a626ec') + version('4.7.4', '4c696da46297de6ae77a82797d2abe28') + version('4.6.4', 'b407a3d1480c11667f293bfb1f17d1a4') + version('4.5.4', '27e459c2566b8209ab064570e1b378f7') - depends_on("mpc") depends_on("mpfr") depends_on("gmp") + depends_on("mpc") # when @4.5: depends_on("libelf") + # Save these until we can do optional deps. + #depends_on("isl") + #depends_on("ppl") + #depends_on("cloog") def install(self, spec, prefix): # libjava/configure needs a minor fix to install into spack paths. filter_file(r"'@.*@'", "'@[[:alnum:]]*@'", 'libjava/configure', string=True) + enabled_languages = set(('c', 'c++', 'fortran', 'java', 'objc')) + if spec.satisfies("@4.7.1:"): + enabled_languages.add('go') + # Rest of install is straightforward. configure("--prefix=%s" % prefix, "--libdir=%s/lib64" % prefix, "--disable-multilib", - "--enable-languages=c,c++,fortran,java,objc,go", + "--enable-languages=" + ','.join(enabled_languages), + "--with-mpc=%s" % spec['mpc'].prefix, + "--with-mpfr=%s" % spec['mpfr'].prefix, + "--with-gmp=%s" % spec['gmp'].prefix, + "--with-libelf=%s" % spec['libelf'].prefix, + "--with-stage1-ldflags=%s" % self.rpath_args, + "--with-boot-ldflags=%s" % self.rpath_args, "--enable-lto", "--with-quad") make() -- cgit v1.2.3-70-g09d2 From ba593ccb260e46568fc2620ad5f300f89495a604 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 7 Jan 2015 17:41:24 -0500 Subject: Fix bug in mirror path construction. --- lib/spack/spack/mirror.py | 13 ++++++++----- lib/spack/spack/package.py | 2 +- lib/spack/spack/test/mirror.py | 8 ++++---- 3 files changed, 13 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 929c514b61..114c7b6a35 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -46,7 +46,7 @@ from spack.util.compression import extension def mirror_archive_filename(spec): - """Get the path that this spec will live at within a mirror.""" + """Get the name of the spec's archive in the mirror.""" if not spec.version.concrete: raise ValueError("mirror.path requires spec with concrete version.") @@ -61,6 +61,11 @@ def mirror_archive_filename(spec): return "%s-%s.%s" % (spec.package.name, spec.version, ext) +def mirror_archive_path(spec): + """Get the relative path to the spec's archive within a mirror.""" + return join_path(spec.name, mirror_archive_filename(spec)) + + def get_matching_versions(specs, **kwargs): """Get a spec for EACH known version matching any spec in the list.""" matching = [] @@ -141,12 +146,10 @@ def create(path, specs, **kwargs): stage = None try: # create a subdirectory for the current package@version - subdir = join_path(mirror_root, pkg.name) + archive_path = join_path(path, mirror_archive_path(spec)) + subdir = os.path.dirname(archive_path) mkdirp(subdir) - archive_file = mirror_archive_filename(spec) - archive_path = join_path(subdir, archive_file) - if os.path.exists(archive_path): tty.msg("Already added %s" % spec.format("$_$@")) present.append(spec) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index fe4fc748d8..1dfd3d1c83 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -459,7 +459,7 @@ class Package(object): raise ValueError("Can only get a stage for a concrete package.") if self._stage is None: - mp = spack.mirror.mirror_archive_filename(self.spec) + mp = spack.mirror.mirror_archive_path(self.spec) self._stage = Stage( self.fetcher, mirror_path=mp, name=self.spec.short_spec) return self._stage diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 51334198ec..89ab14359e 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -44,7 +44,7 @@ class MirrorTest(MockPackagesTest): self.repos = {} - def set_up_package(self, name, mock_repo_class, url_attr): + def set_up_package(self, name, MockRepoClass, url_attr): """Use this to set up a mock package to be mirrored. Each package needs us to: 1. Set up a mock repo/archive to fetch from. @@ -56,7 +56,7 @@ class MirrorTest(MockPackagesTest): # Get the package and fix its fetch args to point to a mock repo pkg = spack.db.get(spec) - repo = mock_repo_class() + repo = MockRepoClass() self.repos[name] = repo # change the fetch args of the first (only) version. @@ -71,7 +71,7 @@ class MirrorTest(MockPackagesTest): for name, repo in self.repos.items(): if repo.stage: - repo.stage.destroy() + pass #repo.stage.destroy() self.repos.clear() @@ -129,7 +129,7 @@ class MirrorTest(MockPackagesTest): self.assertTrue(all(l in exclude for l in dcmp.left_only)) finally: - stage.destroy() + pass #stage.destroy() def test_git_mirror(self): -- cgit v1.2.3-70-g09d2 From 88afad3e46326c56aaeaed6adc2058033ad7ad33 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 6 Jan 2015 14:47:51 -0500 Subject: Directory layout can now track installed extensions per package. --- lib/spack/spack/directory_layout.py | 97 ++++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 42cac0c9d2..4ab9a515cf 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -71,6 +71,21 @@ class DirectoryLayout(object): raise NotImplementedError() + def get_extensions(self, spec): + """Get a set of currently installed extension packages for a spec.""" + raise NotImplementedError() + + + def add_extension(self, spec, extension_spec): + """Add to the list of currently installed extensions.""" + raise NotImplementedError() + + + def remove_extension(self, spec, extension_spec): + """Remove from the list of currently installed extensions.""" + raise NotImplementedError() + + def path_for_spec(self, spec): """Return an absolute path from the root to a directory for the spec.""" _check_concrete(spec) @@ -134,9 +149,11 @@ class SpecHashDirectoryLayout(DirectoryLayout): """Prefix size is number of characters in the SHA-1 prefix to use to make each hash unique. """ - spec_file_name = kwargs.get('spec_file_name', '.spec') + spec_file_name = kwargs.get('spec_file_name', '.spec') + extension_file_name = kwargs.get('extension_file_name', '.extensions') super(SpecHashDirectoryLayout, self).__init__(root) self.spec_file_name = spec_file_name + self.extension_file_name = extension_file_name def relative_path_for_spec(self, spec): @@ -225,6 +242,55 @@ class SpecHashDirectoryLayout(DirectoryLayout): yield spec + def extension_file_path(self, spec): + """Gets full path to an installed package's extension file""" + _check_concrete(spec) + return join_path(self.path_for_spec(spec), self.extension_file_name) + + + def get_extensions(self, spec): + path = self.extension_file_path(spec) + + extensions = set() + if os.path.exists(path): + with closing(open(path)) as spec_file: + for line in spec_file: + try: + extensions.add(Spec(line)) + except SpecError, e: + raise InvalidExtensionSpecError(str(e)) + return extensions + + + def write_extensions(self, extensions): + path = self.extension_file_path(spec) + with closing(open(path, 'w')) as spec_file: + for extension in sorted(extensions): + spec_file.write("%s\n" % extensions) + + + def add_extension(self, spec, extension_spec): + exts = get_extensions(spec) + if extension_spec in exts: + raise ExtensionAlreadyInstalledError(spec, extension_spec) + else: + for already_installed in exts: + if spec.name == extension_spec.name: + raise ExtensionConflictError(spec, extension_spec, already_installed) + + exts.add(extension_spec) + self.write_extensions(exts) + + + def remove_extension(self, spec, extension_spec): + exts = get_extensions(spec) + if not extension_spec in exts: + raise NoSuchExtensionError(spec, extension_spec) + + exts.remove(extension_spec) + self.write_extensions(exts) + + class DirectoryLayoutError(SpackError): """Superclass for directory layout errors.""" def __init__(self, message): @@ -250,3 +316,32 @@ class InstallDirectoryAlreadyExistsError(DirectoryLayoutError): def __init__(self, path): super(InstallDirectoryAlreadyExistsError, self).__init__( "Install path %s already exists!") + + +class InvalidExtensionSpecError(DirectoryLayoutError): + """Raised when an extension file has a bad spec in it.""" + def __init__(self, message): + super(InvalidExtensionSpecError, self).__init__(message) + + +class ExtensionAlreadyInstalledError(DirectoryLayoutError): + """Raised when an extension is added to a package that already has it.""" + def __init__(self, spec, extension_spec): + super(ExtensionAlreadyInstalledError, self).__init__( + "%s is already installed in %s" % (extension_spec, spec)) + + +class ExtensionConflictError(DirectoryLayoutError): + """Raised when an extension is added to a package that already has it.""" + def __init__(self, spec, extension_spec, conflict): + super(ExtensionConflictError, self).__init__( + "%s cannot be installed in %s because it conflicts with %s."% ( + extension_spec, spec, conflict)) + + +class NoSuchExtensionError(DirectoryLayoutError): + """Raised when an extension isn't there on remove.""" + def __init__(self, spec, extension_spec): + super(NoSuchExtensionError, self).__init__( + "%s cannot be removed from %s beacuse it's not installed."% ( + extension_spec, spec, conflict)) -- cgit v1.2.3-70-g09d2 From ebe0c1d83ac1380a6320a8dadcfa2ad4fc07c279 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 6 Jan 2015 14:49:13 -0500 Subject: New "extends" relation adds another special list to the package class. --- lib/spack/spack/package.py | 3 +++ lib/spack/spack/relations.py | 26 +++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 1dfd3d1c83..c256ea479f 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -314,6 +314,9 @@ class Package(object): """Specs of virtual packages provided by this package, keyed by name.""" provided = {} + """Specs of packages this one extends, keyed by name.""" + extendees = {} + """Specs of conflicting packages, keyed by name. """ conflicted = {} diff --git a/lib/spack/spack/relations.py b/lib/spack/spack/relations.py index b1f4348945..aaca9c199e 100644 --- a/lib/spack/spack/relations.py +++ b/lib/spack/spack/relations.py @@ -107,8 +107,9 @@ def depends_on(*specs): """Adds a dependencies local variable in the locals of the calling class, based on args. """ pkg = get_calling_package_name() + clocals = caller_locals() + dependencies = clocals.setdefault('dependencies', {}) - dependencies = caller_locals().setdefault('dependencies', {}) for string in specs: for spec in spack.spec.parse(string): if pkg == spec.name: @@ -116,6 +117,29 @@ def depends_on(*specs): dependencies[spec.name] = spec +def extends(*specs): + """Same as depends_on, but dependency is symlinked into parent prefix. + + This is for Python and other language modules where the module + needs to be installed into the prefix of the Python installation. + Spack handles this by installing modules into their own prefix, + but allowing ONE module version to be symlinked into a parent + Python install at a time. + + """ + pkg = get_calling_package_name() + clocals = caller_locals() + dependencies = clocals.setdefault('dependencies', {}) + extendees = clocals.setdefault('extendees', {}) + + for string in specs: + for spec in spack.spec.parse(string): + if pkg == spec.name: + raise CircularReferenceError('depends_on', pkg) + dependencies[spec.name] = spec + extendees[spec.name] = spec + + def provides(*specs, **kwargs): """Allows packages to provide a virtual dependency. If a package provides 'mpi', other packages can declare that they depend on "mpi", and spack -- cgit v1.2.3-70-g09d2 From adb7d614e69a0c176c86b3b4aaa1e81d403d0a71 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 6 Jan 2015 14:50:14 -0500 Subject: Add pre-install and pre-uninstall hooks. --- lib/spack/spack/hooks/__init__.py | 7 ++++++- lib/spack/spack/package.py | 8 ++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/hooks/__init__.py b/lib/spack/spack/hooks/__init__.py index 98b7f2323f..1c44e8abaa 100644 --- a/lib/spack/spack/hooks/__init__.py +++ b/lib/spack/spack/hooks/__init__.py @@ -31,7 +31,9 @@ Currently the following hooks are supported: + * pre_install() * post_install() + * pre_uninstall() * post_uninstall() This can be used to implement support for things like module @@ -70,5 +72,8 @@ class HookRunner(object): # # Define some functions that can be called to fire off hooks. # -post_install = HookRunner('post_install') +pre_install = HookRunner('pre_install') +post_install = HookRunner('post_install') + +pre_uninstall = HookRunner('pre_uninstall') post_uninstall = HookRunner('post_uninstall') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index c256ea479f..aa79721266 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -768,6 +768,10 @@ class Package(object): # package naming scheme it likes. spack.install_layout.make_path_for_spec(self.spec) + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) + # Set up process's build environment before running install. self.stage.chdir_to_source() build_env.setup_package(self) @@ -862,6 +866,10 @@ class Package(object): "The following installed packages depend on it: %s" % ' '.join(formatted_deps)) + # Pre-uninstall hook runs first. + spack.hooks.pre_uninstall(self) + + # Uninstalling in Spack only requires removing the prefix. self.remove_prefix() tty.msg("Successfully uninstalled %s." % self.spec.short_spec) -- cgit v1.2.3-70-g09d2 From 7215aee224150d954e8a5bd6b632b6d8f66948d2 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 7 Jan 2015 17:45:14 -0500 Subject: do_install() passes kwargs to dependencies. --- lib/spack/spack/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index aa79721266..04f0d842da 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -744,7 +744,7 @@ class Package(object): tty.msg("Installing %s" % self.name) if not ignore_deps: - self.do_install_dependencies() + self.do_install_dependencies(**kwargs) start_time = time.time() if not fake_install: @@ -832,10 +832,10 @@ class Package(object): spack.hooks.post_install(self) - def do_install_dependencies(self): + def do_install_dependencies(self, **kwargs): # Pass along paths of dependencies here for dep in self.spec.dependencies.values(): - dep.package.do_install() + dep.package.do_install(**kwargs) @property -- cgit v1.2.3-70-g09d2 From 99775434785779d223997a9e41972da470214e5d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 7 Jan 2015 11:48:21 -0800 Subject: Added feature: package extensions - packages can be "extended" by others - allows extension to be symlinked into extendee's prefix. - used for python modules. - first module: py-setuptools --- lib/spack/llnl/util/filesystem.py | 82 +++++++++++++++++++++++++++++++++- lib/spack/spack/__init__.py | 2 +- lib/spack/spack/directory_layout.py | 55 ++++++++++++++++------- lib/spack/spack/hooks/extensions.py | 49 +++++++++++++++++++++ lib/spack/spack/package.py | 85 ++++++++++++++++++++++++++++++++++++ lib/spack/spack/relations.py | 4 +- var/spack/packages/python/package.py | 3 ++ 7 files changed, 261 insertions(+), 19 deletions(-) create mode 100644 lib/spack/spack/hooks/extensions.py (limited to 'lib') diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 0578415653..9fb76d3a35 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -24,7 +24,8 @@ ############################################################################## __all__ = ['set_install_permissions', 'install', 'expand_user', 'working_dir', 'touch', 'mkdirp', 'force_remove', 'join_path', 'ancestor', - 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe'] + 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe', + 'check_link_tree', 'merge_link_tree', 'unmerge_link_tree'] import os import sys @@ -222,3 +223,82 @@ def ancestor(dir, n=1): def can_access(file_name): """True if we have read/write access to the file.""" return os.access(file_name, os.R_OK|os.W_OK) + + +def traverse_link_tree(src_root, dest_root, follow_nonexisting=True, **kwargs): + # Yield directories before or after their contents. + order = kwargs.get('order', 'pre') + if order not in ('pre', 'post'): + raise ValueError("Order must be 'pre' or 'post'.") + + # List of relative paths to ignore under the src root. + ignore = kwargs.get('ignore', None) + if isinstance(ignore, basestring): + ignore = (ignore,) + + for dirpath, dirnames, filenames in os.walk(src_root): + rel_path = dirpath[len(src_root):] + rel_path = rel_path.lstrip(os.path.sep) + dest_dirpath = os.path.join(dest_root, rel_path) + + # Don't descend into ignored directories + if ignore and dest_dirpath in ignore: + return + + # Don't descend into dirs in dest that do not exist in src. + if not follow_nonexisting: + dirnames[:] = [ + d for d in dirnames + if os.path.exists(os.path.join(dest_dirpath, d))] + + # preorder yields directories before children + if order == 'pre': + yield (dirpath, dest_dirpath) + + for name in filenames: + src_file = os.path.join(dirpath, name) + dest_file = os.path.join(dest_dirpath, name) + + # Ignore particular paths inside the install root. + src_relpath = src_file[len(src_root):] + src_relpath = src_relpath.lstrip(os.path.sep) + if ignore and src_relpath in ignore: + continue + + yield (src_file, dest_file) + + # postorder yields directories after children + if order == 'post': + yield (dirpath, dest_dirpath) + + + +def check_link_tree(src_root, dest_root, **kwargs): + for src, dest in traverse_link_tree(src_root, dest_root, False, **kwargs): + if os.path.exists(dest) and not os.path.isdir(dest): + return dest + return None + + +def merge_link_tree(src_root, dest_root, **kwargs): + kwargs['order'] = 'pre' + for src, dest in traverse_link_tree(src_root, dest_root, **kwargs): + if os.path.isdir(src): + mkdirp(dest) + else: + assert(not os.path.exists(dest)) + os.symlink(src, dest) + + +def unmerge_link_tree(src_root, dest_root, **kwargs): + kwargs['order'] = 'post' + for src, dest in traverse_link_tree(src_root, dest_root, **kwargs): + if os.path.isdir(dest): + if not os.listdir(dest): + # TODO: what if empty directories were present pre-merge? + shutil.rmtree(dest, ignore_errors=True) + + elif os.path.exists(dest): + if not os.path.islink(dest): + raise ValueError("%s is not a link tree!" % dest) + os.remove(dest) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 6697e00e40..6763411f7d 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -138,7 +138,7 @@ sys_type = None # should live. This file is overloaded for spack core vs. for packages. # __all__ = ['Package', 'Version', 'when', 'ver'] -from spack.package import Package +from spack.package import Package, ExtensionConflictError from spack.version import Version, ver from spack.multimethod import when diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 4ab9a515cf..ff327ed504 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -53,6 +53,19 @@ class DirectoryLayout(object): self.root = root + @property + def hidden_file_paths(self): + """Return a list of hidden files used by the directory layout. + + Paths are relative to the root of an install directory. + + If the directory layout uses no hidden files to maintain + state, this should return an empty container, e.g. [] or (,). + + """ + raise NotImplementedError() + + def all_specs(self): """To be implemented by subclasses to traverse all specs for which there is a directory within the root. @@ -156,6 +169,11 @@ class SpecHashDirectoryLayout(DirectoryLayout): self.extension_file_name = extension_file_name + @property + def hidden_file_paths(self): + return ('.spec', '.extensions') + + def relative_path_for_spec(self, spec): _check_concrete(spec) dir_name = spec.format('$_$@$+$#') @@ -249,28 +267,32 @@ class SpecHashDirectoryLayout(DirectoryLayout): def get_extensions(self, spec): - path = self.extension_file_path(spec) + _check_concrete(spec) + path = self.extension_file_path(spec) extensions = set() if os.path.exists(path): - with closing(open(path)) as spec_file: - for line in spec_file: + with closing(open(path)) as ext_file: + for line in ext_file: try: - extensions.add(Spec(line)) - except SpecError, e: + extensions.add(Spec(line.strip())) + except spack.error.SpackError, e: raise InvalidExtensionSpecError(str(e)) return extensions - def write_extensions(self, extensions): + def write_extensions(self, spec, extensions): path = self.extension_file_path(spec) with closing(open(path, 'w')) as spec_file: for extension in sorted(extensions): - spec_file.write("%s\n" % extensions) + spec_file.write("%s\n" % extension) def add_extension(self, spec, extension_spec): - exts = get_extensions(spec) + _check_concrete(spec) + _check_concrete(extension_spec) + + exts = self.get_extensions(spec) if extension_spec in exts: raise ExtensionAlreadyInstalledError(spec, extension_spec) else: @@ -279,16 +301,19 @@ class SpecHashDirectoryLayout(DirectoryLayout): raise ExtensionConflictError(spec, extension_spec, already_installed) exts.add(extension_spec) - self.write_extensions(exts) + self.write_extensions(spec, exts) def remove_extension(self, spec, extension_spec): - exts = get_extensions(spec) + _check_concrete(spec) + _check_concrete(extension_spec) + + exts = self.get_extensions(spec) if not extension_spec in exts: raise NoSuchExtensionError(spec, extension_spec) exts.remove(extension_spec) - self.write_extensions(exts) + self.write_extensions(spec, exts) class DirectoryLayoutError(SpackError): @@ -328,7 +353,7 @@ class ExtensionAlreadyInstalledError(DirectoryLayoutError): """Raised when an extension is added to a package that already has it.""" def __init__(self, spec, extension_spec): super(ExtensionAlreadyInstalledError, self).__init__( - "%s is already installed in %s" % (extension_spec, spec)) + "%s is already installed in %s" % (extension_spec.short_spec, spec.short_spec)) class ExtensionConflictError(DirectoryLayoutError): @@ -336,12 +361,12 @@ class ExtensionConflictError(DirectoryLayoutError): def __init__(self, spec, extension_spec, conflict): super(ExtensionConflictError, self).__init__( "%s cannot be installed in %s because it conflicts with %s."% ( - extension_spec, spec, conflict)) + extension_spec.short_spec, spec.short_spec, conflict.short_spec)) class NoSuchExtensionError(DirectoryLayoutError): """Raised when an extension isn't there on remove.""" def __init__(self, spec, extension_spec): super(NoSuchExtensionError, self).__init__( - "%s cannot be removed from %s beacuse it's not installed."% ( - extension_spec, spec, conflict)) + "%s cannot be removed from %s because it's not installed."% ( + extension_spec.short_spec, spec.short_spec)) diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py new file mode 100644 index 0000000000..444472bffa --- /dev/null +++ b/lib/spack/spack/hooks/extensions.py @@ -0,0 +1,49 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import spack + + +def post_install(pkg): + assert(pkg.spec.concrete) + for name, spec in pkg.extendees.items(): + ext = pkg.spec[name] + epkg = ext.package + if epkg.installed: + epkg.do_activate(pkg) + + +def pre_uninstall(pkg): + assert(pkg.spec.concrete) + + # Need to do this b/c uninstall does not automatically do it. + # TODO: store full graph info in stored .spec file. + pkg.spec.normalize() + + for name, spec in pkg.extendees.items(): + ext = pkg.spec[name] + epkg = ext.package + if epkg.installed: + epkg.do_deactivate(pkg) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 04f0d842da..b7dae552e4 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -329,6 +329,9 @@ class Package(object): """By default we build in parallel. Subclasses can override this.""" parallel = True + """Most packages are NOT extendable. Set to True if you want extensions.""" + extendable = False + def __init__(self, spec): # this determines how the package should be built. @@ -398,6 +401,9 @@ class Package(object): self._fetch_time = 0.0 self._total_time = 0.0 + for name, spec in self.extendees.items(): + spack.db.get(spec)._check_extendable() + @property def version(self): @@ -877,6 +883,79 @@ class Package(object): spack.hooks.post_uninstall(self) + def _check_extendable(self): + if not self.extendable: + raise ValueError("Package %s is not extendable!" % self.name) + + + def _sanity_check_extension(self, extension): + self._check_extendable() + if not self.installed: + raise ValueError("Can only (de)activate extensions for installed packages.") + if not extension.installed: + raise ValueError("Extensions must first be installed.") + if not self.name in extension.extendees: + raise ValueError("%s does not extend %s!" % (extension.name, self.name)) + if not self.spec.satisfies(extension.extendees[self.name]): + raise ValueError("%s does not satisfy %s!" % (self.spec, extension.spec)) + + + def do_activate(self, extension): + self._sanity_check_extension(extension) + + self.activate(extension) + spack.install_layout.add_extension(self.spec, extension.spec) + tty.msg("Activated extension %s for %s." + % (extension.spec.short_spec, self.spec.short_spec)) + + + def activate(self, extension): + """Symlinks all files from the extension into extendee's install dir. + + Package authors can override this method to support other + extension mechanisms. Spack internals (commands, hooks, etc.) + should call do_activate() method so that proper checks are + always executed. + + """ + conflict = check_link_tree( + extension.prefix, self.prefix, + ignore=spack.install_layout.hidden_file_paths) + + if conflict: + raise ExtensionConflictError(conflict) + + merge_link_tree(extension.prefix, self.prefix, + ignore=spack.install_layout.hidden_file_paths) + + + def do_deactivate(self, extension): + self._sanity_check_extension(extension) + self.deactivate(extension) + + ext = extension.spec + if ext in spack.install_layout.get_extensions(self.spec): + spack.install_layout.remove_extension(self.spec, ext) + + tty.msg("Deactivated extension %s for %s." + % (extension.spec.short_spec, self.spec.short_spec)) + + + def deactivate(self, extension): + """Unlinks all files from extension out of extendee's install dir. + + Package authors can override this method to support other + extension mechanisms. Spack internals (commands, hooks, etc.) + should call do_deactivate() method so that proper checks are + always executed. + + """ + unmerge_link_tree(extension.prefix, self.prefix, + ignore=spack.install_layout.hidden_file_paths) + tty.msg("Deactivated %s as extension of %s." + % (extension.spec.short_spec, self.spec.short_spec)) + + def do_clean(self): if self.stage.expanded_archive_path: self.stage.chdir_to_source() @@ -1068,3 +1147,9 @@ class NoURLError(PackageError): def __init__(self, cls): super(NoURLError, self).__init__( "Package %s has no version with a URL." % cls.__name__) + + +class ExtensionConflictError(PackageError): + def __init__(self, path): + super(ExtensionConflictError, self).__init__( + "Extension blocked by file: %s" % path) diff --git a/lib/spack/spack/relations.py b/lib/spack/spack/relations.py index aaca9c199e..17bec1664f 100644 --- a/lib/spack/spack/relations.py +++ b/lib/spack/spack/relations.py @@ -68,7 +68,7 @@ provides spack install mpileaks ^mvapich spack install mpileaks ^mpich """ -__all__ = [ 'depends_on', 'provides', 'patch', 'version' ] +__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version' ] import re import inspect @@ -135,7 +135,7 @@ def extends(*specs): for string in specs: for spec in spack.spec.parse(string): if pkg == spec.name: - raise CircularReferenceError('depends_on', pkg) + raise CircularReferenceError('extends', pkg) dependencies[spec.name] = spec extendees[spec.name] = spec diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index e6c3e28820..953be69cc2 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -1,10 +1,13 @@ from spack import * + class Python(Package): """The Python programming language.""" homepage = "http://www.python.org" url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tar.xz" + extendable = True + version('2.7.8', 'd235bdfa75b8396942e360a70487ee00') depends_on("openssl") -- cgit v1.2.3-70-g09d2 From 82946d29147bbe63855f94b9c2ebd4a21cd0a3d6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 8 Jan 2015 22:46:31 -0800 Subject: Move symlink tree routines to LinkTree class. --- lib/spack/llnl/util/filesystem.py | 82 +------------------ lib/spack/llnl/util/link_tree.py | 168 ++++++++++++++++++++++++++++++++++++++ lib/spack/spack/package.py | 16 ++-- 3 files changed, 176 insertions(+), 90 deletions(-) create mode 100644 lib/spack/llnl/util/link_tree.py (limited to 'lib') diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 9fb76d3a35..0578415653 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -24,8 +24,7 @@ ############################################################################## __all__ = ['set_install_permissions', 'install', 'expand_user', 'working_dir', 'touch', 'mkdirp', 'force_remove', 'join_path', 'ancestor', - 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe', - 'check_link_tree', 'merge_link_tree', 'unmerge_link_tree'] + 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe'] import os import sys @@ -223,82 +222,3 @@ def ancestor(dir, n=1): def can_access(file_name): """True if we have read/write access to the file.""" return os.access(file_name, os.R_OK|os.W_OK) - - -def traverse_link_tree(src_root, dest_root, follow_nonexisting=True, **kwargs): - # Yield directories before or after their contents. - order = kwargs.get('order', 'pre') - if order not in ('pre', 'post'): - raise ValueError("Order must be 'pre' or 'post'.") - - # List of relative paths to ignore under the src root. - ignore = kwargs.get('ignore', None) - if isinstance(ignore, basestring): - ignore = (ignore,) - - for dirpath, dirnames, filenames in os.walk(src_root): - rel_path = dirpath[len(src_root):] - rel_path = rel_path.lstrip(os.path.sep) - dest_dirpath = os.path.join(dest_root, rel_path) - - # Don't descend into ignored directories - if ignore and dest_dirpath in ignore: - return - - # Don't descend into dirs in dest that do not exist in src. - if not follow_nonexisting: - dirnames[:] = [ - d for d in dirnames - if os.path.exists(os.path.join(dest_dirpath, d))] - - # preorder yields directories before children - if order == 'pre': - yield (dirpath, dest_dirpath) - - for name in filenames: - src_file = os.path.join(dirpath, name) - dest_file = os.path.join(dest_dirpath, name) - - # Ignore particular paths inside the install root. - src_relpath = src_file[len(src_root):] - src_relpath = src_relpath.lstrip(os.path.sep) - if ignore and src_relpath in ignore: - continue - - yield (src_file, dest_file) - - # postorder yields directories after children - if order == 'post': - yield (dirpath, dest_dirpath) - - - -def check_link_tree(src_root, dest_root, **kwargs): - for src, dest in traverse_link_tree(src_root, dest_root, False, **kwargs): - if os.path.exists(dest) and not os.path.isdir(dest): - return dest - return None - - -def merge_link_tree(src_root, dest_root, **kwargs): - kwargs['order'] = 'pre' - for src, dest in traverse_link_tree(src_root, dest_root, **kwargs): - if os.path.isdir(src): - mkdirp(dest) - else: - assert(not os.path.exists(dest)) - os.symlink(src, dest) - - -def unmerge_link_tree(src_root, dest_root, **kwargs): - kwargs['order'] = 'post' - for src, dest in traverse_link_tree(src_root, dest_root, **kwargs): - if os.path.isdir(dest): - if not os.listdir(dest): - # TODO: what if empty directories were present pre-merge? - shutil.rmtree(dest, ignore_errors=True) - - elif os.path.exists(dest): - if not os.path.islink(dest): - raise ValueError("%s is not a link tree!" % dest) - os.remove(dest) diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py new file mode 100644 index 0000000000..19c2d46938 --- /dev/null +++ b/lib/spack/llnl/util/link_tree.py @@ -0,0 +1,168 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""LinkTree class for setting up trees of symbolic links.""" +__all__ = ['LinkTree'] + +import os +import shutil +from llnl.util.filesystem import mkdirp + + +class LinkTree(object): + """Class to create trees of symbolic links from a source directory. + + LinkTree objects are constructed with a source root. Their + methods allow you to create and delete trees of symbolic links + back to the source tree in specific destination directories. + Trees comprise symlinks only to files; directries are never + symlinked to, to prevent the source directory from ever being + modified. + + """ + def __init__(self, source_root): + self._root = source_root + + + def traverse(self, dest_root, **kwargs): + """Traverse LinkTree root and dest simultaneously. + + Walks the LinkTree directory in pre or post order. Yields + each file in the source directory with a matching path from + the dest directory. e.g., for this tree:: + + root/ + a/ + file1 + file2 + b/ + file3 + + When called on dest, this yields:: + + ('root', 'dest') + ('root/a', 'dest/a') + ('root/a/file1', 'dest/a/file1') + ('root/a/file2', 'dest/a/file2') + ('root/b', 'dest/b') + ('root/b/file3', 'dest/b/file3') + + Optional args: + + order=[pre|post] -- Whether to do pre- or post-order traveral. + + ignore= -- Optional container of root-relative + paths to ignore. + + follow_nonexisting -- Whether to descend into directories in + src that do not exit in dest. + + """ + # Yield directories before or after their contents. + order = kwargs.get('order', 'pre') + if order not in ('pre', 'post'): + raise ValueError("Order must be 'pre' or 'post'.") + + # List of relative paths to ignore under the src root. + ignore = kwargs.get('ignore', None) + if isinstance(ignore, basestring): + ignore = (ignore,) + + # Whether to descend when dirs dont' exist in dest. + follow_nonexisting = kwargs.get('follow_nonexisting', True) + + for dirpath, dirnames, filenames in os.walk(self._root): + rel_path = dirpath[len(self._root):] + rel_path = rel_path.lstrip(os.path.sep) + dest_dirpath = os.path.join(dest_root, rel_path) + + # Don't descend into ignored directories + if ignore and dest_dirpath in ignore: + return + + # Don't descend into dirs in dest that do not exist in src. + if not follow_nonexisting: + dirnames[:] = [ + d for d in dirnames + if os.path.exists(os.path.join(dest_dirpath, d))] + + # preorder yields directories before children + if order == 'pre': + yield (dirpath, dest_dirpath) + + for name in filenames: + src_file = os.path.join(dirpath, name) + dest_file = os.path.join(dest_dirpath, name) + + # Ignore particular paths inside the install root. + src_relpath = src_file[len(self._root):] + src_relpath = src_relpath.lstrip(os.path.sep) + if ignore and src_relpath in ignore: + continue + + yield (src_file, dest_file) + + # postorder yields directories after children + if order == 'post': + yield (dirpath, dest_dirpath) + + + + def find_conflict(self, dest_root, **kwargs): + """Returns the first file in dest that also exists in src.""" + kwargs['follow_nonexisting'] = False + for src, dest in self.traverse(dest_root, **kwargs): + if os.path.exists(dest) and not os.path.isdir(dest): + return dest + return None + + + def merge(self, dest_root, **kwargs): + """Link all files in src into dest, creating directories if necessary.""" + kwargs['order'] = 'pre' + for src, dest in self.traverse(dest_root, **kwargs): + if os.path.isdir(src): + mkdirp(dest) + else: + assert(not os.path.exists(dest)) + os.symlink(src, dest) + + + def unmerge(self, dest_root, **kwargs): + """Unlink all files in dest that exist in src. + + Unlinks directories in dest if they are empty. + + """ + kwargs['order'] = 'post' + for src, dest in self.traverse(dest_root, **kwargs): + if os.path.isdir(dest): + if not os.listdir(dest): + # TODO: what if empty directories were present pre-merge? + shutil.rmtree(dest, ignore_errors=True) + + elif os.path.exists(dest): + if not os.path.islink(dest): + raise ValueError("%s is not a link tree!" % dest) + os.remove(dest) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b7dae552e4..da251dc4e8 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -45,6 +45,7 @@ import textwrap from StringIO import StringIO import llnl.util.tty as tty +from llnl.util.link_tree import LinkTree from llnl.util.filesystem import * from llnl.util.lang import * @@ -918,15 +919,12 @@ class Package(object): always executed. """ - conflict = check_link_tree( - extension.prefix, self.prefix, - ignore=spack.install_layout.hidden_file_paths) - + tree = LinkTree(extension.prefix) + conflict = tree.find_conflict( + self.prefix, ignore=spack.install_layout.hidden_file_paths) if conflict: raise ExtensionConflictError(conflict) - - merge_link_tree(extension.prefix, self.prefix, - ignore=spack.install_layout.hidden_file_paths) + tree.merge(self.prefix, ignore=spack.install_layout.hidden_file_paths) def do_deactivate(self, extension): @@ -950,8 +948,8 @@ class Package(object): always executed. """ - unmerge_link_tree(extension.prefix, self.prefix, - ignore=spack.install_layout.hidden_file_paths) + tree = LinkTree(extension.prefix) + tree.unmerge(self.prefix, ignore=spack.install_layout.hidden_file_paths) tty.msg("Deactivated %s as extension of %s." % (extension.spec.short_spec, self.spec.short_spec)) -- cgit v1.2.3-70-g09d2 From bcccf020204a556e382c0af2897ad9126bb24984 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 10 Jan 2015 19:37:01 -0800 Subject: Add setup_extension_environment() method. - lets packages do some setup before their extensions run install() --- lib/spack/spack/package.py | 30 +++++++++++++++++++++++++++++ var/spack/packages/py-setuptools/package.py | 6 ------ var/spack/packages/python/package.py | 23 +++++++++++++++++++++- 3 files changed, 52 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index da251dc4e8..8504b96fcf 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -783,6 +783,12 @@ class Package(object): self.stage.chdir_to_source() build_env.setup_package(self) + # Allow extendees to further set up the environment. + for ext_name in self.extendees: + ext_spec = self.spec[ext_name] + ext_spec.package.setup_extension_environment( + self.module, ext_spec, self.spec) + if fake_install: self.do_fake_install() else: @@ -854,6 +860,30 @@ class Package(object): fromlist=[self.__class__.__name__]) + def setup_extension_environment(self, module, spec, ext_spec): + """Called before the install() method of extensions. + + Default implementation does nothing, but this can be + overridden by an extendable package to set up the install + environment for its extensions. This is useful if there are + some common steps to installing all extensions for a + certain package. + + Some examples: + + 1. Installing python modules generally requires PYTHONPATH to + point to the lib/pythonX.Y/site-packages directory in the + module's install prefix. This could set that variable. + + 2. Extensions often need to invoke the 'python' interpreter + from the Python installation being extended. This routine can + put a 'python' Execuable object in the module scope for the + extension package to simplify extension installs. + + """ + pass + + def install(self, spec, prefix): """Package implementations override this with their own build configuration.""" raise InstallError("Package %s provides no install method!" % self.name) diff --git a/var/spack/packages/py-setuptools/package.py b/var/spack/packages/py-setuptools/package.py index e2c4e1a0be..755288d55c 100644 --- a/var/spack/packages/py-setuptools/package.py +++ b/var/spack/packages/py-setuptools/package.py @@ -10,10 +10,4 @@ class PySetuptools(Package): extends('python') def install(self, spec, prefix): - site_packages_dir = "%s/lib/python2.7/site-packages" % prefix - mkdirp(site_packages_dir) - - env['PYTHONPATH'] = site_packages_dir - - python = which('python') python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 953be69cc2..9700179ab8 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -1,5 +1,5 @@ from spack import * - +import os class Python(Package): """The Python programming language.""" @@ -26,3 +26,24 @@ class Python(Package): "--enable-shared") make() make("install") + + + def setup_extension_environment(self, module, spec, ext_spec): + """Called before python modules' install() methods. + + In most cases, extensions will only need to have one line:: + + python('setup.py', 'install', '--prefix=%s' % prefix) + """ + # Python extension builds can have a global python executable function + module.python = Executable(join_path(spec.prefix.bin, 'python')) + + # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. + module.python_lib_dir = join_path(ext_spec.prefix.lib, 'python%d.%d' % self.version[:2]) + module.site_packages_dir = join_path(module.python_lib_dir, 'site-packages') + + # Add site packages directory to the PYTHONPATH + os.environ['PYTHONPATH'] = module.site_packages_dir + + # Make the site packages directory if it does not exist already. + mkdirp(module.site_packages_dir) -- cgit v1.2.3-70-g09d2 From d13bbeb605f56214db919f6f122a8fa6ba67ddbc Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 10 Jan 2015 19:52:07 -0800 Subject: Add PYTOHNPATH to modules for python extensions. --- lib/spack/spack/modules.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 755e9ea900..7d2ca97a62 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -49,6 +49,7 @@ import os import re import textwrap import shutil +from glob import glob from contextlib import closing import llnl.util.tty as tty @@ -123,6 +124,13 @@ class EnvModule(object): if os.path.isdir(directory): add_path(var, directory) + # Add python path unless it's an actual python installation + # TODO: is there a better way to do this? + if self.spec.name != 'python': + site_packages = glob(join_path(self.spec.prefix.lib, "python*/site-packages")) + if site_packages: + add_path('PYTHONPATH', site_packages[0]) + # short description is just the package + version # TODO: maybe packages can optionally provide it. self.short_description = self.spec.format("$_ $@") -- cgit v1.2.3-70-g09d2 From acc62abbd08f6d44c930b0ceed74fb9e47f365cf Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 12 Jan 2015 22:39:18 -0800 Subject: Rework do_activate/activate and do_deactivate/deactivate semantics. - packages can now extend only one other package. - do_activate() and do_deactivate() are now called on the extension, and they automatically find the extendee - activate() and deactivate() are still called on the extendee and are passed the extension. --- lib/spack/spack/cmd/__init__.py | 15 ++++++ lib/spack/spack/cmd/location.py | 47 ++++++++---------- lib/spack/spack/cmd/uninstall.py | 1 - lib/spack/spack/hooks/extensions.py | 15 +----- lib/spack/spack/package.py | 97 ++++++++++++++++++++++++------------- lib/spack/spack/packages.py | 2 + lib/spack/spack/relations.py | 2 + 7 files changed, 105 insertions(+), 74 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 537db536dd..b96ac5af51 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -121,3 +121,18 @@ def elide_list(line_list, max_num=10): return line_list[:max_num-1] + ['...'] + line_list[-1:] else: return line_list + + +def disambiguate_spec(spec): + matching_specs = spack.db.get_installed(spec) + if not matching_specs: + tty.die("Spec '%s' matches no installed packages." % spec) + + elif len(matching_specs) > 1: + args = ["%s matches multiple packages." % spec, + "Matching packages:"] + args += [" " + str(s) for s in matching_specs] + args += ["Use a more specific spec."] + tty.die(*args) + + return matching_specs[0] diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 509c336b69..810c34d0a6 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -77,37 +77,30 @@ def location(parser, args): tty.die("You must supply a spec.") if len(specs) != 1: tty.die("Too many specs. Supply only one.") - spec = specs[0] if args.install_dir: # install_dir command matches against installed specs. - matching_specs = spack.db.get_installed(spec) - if not matching_specs: - tty.die("Spec '%s' matches no installed packages." % spec) + spec = spack.cmd.disambiguate_spec(specs[0]) + print spec.prefix - elif len(matching_specs) > 1: - args = ["%s matches multiple packages." % spec, - "Matching packages:"] - args += [" " + str(s) for s in matching_specs] - args += ["Use a more specific spec."] - tty.die(*args) + else: + spec = specs[0] - print matching_specs[0].prefix + if args.package_dir: + # This one just needs the spec name. + print join_path(spack.db.root, spec.name) - elif args.package_dir: - # This one just needs the spec name. - print join_path(spack.db.root, spec.name) + else: + # These versions need concretized specs. + spec.concretize() + pkg = spack.db.get(spec) + + if args.stage_dir: + print pkg.stage.path + + else: # args.build_dir is the default. + if not pkg.stage.source_path: + tty.die("Build directory does not exist yet. Run this to create it:", + "spack stage " + " ".join(args.spec)) + print pkg.stage.source_path - else: - # These versions need concretized specs. - spec.concretize() - pkg = spack.db.get(spec) - - if args.stage_dir: - print pkg.stage.path - - else: # args.build_dir is the default. - if not pkg.stage.source_path: - tty.die("Build directory does not exist yet. Run this to create it:", - "spack stage " + " ".join(args.spec)) - print pkg.stage.source_path diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index e787c460ad..0962942f43 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -65,7 +65,6 @@ def uninstall(parser, args): " b) use a more specific spec."] tty.die(*args) - if len(matching_specs) == 0: tty.die("%s does not match any installed packages." % spec) diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py index 444472bffa..2cf506beed 100644 --- a/lib/spack/spack/hooks/extensions.py +++ b/lib/spack/spack/hooks/extensions.py @@ -27,23 +27,12 @@ import spack def post_install(pkg): - assert(pkg.spec.concrete) - for name, spec in pkg.extendees.items(): - ext = pkg.spec[name] - epkg = ext.package - if epkg.installed: - epkg.do_activate(pkg) + pkg.do_activate() def pre_uninstall(pkg): - assert(pkg.spec.concrete) - # Need to do this b/c uninstall does not automatically do it. # TODO: store full graph info in stored .spec file. pkg.spec.normalize() - for name, spec in pkg.extendees.items(): - ext = pkg.spec[name] - epkg = ext.package - if epkg.installed: - epkg.do_deactivate(pkg) + pkg.do_deactivate() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8504b96fcf..ae34f8ae45 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -315,15 +315,18 @@ class Package(object): """Specs of virtual packages provided by this package, keyed by name.""" provided = {} - """Specs of packages this one extends, keyed by name.""" - extendees = {} - """Specs of conflicting packages, keyed by name. """ conflicted = {} """Patches to apply to newly expanded source, if any.""" patches = {} + """Specs of package this one extends, or None. + + Currently, ppackages can extend at most one other package. + """ + extendees = {} + # # These are default values for instance variables. # @@ -402,8 +405,8 @@ class Package(object): self._fetch_time = 0.0 self._total_time = 0.0 - for name, spec in self.extendees.items(): - spack.db.get(spec)._check_extendable() + if self.is_extension: + spack.db.get(self.extendee_spec)._check_extendable() @property @@ -491,6 +494,34 @@ class Package(object): self._fetcher = f + @property + def extendee_spec(self): + """Spec of the extendee of this package, or None if it is not an extension.""" + if not self.extendees: return None + + name = next(iter(self.extendees)) + if not name in self.spec: + return self.extendees[name] + + # Need to do this to get the concrete version of the spec + return self.spec[name] + + + @property + def is_extension(self): + return len(self.extendees) > 0 + + + @property + def activated(self): + if not self.spec.concrete: + raise ValueError("Only concrete package extensions can be activated.") + if not self.is_extension: + raise ValueError("is_extension called on package that is not an extension.") + + return self.spec in spack.install_layout.get_extensions(self.extendee_spec) + + def preorder_traversal(self, visited=None, **kwargs): """This does a preorder traversal of the package's dependence DAG.""" virtual = kwargs.get("virtual", False) @@ -784,10 +815,9 @@ class Package(object): build_env.setup_package(self) # Allow extendees to further set up the environment. - for ext_name in self.extendees: - ext_spec = self.spec[ext_name] - ext_spec.package.setup_extension_environment( - self.module, ext_spec, self.spec) + if self.is_extension: + self.extendee_spec.package.setup_extension_environment( + self.module, self.extendee_spec, self.spec) if fake_install: self.do_fake_install() @@ -840,7 +870,6 @@ class Package(object): if returncode != 0: sys.exit(1) - # Once everything else is done, run post install hooks spack.hooks.post_install(self) @@ -919,25 +948,30 @@ class Package(object): raise ValueError("Package %s is not extendable!" % self.name) - def _sanity_check_extension(self, extension): - self._check_extendable() - if not self.installed: + def _sanity_check_extension(self): + extendee_package = self.extendee_spec.package + extendee_package._check_extendable() + + if not extendee_package.installed: raise ValueError("Can only (de)activate extensions for installed packages.") - if not extension.installed: + if not self.installed: raise ValueError("Extensions must first be installed.") - if not self.name in extension.extendees: - raise ValueError("%s does not extend %s!" % (extension.name, self.name)) - if not self.spec.satisfies(extension.extendees[self.name]): - raise ValueError("%s does not satisfy %s!" % (self.spec, extension.spec)) + if not self.extendee_spec.name in self.extendees: + raise ValueError("%s does not extend %s!" % (self.name, self.extendee.name)) + + def do_activate(self): + """Called on an etension to invoke the extendee's activate method. - def do_activate(self, extension): - self._sanity_check_extension(extension) + Commands should call this routine, and should not call + activate() directly. + """ + self._sanity_check_extension() + self.extendee_spec.package.activate(self) - self.activate(extension) - spack.install_layout.add_extension(self.spec, extension.spec) + spack.install_layout.add_extension(self.extendee_spec, self.spec) tty.msg("Activated extension %s for %s." - % (extension.spec.short_spec, self.spec.short_spec)) + % (self.spec.short_spec, self.extendee_spec.short_spec)) def activate(self, extension): @@ -957,20 +991,19 @@ class Package(object): tree.merge(self.prefix, ignore=spack.install_layout.hidden_file_paths) - def do_deactivate(self, extension): - self._sanity_check_extension(extension) - self.deactivate(extension) + def do_deactivate(self): + self._sanity_check_extension() + self.extendee_spec.package.deactivate(self) - ext = extension.spec - if ext in spack.install_layout.get_extensions(self.spec): - spack.install_layout.remove_extension(self.spec, ext) + if self.spec in spack.install_layout.get_extensions(self.extendee_spec): + spack.install_layout.remove_extension(self.extendee_spec, self.spec) tty.msg("Deactivated extension %s for %s." - % (extension.spec.short_spec, self.spec.short_spec)) + % (self.spec.short_spec, self.extendee_spec.short_spec)) def deactivate(self, extension): - """Unlinks all files from extension out of extendee's install dir. + """Unlinks all files from extension out of this package's install dir. Package authors can override this method to support other extension mechanisms. Spack internals (commands, hooks, etc.) @@ -980,8 +1013,6 @@ class Package(object): """ tree = LinkTree(extension.prefix) tree.unmerge(self.prefix, ignore=spack.install_layout.hidden_file_paths) - tty.msg("Deactivated %s as extension of %s." - % (extension.spec.short_spec, self.spec.short_spec)) def do_clean(self): diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index db43d3909a..bb5a94bcab 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -77,6 +77,8 @@ class PackageDB(object): copy = spec.copy() self.instances[copy] = package_class(copy) except Exception, e: + if spack.debug: + sys.excepthook(*sys.exc_info()) raise FailedConstructorError(spec.name, e) return self.instances[spec] diff --git a/lib/spack/spack/relations.py b/lib/spack/spack/relations.py index 17bec1664f..60ff5bef34 100644 --- a/lib/spack/spack/relations.py +++ b/lib/spack/spack/relations.py @@ -131,6 +131,8 @@ def extends(*specs): clocals = caller_locals() dependencies = clocals.setdefault('dependencies', {}) extendees = clocals.setdefault('extendees', {}) + if extendees: + raise RelationError("Packages can extend at most one other package.") for string in specs: for spec in spack.spec.parse(string): -- cgit v1.2.3-70-g09d2 From 89ccdf92cd42a52fa42d39e2be580eac110e264a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 12 Jan 2015 22:41:27 -0800 Subject: Add activate and deactivate commands for extensions. --- lib/spack/spack/cmd/activate.py | 50 +++++++++++++++++++++++++++++++++++++++ lib/spack/spack/cmd/deactivate.py | 50 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 lib/spack/spack/cmd/activate.py create mode 100644 lib/spack/spack/cmd/deactivate.py (limited to 'lib') diff --git a/lib/spack/spack/cmd/activate.py b/lib/spack/spack/cmd/activate.py new file mode 100644 index 0000000000..c1e23852d6 --- /dev/null +++ b/lib/spack/spack/cmd/activate.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from external import argparse +import llnl.util.tty as tty +import spack +import spack.cmd + +description = "Activate a package extension." + +def setup_parser(subparser): + subparser.add_argument( + 'spec', nargs=argparse.REMAINDER, help="spec of package extension to activate.") + + +def activate(parser, args): + specs = spack.cmd.parse_specs(args.spec, concretize=True) + if len(specs) != 1: + tty.die("activate requires one spec. %d given." % len(specs)) + + # TODO: remove this hack when DAG info is stored in dir layout. + # This ensures the ext spec is always normalized properly. + spack.db.get(specs[0]) + + spec = spack.cmd.disambiguate_spec(specs[0]) + if spec.package.activated: + tty.die("Package %s is already activated." % specs[0].short_spec) + + spec.package.do_activate() diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py new file mode 100644 index 0000000000..fd13f051df --- /dev/null +++ b/lib/spack/spack/cmd/deactivate.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from external import argparse +import llnl.util.tty as tty +import spack +import spack.cmd + +description = "Deactivate a package extension." + +def setup_parser(subparser): + subparser.add_argument( + 'spec', nargs=argparse.REMAINDER, help="spec of package extension to deactivate.") + + +def deactivate(parser, args): + specs = spack.cmd.parse_specs(args.spec, concretize=True) + if len(specs) != 1: + tty.die("deactivate requires one spec. %d given." % len(specs)) + + # TODO: remove this hack when DAG info is stored in dir layout. + # This ensures the ext spec is always normalized properly. + spack.db.get(specs[0]) + + spec = spack.cmd.disambiguate_spec(specs[0]) + if not spec.package.activated: + tty.die("Package %s is not activated." % specs[0].short_spec) + + spec.package.do_deactivate() -- cgit v1.2.3-70-g09d2 From 2ae7f53b8359841fc5d1d7b6c70ff72f38bf2d88 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 15 Jan 2015 09:33:51 -0800 Subject: Bugfix: Extension hooks shoudl only run for extensions. --- lib/spack/spack/hooks/extensions.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py index 2cf506beed..718b24b965 100644 --- a/lib/spack/spack/hooks/extensions.py +++ b/lib/spack/spack/hooks/extensions.py @@ -27,7 +27,8 @@ import spack def post_install(pkg): - pkg.do_activate() + if pkg.is_extension: + pkg.do_activate() def pre_uninstall(pkg): @@ -35,4 +36,5 @@ def pre_uninstall(pkg): # TODO: store full graph info in stored .spec file. pkg.spec.normalize() - pkg.do_deactivate() + if pkg.is_extension: + pkg.do_deactivate() -- cgit v1.2.3-70-g09d2 From ff9cb94f4f92112739f53881bcb0a9a19811684d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 20 Jan 2015 00:23:16 -0800 Subject: Add arguements to extends() and activate/deactivate. --- lib/spack/spack/package.py | 40 ++++++++++++++++++++++++++---------- lib/spack/spack/relations.py | 17 ++++++++------- var/spack/packages/python/package.py | 32 +++++++++++++++++++++++++++-- 3 files changed, 69 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index ae34f8ae45..bd63c2e0c0 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -497,16 +497,26 @@ class Package(object): @property def extendee_spec(self): """Spec of the extendee of this package, or None if it is not an extension.""" - if not self.extendees: return None - + if not self.extendees: + return None name = next(iter(self.extendees)) if not name in self.spec: - return self.extendees[name] + spec, kwargs = self.extendees[name] + return spec # Need to do this to get the concrete version of the spec return self.spec[name] + @property + def extendee_args(self): + """Spec of the extendee of this package, or None if it is not an extension.""" + if not self.extendees: + return None + name = next(iter(self.extendees)) + return self.extendees[name][1] + + @property def is_extension(self): return len(self.extendees) > 0 @@ -949,6 +959,8 @@ class Package(object): def _sanity_check_extension(self): + if not self.is_extension: + raise ValueError("This package is not an extension.") extendee_package = self.extendee_spec.package extendee_package._check_extendable() @@ -967,14 +979,14 @@ class Package(object): activate() directly. """ self._sanity_check_extension() - self.extendee_spec.package.activate(self) + self.extendee_spec.package.activate(self, **self.extendee_args) spack.install_layout.add_extension(self.extendee_spec, self.spec) tty.msg("Activated extension %s for %s." % (self.spec.short_spec, self.extendee_spec.short_spec)) - def activate(self, extension): + def activate(self, extension, **kwargs): """Symlinks all files from the extension into extendee's install dir. Package authors can override this method to support other @@ -983,17 +995,20 @@ class Package(object): always executed. """ + ignore_files = set(spack.install_layout.hidden_file_paths) + ignore_files.update(kwargs.get('ignore', ())) + tree = LinkTree(extension.prefix) - conflict = tree.find_conflict( - self.prefix, ignore=spack.install_layout.hidden_file_paths) + conflict = tree.find_conflict(self.prefix, ignore=ignore_files) if conflict: raise ExtensionConflictError(conflict) - tree.merge(self.prefix, ignore=spack.install_layout.hidden_file_paths) + tree.merge(self.prefix, ignore=ignore_files) def do_deactivate(self): + """Called on the extension to invoke extendee's deactivate() method.""" self._sanity_check_extension() - self.extendee_spec.package.deactivate(self) + self.extendee_spec.package.deactivate(self, **self.extendee_args) if self.spec in spack.install_layout.get_extensions(self.extendee_spec): spack.install_layout.remove_extension(self.extendee_spec, self.spec) @@ -1002,7 +1017,7 @@ class Package(object): % (self.spec.short_spec, self.extendee_spec.short_spec)) - def deactivate(self, extension): + def deactivate(self, extension, **kwargs): """Unlinks all files from extension out of this package's install dir. Package authors can override this method to support other @@ -1011,8 +1026,11 @@ class Package(object): always executed. """ + ignore_files = set(spack.install_layout.hidden_file_paths) + ignore_files.update(kwargs.get('ignore', ())) + tree = LinkTree(extension.prefix) - tree.unmerge(self.prefix, ignore=spack.install_layout.hidden_file_paths) + tree.unmerge(self.prefix, ignore=ignore_files) def do_clean(self): diff --git a/lib/spack/spack/relations.py b/lib/spack/spack/relations.py index 60ff5bef34..a0c7723473 100644 --- a/lib/spack/spack/relations.py +++ b/lib/spack/spack/relations.py @@ -117,7 +117,7 @@ def depends_on(*specs): dependencies[spec.name] = spec -def extends(*specs): +def extends(spec, **kwargs): """Same as depends_on, but dependency is symlinked into parent prefix. This is for Python and other language modules where the module @@ -126,6 +126,10 @@ def extends(*specs): but allowing ONE module version to be symlinked into a parent Python install at a time. + keyword arguments can be passed to extends() so that extension + packages can pass parameters to the extendee's extension + mechanism. + """ pkg = get_calling_package_name() clocals = caller_locals() @@ -134,12 +138,11 @@ def extends(*specs): if extendees: raise RelationError("Packages can extend at most one other package.") - for string in specs: - for spec in spack.spec.parse(string): - if pkg == spec.name: - raise CircularReferenceError('extends', pkg) - dependencies[spec.name] = spec - extendees[spec.name] = spec + spec = Spec(spec) + if pkg == spec.name: + raise CircularReferenceError('extends', pkg) + dependencies[spec.name] = spec + extendees[spec.name] = (spec, kwargs) def provides(*specs, **kwargs): diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 9700179ab8..86b903bc23 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -28,6 +28,16 @@ class Python(Package): make("install") + @property + def python_lib_dir(self): + return os.path.join('lib', 'python%d.%d' % self.version[:2]) + + + @property + def site_packages_dir(self): + return os.path.join(self.python_lib_dir, 'site-packages') + + def setup_extension_environment(self, module, spec, ext_spec): """Called before python modules' install() methods. @@ -39,11 +49,29 @@ class Python(Package): module.python = Executable(join_path(spec.prefix.bin, 'python')) # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. - module.python_lib_dir = join_path(ext_spec.prefix.lib, 'python%d.%d' % self.version[:2]) - module.site_packages_dir = join_path(module.python_lib_dir, 'site-packages') + module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) + module.site_packages_dir = os.path.join(ext_spec.prefix, self.site_packages_dir) # Add site packages directory to the PYTHONPATH os.environ['PYTHONPATH'] = module.site_packages_dir # Make the site packages directory if it does not exist already. mkdirp(module.site_packages_dir) + + + def add_ignore_files(self, args): + """Add some ignore files to activate/deactivate args.""" + ignore = set(args.get('ignore', ())) + ignore.add(os.path.join(self.site_packages_dir, 'site.py')) + ignore.add(os.path.join(self.site_packages_dir, 'site.pyc')) + args.update(ignore=ignore) + + + def activate(self, ext_pkg, **args): + self.add_ignore_files(args) + super(Python, self).activate(ext_pkg, **args) + + + def deactivate(self, ext_pkg, **args): + self.add_ignore_files(args) + super(Python, self).deactivate(ext_pkg, **args) -- cgit v1.2.3-70-g09d2 From de91c95e8e45b8ab066ba3dfc8f89c92da761b5a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 20 Jan 2015 15:07:53 -0800 Subject: Ability to ignore files in activate/deactivate for extensions. --- lib/spack/llnl/util/link_tree.py | 11 ++++------- lib/spack/spack/package.py | 16 +++++++++------- var/spack/packages/py-nose/package.py | 4 +++- var/spack/packages/python/package.py | 17 ++++++++++------- 4 files changed, 26 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py index 19c2d46938..2d7126be2c 100644 --- a/lib/spack/llnl/util/link_tree.py +++ b/lib/spack/llnl/util/link_tree.py @@ -72,8 +72,7 @@ class LinkTree(object): order=[pre|post] -- Whether to do pre- or post-order traveral. - ignore= -- Optional container of root-relative - paths to ignore. + ignore= -- Predicate indicating which files to ignore. follow_nonexisting -- Whether to descend into directories in src that do not exit in dest. @@ -85,9 +84,7 @@ class LinkTree(object): raise ValueError("Order must be 'pre' or 'post'.") # List of relative paths to ignore under the src root. - ignore = kwargs.get('ignore', None) - if isinstance(ignore, basestring): - ignore = (ignore,) + ignore = kwargs.get('ignore', lambda filename: False) # Whether to descend when dirs dont' exist in dest. follow_nonexisting = kwargs.get('follow_nonexisting', True) @@ -98,7 +95,7 @@ class LinkTree(object): dest_dirpath = os.path.join(dest_root, rel_path) # Don't descend into ignored directories - if ignore and dest_dirpath in ignore: + if ignore(dest_dirpath): return # Don't descend into dirs in dest that do not exist in src. @@ -118,7 +115,7 @@ class LinkTree(object): # Ignore particular paths inside the install root. src_relpath = src_file[len(self._root):] src_relpath = src_relpath.lstrip(os.path.sep) - if ignore and src_relpath in ignore: + if ignore(src_relpath): continue yield (src_file, dest_file) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index bd63c2e0c0..43b1fcd9c8 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -995,14 +995,15 @@ class Package(object): always executed. """ - ignore_files = set(spack.install_layout.hidden_file_paths) - ignore_files.update(kwargs.get('ignore', ())) + def ignore(filename): + return (filename in spack.install_layout.hidden_file_paths or + kwargs.get('ignore', lambda f: False)(filename)) tree = LinkTree(extension.prefix) - conflict = tree.find_conflict(self.prefix, ignore=ignore_files) + conflict = tree.find_conflict(self.prefix, ignore=ignore) if conflict: raise ExtensionConflictError(conflict) - tree.merge(self.prefix, ignore=ignore_files) + tree.merge(self.prefix, ignore=ignore) def do_deactivate(self): @@ -1026,11 +1027,12 @@ class Package(object): always executed. """ - ignore_files = set(spack.install_layout.hidden_file_paths) - ignore_files.update(kwargs.get('ignore', ())) + def ignore(filename): + return (filename in spack.install_layout.hidden_file_paths or + kwargs.get('ignore', lambda f: False)(filename)) tree = LinkTree(extension.prefix) - tree.unmerge(self.prefix, ignore=ignore_files) + tree.unmerge(self.prefix, ignore=ignore) def do_clean(self): diff --git a/var/spack/packages/py-nose/package.py b/var/spack/packages/py-nose/package.py index 7bd7106b8c..6df84e831d 100644 --- a/var/spack/packages/py-nose/package.py +++ b/var/spack/packages/py-nose/package.py @@ -1,7 +1,9 @@ from spack import * class PyNose(Package): - """nose extends the test loading and running features of unittest, making it easier to write, find and run tests.""" + """nose extends the test loading and running features of unittest, + making it easier to write, find and run tests.""" + homepage = "https://pypi.python.org/pypi/nose" url = "https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz" diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 86b903bc23..a22bd54c82 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -1,5 +1,6 @@ from spack import * import os +import re class Python(Package): """The Python programming language.""" @@ -59,19 +60,21 @@ class Python(Package): mkdirp(module.site_packages_dir) - def add_ignore_files(self, args): + def make_ignore(self, args): """Add some ignore files to activate/deactivate args.""" - ignore = set(args.get('ignore', ())) - ignore.add(os.path.join(self.site_packages_dir, 'site.py')) - ignore.add(os.path.join(self.site_packages_dir, 'site.pyc')) - args.update(ignore=ignore) + orig_ignore = args.get('ignore', lambda f: False) + def ignore(filename): + return (re.search(r'/site\.pyc?$', filename) or + re.search(r'\.pth$', filename) or + orig_ignore(filename)) + return ignore def activate(self, ext_pkg, **args): - self.add_ignore_files(args) + args.update(ignore=self.make_ignore(args)) super(Python, self).activate(ext_pkg, **args) def deactivate(self, ext_pkg, **args): - self.add_ignore_files(args) + args.update(ignore=self.make_ignore(args)) super(Python, self).deactivate(ext_pkg, **args) -- cgit v1.2.3-70-g09d2 From 70c8bf44b8d2613c54423562f81e049fbb956780 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 26 Jan 2015 14:47:33 -0800 Subject: Fix for install sanity check -- don't count hidden dir layout files. --- lib/spack/spack/package.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 43b1fcd9c8..0b6bc4ce6c 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -836,10 +836,7 @@ class Package(object): self.install(self.spec, self.prefix) # Ensure that something was actually installed. - if not os.listdir(self.prefix): - raise InstallError( - "Install failed for %s. Nothing was installed!" - % self.name) + self._sanity_check_install() # On successful install, remove the stage. if not keep_stage: @@ -884,6 +881,15 @@ class Package(object): spack.hooks.post_install(self) + + def _sanity_check_install(self): + installed = set(os.listdir(self.prefix)) + installed.difference_update(spack.install_layout.hidden_file_paths) + if not installed: + raise InstallError( + "Install failed for %s. Nothing was installed!" % self.name) + + def do_install_dependencies(self, **kwargs): # Pass along paths of dependencies here for dep in self.spec.dependencies.values(): -- cgit v1.2.3-70-g09d2 From 6400ace90152a08a32684f97490369467ae1e37d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 26 Jan 2015 15:45:19 -0800 Subject: Add "spack extensions" command to list activated extensions. --- lib/spack/spack/cmd/extensions.py | 75 +++++++++++++++++++++++++++++++++++++++ lib/spack/spack/cmd/find.py | 69 +++++++++++++++++++++-------------- 2 files changed, 117 insertions(+), 27 deletions(-) create mode 100644 lib/spack/spack/cmd/extensions.py (limited to 'lib') diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py new file mode 100644 index 0000000000..961d7e3f24 --- /dev/null +++ b/lib/spack/spack/cmd/extensions.py @@ -0,0 +1,75 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import sys +from external import argparse + +import llnl.util.tty as tty + +import spack +import spack.cmd +import spack.cmd.find + +description = "List extensions for package." + +def setup_parser(subparser): + format_group = subparser.add_mutually_exclusive_group() + format_group.add_argument( + '-l', '--long', action='store_const', dest='mode', const='long', + help='Show dependency hashes as well as versions.') + format_group.add_argument( + '-p', '--paths', action='store_const', dest='mode', const='paths', + help='Show paths to extension install directories') + format_group.add_argument( + '-d', '--deps', action='store_const', dest='mode', const='deps', + help='Show full dependency DAG of extensions') + + subparser.add_argument( + 'spec', nargs=argparse.REMAINDER, help='Spec of package to list extensions for') + + +def extensions(parser, args): + if not args.spec: + tty.die("extensions requires a package spec.") + + spec = spack.cmd.parse_specs(args.spec) + if len(spec) > 1: + tty.die("Can only list extensions for one package.") + spec = spack.cmd.disambiguate_spec(spec[0]) + + if not spec.package.extendable: + tty.die("%s does not have extensions." % spec.short_spec) + + if not args.mode: + args.mode = 'short' + + exts = spack.install_layout.get_extensions(spec) + if not exts: + tty.msg("%s has no activated extensions." % spec.short_spec) + else: + tty.msg("Showing %d activated extension%s for package:" + % (len(exts), 's' if len(exts) > 1 else ''), + spec.short_spec) + print + spack.cmd.find.display_specs(exts, mode=args.mode) diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 1de3413d42..f6f503afe5 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -41,13 +41,13 @@ description ="Find installed spack packages" def setup_parser(subparser): format_group = subparser.add_mutually_exclusive_group() format_group.add_argument( - '-l', '--long', action='store_true', dest='long', + '-l', '--long', action='store_const', dest='mode', const='long', help='Show dependency hashes as well as versions.') format_group.add_argument( - '-p', '--paths', action='store_true', dest='paths', + '-p', '--paths', action='store_const', dest='mode', const='paths', help='Show paths to package install directories') format_group.add_argument( - '-d', '--deps', action='store_true', dest='full_deps', + '-d', '--deps', action='store_const', dest='mode', const='deps', help='Show full dependency DAG of installed packages') subparser.add_argument( @@ -55,26 +55,8 @@ def setup_parser(subparser): help='optional specs to filter results') -def find(parser, args): - # Filter out specs that don't exist. - query_specs = spack.cmd.parse_specs(args.query_specs) - query_specs, nonexisting = partition_list( - query_specs, lambda s: spack.db.exists(s.name)) - - if nonexisting: - msg = "No such package%s: " % ('s' if len(nonexisting) > 1 else '') - msg += ", ".join(s.name for s in nonexisting) - tty.msg(msg) - - if not query_specs: - return - - # Get all the specs the user asked for - if not query_specs: - specs = set(spack.db.installed_package_specs()) - else: - results = [set(spack.db.get_installed(qs)) for qs in query_specs] - specs = set.union(*results) +def display_specs(specs, **kwargs): + mode = kwargs.get('mode', 'short') # Make a dict with specs keyed by architecture and compiler. index = index_by(specs, ('architecture', 'compiler')) @@ -92,7 +74,7 @@ def find(parser, args): specs.sort() abbreviated = [s.format('$_$@$+', color=True) for s in specs] - if args.paths: + if mode == 'paths': # Print one spec per line along with prefix path width = max(len(s) for s in abbreviated) width += 2 @@ -101,11 +83,44 @@ def find(parser, args): for abbrv, spec in zip(abbreviated, specs): print format % (abbrv, spec.prefix) - elif args.full_deps: + elif mode == 'deps': for spec in specs: print spec.tree(indent=4, format='$_$@$+', color=True), - else: + + elif mode in ('short', 'long'): fmt = '$-_$@$+' - if args.long: + if mode == 'long': fmt += '$#' colify(s.format(fmt, color=True) for s in specs) + + else: + raise ValueError( + "Invalid mode for display_specs: %s. Must be one of (paths, deps, short)." % mode) + + + +def find(parser, args): + # Filter out specs that don't exist. + query_specs = spack.cmd.parse_specs(args.query_specs) + query_specs, nonexisting = partition_list( + query_specs, lambda s: spack.db.exists(s.name)) + + if nonexisting: + msg = "No such package%s: " % ('s' if len(nonexisting) > 1 else '') + msg += ", ".join(s.name for s in nonexisting) + tty.msg(msg) + + if not query_specs: + return + + # Get all the specs the user asked for + if not query_specs: + specs = set(spack.db.installed_package_specs()) + else: + results = [set(spack.db.get_installed(qs)) for qs in query_specs] + specs = set.union(*results) + + if not args.mode: + args.mode = 'short' + display_specs(specs, mode=args.mode) + -- cgit v1.2.3-70-g09d2 From 6b90017efa1f3157fe4be7d0c7b199b6e51b9fa8 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 28 Jan 2015 22:05:57 -0800 Subject: Fixed dumb link_tree bug, added test for link tree. --- lib/spack/llnl/util/filesystem.py | 8 +- lib/spack/llnl/util/link_tree.py | 197 ++++++++++++++++++++++---------------- lib/spack/spack/test/__init__.py | 3 +- lib/spack/spack/test/link_tree.py | 153 +++++++++++++++++++++++++++++ 4 files changed, 274 insertions(+), 87 deletions(-) create mode 100644 lib/spack/spack/test/link_tree.py (limited to 'lib') diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 0578415653..576aeb16bd 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## __all__ = ['set_install_permissions', 'install', 'expand_user', 'working_dir', - 'touch', 'mkdirp', 'force_remove', 'join_path', 'ancestor', + 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe'] import os @@ -204,6 +204,12 @@ def touch(path): os.utime(path, None) +def touchp(path): + """Like touch, but creates any parent directories needed for the file.""" + mkdirp(os.path.dirname(path)) + touch(path) + + def join_path(prefix, *args): path = str(prefix) for elt in args: diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py index 2d7126be2c..887f6f4d26 100644 --- a/lib/spack/llnl/util/link_tree.py +++ b/lib/spack/llnl/util/link_tree.py @@ -29,108 +29,116 @@ import os import shutil from llnl.util.filesystem import mkdirp +empty_file_name = '.spack-empty' -class LinkTree(object): - """Class to create trees of symbolic links from a source directory. - LinkTree objects are constructed with a source root. Their - methods allow you to create and delete trees of symbolic links - back to the source tree in specific destination directories. - Trees comprise symlinks only to files; directries are never - symlinked to, to prevent the source directory from ever being - modified. +def traverse_tree(source_root, dest_root, rel_path='', **kwargs): + """Traverse two filesystem trees simultaneously. + + Walks the LinkTree directory in pre or post order. Yields each + file in the source directory with a matching path from the dest + directory, along with whether the file is a directory. + e.g., for this tree:: + + root/ + a/ + file1 + file2 + b/ + file3 + + When called on dest, this yields:: + + ('root', 'dest') + ('root/a', 'dest/a') + ('root/a/file1', 'dest/a/file1') + ('root/a/file2', 'dest/a/file2') + ('root/b', 'dest/b') + ('root/b/file3', 'dest/b/file3') + + Optional args: + + order=[pre|post] -- Whether to do pre- or post-order traveral. + + ignore= -- Predicate indicating which files to ignore. + + follow_nonexisting -- Whether to descend into directories in + src that do not exit in dest. Default True. + + follow_links -- Whether to descend into symlinks in src. """ - def __init__(self, source_root): - self._root = source_root + follow_nonexisting = kwargs.get('follow_nonexisting', True) + follow_links = kwargs.get('follow_link', False) + # Yield in pre or post order? + order = kwargs.get('order', 'pre') + if order not in ('pre', 'post'): + raise ValueError("Order must be 'pre' or 'post'.") - def traverse(self, dest_root, **kwargs): - """Traverse LinkTree root and dest simultaneously. + # List of relative paths to ignore under the src root. + ignore = kwargs.get('ignore', lambda filename: False) - Walks the LinkTree directory in pre or post order. Yields - each file in the source directory with a matching path from - the dest directory. e.g., for this tree:: + # Don't descend into ignored directories + if ignore(rel_path): + return - root/ - a/ - file1 - file2 - b/ - file3 + source_path = os.path.join(source_root, rel_path) + dest_path = os.path.join(dest_root, rel_path) - When called on dest, this yields:: + # preorder yields directories before children + if order == 'pre': + yield (source_path, dest_path) - ('root', 'dest') - ('root/a', 'dest/a') - ('root/a/file1', 'dest/a/file1') - ('root/a/file2', 'dest/a/file2') - ('root/b', 'dest/b') - ('root/b/file3', 'dest/b/file3') + for f in os.listdir(source_path): + source_child = os.path.join(source_path, f) + dest_child = os.path.join(dest_path, f) - Optional args: + # Treat as a directory + if os.path.isdir(source_child) and ( + follow_links or not os.path.islink(source_child)): - order=[pre|post] -- Whether to do pre- or post-order traveral. + # When follow_nonexisting isn't set, don't descend into dirs + # in source that do not exist in dest + if follow_nonexisting or os.path.exists(dest_child): + tuples = traverse_tree(source_child, dest_child, rel_path, **kwargs) + for t in tuples: yield t - ignore= -- Predicate indicating which files to ignore. + # Treat as a file. + elif not ignore(os.path.join(rel_path, f)): + yield (source_child, dest_child) + + if order == 'post': + yield (source_path, dest_path) - follow_nonexisting -- Whether to descend into directories in - src that do not exit in dest. - """ - # Yield directories before or after their contents. - order = kwargs.get('order', 'pre') - if order not in ('pre', 'post'): - raise ValueError("Order must be 'pre' or 'post'.") - - # List of relative paths to ignore under the src root. - ignore = kwargs.get('ignore', lambda filename: False) - - # Whether to descend when dirs dont' exist in dest. - follow_nonexisting = kwargs.get('follow_nonexisting', True) - - for dirpath, dirnames, filenames in os.walk(self._root): - rel_path = dirpath[len(self._root):] - rel_path = rel_path.lstrip(os.path.sep) - dest_dirpath = os.path.join(dest_root, rel_path) - - # Don't descend into ignored directories - if ignore(dest_dirpath): - return - - # Don't descend into dirs in dest that do not exist in src. - if not follow_nonexisting: - dirnames[:] = [ - d for d in dirnames - if os.path.exists(os.path.join(dest_dirpath, d))] - - # preorder yields directories before children - if order == 'pre': - yield (dirpath, dest_dirpath) - - for name in filenames: - src_file = os.path.join(dirpath, name) - dest_file = os.path.join(dest_dirpath, name) - - # Ignore particular paths inside the install root. - src_relpath = src_file[len(self._root):] - src_relpath = src_relpath.lstrip(os.path.sep) - if ignore(src_relpath): - continue - yield (src_file, dest_file) +class LinkTree(object): + """Class to create trees of symbolic links from a source directory. - # postorder yields directories after children - if order == 'post': - yield (dirpath, dest_dirpath) + LinkTree objects are constructed with a source root. Their + methods allow you to create and delete trees of symbolic links + back to the source tree in specific destination directories. + Trees comprise symlinks only to files; directries are never + symlinked to, to prevent the source directory from ever being + modified. + + """ + def __init__(self, source_root): + if not os.path.exists(source_root): + raise IOError("No such file or directory: '%s'", source_root) + self._root = source_root def find_conflict(self, dest_root, **kwargs): - """Returns the first file in dest that also exists in src.""" + """Returns the first file in dest that conflicts with src""" kwargs['follow_nonexisting'] = False - for src, dest in self.traverse(dest_root, **kwargs): - if os.path.exists(dest) and not os.path.isdir(dest): + for src, dest in traverse_tree(self._root, dest_root, **kwargs): + if os.path.isdir(src): + if os.path.exists(dest) and not os.path.isdir(dest): + return dest + elif os.path.exists(dest): return dest return None @@ -138,9 +146,20 @@ class LinkTree(object): def merge(self, dest_root, **kwargs): """Link all files in src into dest, creating directories if necessary.""" kwargs['order'] = 'pre' - for src, dest in self.traverse(dest_root, **kwargs): + for src, dest in traverse_tree(self._root, dest_root, **kwargs): if os.path.isdir(src): - mkdirp(dest) + if not os.path.exists(dest): + mkdirp(dest) + continue + + if not os.path.isdir(dest): + raise ValueError("File blocks directory: %s" % dest) + + # mark empty directories so they aren't removed on unmerge. + if not os.listdir(dest): + marker = os.path.join(dest, empty_file_name) + touch(marker) + else: assert(not os.path.exists(dest)) os.symlink(src, dest) @@ -153,12 +172,20 @@ class LinkTree(object): """ kwargs['order'] = 'post' - for src, dest in self.traverse(dest_root, **kwargs): - if os.path.isdir(dest): + for src, dest in traverse_tree(self._root, dest_root, **kwargs): + if os.path.isdir(src): + if not os.path.isdir(dest): + raise ValueError("File blocks directory: %s" % dest) + + # remove directory if it is empty. if not os.listdir(dest): - # TODO: what if empty directories were present pre-merge? shutil.rmtree(dest, ignore_errors=True) + # remove empty dir marker if present. + marker = os.path.join(dest, empty_file_name) + if os.path.exists(marker): + os.remove(marker) + elif os.path.exists(dest): if not os.path.islink(dest): raise ValueError("%s is not a link tree!" % dest) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 0eda667abc..c53e6774fc 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -51,7 +51,8 @@ test_names = ['versions', 'hg_fetch', 'mirror', 'url_extrapolate', - 'cc'] + 'cc', + 'link_tree'] def list_tests(): diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py new file mode 100644 index 0000000000..bc7c2c6b5e --- /dev/null +++ b/lib/spack/spack/test/link_tree.py @@ -0,0 +1,153 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import unittest +import shutil +import tempfile +from contextlib import closing + +from llnl.util.filesystem import * +from llnl.util.link_tree import LinkTree + +from spack.stage import Stage + + +class LinkTreeTest(unittest.TestCase): + """Tests Spack's LinkTree class.""" + + def setUp(self): + self.stage = Stage('link-tree-test') + + with working_dir(self.stage.path): + touchp('source/1') + touchp('source/a/b/2') + touchp('source/a/b/3') + touchp('source/c/4') + touchp('source/c/d/5') + touchp('source/c/d/6') + touchp('source/c/d/e/7') + + source_path = os.path.join(self.stage.path, 'source') + self.link_tree = LinkTree(source_path) + + + def tearDown(self): + if self.stage: + self.stage.destroy() + + + def check_file_link(self, filename): + self.assertTrue(os.path.isfile(filename)) + self.assertTrue(os.path.islink(filename)) + + + def check_dir(self, filename): + self.assertTrue(os.path.isdir(filename)) + + + def test_merge_to_new_directory(self): + with working_dir(self.stage.path): + self.link_tree.merge('dest') + + self.check_file_link('dest/1') + self.check_file_link('dest/a/b/2') + self.check_file_link('dest/a/b/3') + self.check_file_link('dest/c/4') + self.check_file_link('dest/c/d/5') + self.check_file_link('dest/c/d/6') + self.check_file_link('dest/c/d/e/7') + + self.link_tree.unmerge('dest') + + self.assertFalse(os.path.exists('dest')) + + + def test_merge_to_existing_directory(self): + with working_dir(self.stage.path): + + touchp('dest/x') + touchp('dest/a/b/y') + + self.link_tree.merge('dest') + + self.check_file_link('dest/1') + self.check_file_link('dest/a/b/2') + self.check_file_link('dest/a/b/3') + self.check_file_link('dest/c/4') + self.check_file_link('dest/c/d/5') + self.check_file_link('dest/c/d/6') + self.check_file_link('dest/c/d/e/7') + + self.assertTrue(os.path.isfile('dest/x')) + self.assertTrue(os.path.isfile('dest/a/b/y')) + + self.link_tree.unmerge('dest') + + self.assertTrue(os.path.isfile('dest/x')) + self.assertTrue(os.path.isfile('dest/a/b/y')) + + self.assertFalse(os.path.isfile('dest/1')) + self.assertFalse(os.path.isfile('dest/a/b/2')) + self.assertFalse(os.path.isfile('dest/a/b/3')) + self.assertFalse(os.path.isfile('dest/c/4')) + self.assertFalse(os.path.isfile('dest/c/d/5')) + self.assertFalse(os.path.isfile('dest/c/d/6')) + self.assertFalse(os.path.isfile('dest/c/d/e/7')) + + + def test_merge_with_empty_directories(self): + with working_dir(self.stage.path): + mkdirp('dest/f/g') + mkdirp('dest/a/b/h') + + self.link_tree.merge('dest') + self.link_tree.unmerge('dest') + + self.assertFalse(os.path.exists('dest/1')) + self.assertFalse(os.path.exists('dest/a/b/2')) + self.assertFalse(os.path.exists('dest/a/b/3')) + self.assertFalse(os.path.exists('dest/c/4')) + self.assertFalse(os.path.exists('dest/c/d/5')) + self.assertFalse(os.path.exists('dest/c/d/6')) + self.assertFalse(os.path.exists('dest/c/d/e/7')) + + self.assertTrue(os.path.isdir('dest/a/b/h')) + self.assertTrue(os.path.isdir('dest/f/g')) + + + def test_ignore(self): + with working_dir(self.stage.path): + touchp('source/.spec') + touchp('dest/.spec') + + self.link_tree.merge('dest', ignore=lambda x: x == '.spec') + self.link_tree.unmerge('dest', ignore=lambda x: x == '.spec') + + self.assertFalse(os.path.exists('dest/1')) + self.assertFalse(os.path.exists('dest/a')) + self.assertFalse(os.path.exists('dest/c')) + + self.assertTrue(os.path.isfile('source/.spec')) + self.assertTrue(os.path.isfile('dest/.spec')) -- cgit v1.2.3-70-g09d2 From 2d9190d264dd276853aca41998fffbab1baecdb0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 2 Feb 2015 06:09:35 -0800 Subject: Add extensions command. --- lib/spack/llnl/util/link_tree.py | 5 ++- lib/spack/spack/cmd/extensions.py | 9 ++-- lib/spack/spack/directory_layout.py | 2 +- lib/spack/spack/package.py | 5 +++ lib/spack/spack/packages.py | 5 +++ lib/spack/spack/spec.py | 7 +++ var/spack/packages/py-basemap/package.py | 1 + var/spack/packages/python/package.py | 77 +++++++++++++++++++++++++++++--- 8 files changed, 99 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py index 887f6f4d26..4e4e48316e 100644 --- a/lib/spack/llnl/util/link_tree.py +++ b/lib/spack/llnl/util/link_tree.py @@ -27,7 +27,7 @@ __all__ = ['LinkTree'] import os import shutil -from llnl.util.filesystem import mkdirp +from llnl.util.filesystem import * empty_file_name = '.spack-empty' @@ -93,6 +93,7 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): for f in os.listdir(source_path): source_child = os.path.join(source_path, f) dest_child = os.path.join(dest_path, f) + rel_child = os.path.join(rel_path, f) # Treat as a directory if os.path.isdir(source_child) and ( @@ -101,7 +102,7 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): # When follow_nonexisting isn't set, don't descend into dirs # in source that do not exist in dest if follow_nonexisting or os.path.exists(dest_child): - tuples = traverse_tree(source_child, dest_child, rel_path, **kwargs) + tuples = traverse_tree(source_root, dest_root, rel_child, **kwargs) for t in tuples: yield t # Treat as a file. diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index 961d7e3f24..f28a388bf2 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -26,6 +26,7 @@ import sys from external import argparse import llnl.util.tty as tty +from llnl.util.tty.colify import colify import spack import spack.cmd @@ -66,10 +67,10 @@ def extensions(parser, args): exts = spack.install_layout.get_extensions(spec) if not exts: - tty.msg("%s has no activated extensions." % spec.short_spec) + tty.msg("%s has no activated extensions." % spec.cshort_spec) else: - tty.msg("Showing %d activated extension%s for package:" - % (len(exts), 's' if len(exts) > 1 else ''), - spec.short_spec) + tty.msg("Extensions for package %s:" % spec.cshort_spec) + colify(pkg.name for pkg in spack.db.extensions_for(spec)) print + tty.msg("%d currently activated:" % len(exts)) spack.cmd.find.display_specs(exts, mode=args.mode) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index ff327ed504..efc40a17a4 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -269,8 +269,8 @@ class SpecHashDirectoryLayout(DirectoryLayout): def get_extensions(self, spec): _check_concrete(spec) - path = self.extension_file_path(spec) extensions = set() + path = self.extension_file_path(spec) if os.path.exists(path): with closing(open(path)) as ext_file: for line in ext_file: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 0b6bc4ce6c..b905968540 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -522,6 +522,11 @@ class Package(object): return len(self.extendees) > 0 + def extends(self, spec): + return (spec.name in self.extendees and + spec.satisfies(self.extendees[spec.name][0])) + + @property def activated(self): if not self.spec.concrete: diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index bb5a94bcab..b3049e812f 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -112,6 +112,11 @@ class PackageDB(object): return providers + @_autospec + def extensions_for(self, extendee_spec): + return [p for p in self.all_packages() if p.extends(extendee_spec)] + + def dirname_for_package_name(self, pkg_name): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 2f4fe9ca24..dffdccaddb 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -552,6 +552,13 @@ class Spec(object): return self.format('$_$@$%@$+$=$#') + @property + def cshort_spec(self): + """Returns a version of the spec with the dependencies hashed + instead of completely enumerated.""" + return self.format('$_$@$%@$+$=$#', color=True) + + @property def prefix(self): return Prefix(spack.install_layout.path_for_spec(self)) diff --git a/var/spack/packages/py-basemap/package.py b/var/spack/packages/py-basemap/package.py index 8955bf8827..7b6d8e7e65 100644 --- a/var/spack/packages/py-basemap/package.py +++ b/var/spack/packages/py-basemap/package.py @@ -11,6 +11,7 @@ class PyBasemap(Package): geos_version = {'1.0.7' : '3.3.3'} extends('python') + depends_on('py-setuptools') depends_on('py-numpy') depends_on('py-matplotlib') depends_on('py-pil') diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index a22bd54c82..8a6d574d9b 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -1,6 +1,9 @@ from spack import * +import spack import os import re +from contextlib import closing + class Python(Package): """The Python programming language.""" @@ -29,6 +32,10 @@ class Python(Package): make("install") + # ======================================================================== + # Set up environment to make install easy for python extensions. + # ======================================================================== + @property def python_lib_dir(self): return os.path.join('lib', 'python%d.%d' % self.version[:2]) @@ -60,21 +67,81 @@ class Python(Package): mkdirp(module.site_packages_dir) - def make_ignore(self, args): + # ======================================================================== + # Handle specifics of activating and deactivating python modules. + # ======================================================================== + + def python_ignore(self, ext_pkg, args): """Add some ignore files to activate/deactivate args.""" orig_ignore = args.get('ignore', lambda f: False) + def ignore(filename): - return (re.search(r'/site\.pyc?$', filename) or - re.search(r'\.pth$', filename) or + # Always ignore easy-install.pth, as it needs to be merged. + patterns = [r'easy-install\.pth$'] + + # Ignore pieces of setuptools installed by other packages. + if ext_pkg.name != 'py-setuptools': + patterns.append(r'/site\.pyc?$') + patterns.append(r'setuptools\.pth') + patterns.append(r'bin/easy_install[^/]*$') + patterns.append(r'setuptools.*egg$') + + return (any(re.search(p, filename) for p in patterns) or orig_ignore(filename)) + return ignore + def write_easy_install_pth(self, extensions): + paths = [] + for ext in extensions: + ext_site_packages = os.path.join(ext.prefix, self.site_packages_dir) + easy_pth = "%s/easy-install.pth" % ext_site_packages + + if not os.path.isfile(easy_pth): + continue + + with closing(open(easy_pth)) as f: + for line in f: + line = line.rstrip() + + # Skip lines matching these criteria + if not line: continue + if re.search(r'^(import|#)', line): continue + if (ext.name != 'py-setuptools' and + re.search(r'setuptools.*egg$', line)): continue + + paths.append(line) + + site_packages = os.path.join(self.prefix, self.site_packages_dir) + main_pth = "%s/easy-install.pth" % site_packages + + if not paths: + if os.path.isfile(main_pth): + os.remove(main_pth) + + else: + with closing(open(main_pth, 'w')) as f: + f.write("import sys; sys.__plen = len(sys.path)\n") + for path in paths: + f.write("%s\n" % path) + f.write("import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; " + "p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new)\n") + + def activate(self, ext_pkg, **args): - args.update(ignore=self.make_ignore(args)) + args.update(ignore=self.python_ignore(ext_pkg, args)) super(Python, self).activate(ext_pkg, **args) + extensions = set(spack.install_layout.get_extensions(self.spec)) + extensions.add(ext_pkg.spec) + self.write_easy_install_pth(extensions) + def deactivate(self, ext_pkg, **args): - args.update(ignore=self.make_ignore(args)) + args.update(ignore=self.python_ignore(ext_pkg, args)) super(Python, self).deactivate(ext_pkg, **args) + + extensions = set(spack.install_layout.get_extensions(self.spec)) + extensions.remove(ext_pkg.spec) + self.write_easy_install_pth(extensions) -- cgit v1.2.3-70-g09d2 From 5bde8359e8378bf8595a4bef343d1f50258f663d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 2 Feb 2015 07:58:52 -0800 Subject: More information in extensions command. --- lib/spack/spack/cmd/extensions.py | 39 ++++++++++++++++++++++++++++++--------- lib/spack/spack/packages.py | 6 ++++++ 2 files changed, 36 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index f28a388bf2..f6ccd7b515 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -54,9 +54,14 @@ def extensions(parser, args): if not args.spec: tty.die("extensions requires a package spec.") + # Checks spec = spack.cmd.parse_specs(args.spec) if len(spec) > 1: tty.die("Can only list extensions for one package.") + + if not spec[0].package.extendable: + tty.die("%s is not an extendable package." % spec[0].name) + spec = spack.cmd.disambiguate_spec(spec[0]) if not spec.package.extendable: @@ -65,12 +70,28 @@ def extensions(parser, args): if not args.mode: args.mode = 'short' - exts = spack.install_layout.get_extensions(spec) - if not exts: - tty.msg("%s has no activated extensions." % spec.cshort_spec) - else: - tty.msg("Extensions for package %s:" % spec.cshort_spec) - colify(pkg.name for pkg in spack.db.extensions_for(spec)) - print - tty.msg("%d currently activated:" % len(exts)) - spack.cmd.find.display_specs(exts, mode=args.mode) + # List package names of extensions + extensions = spack.db.extensions_for(spec) + if not extensions: + tty.msg("%s has no extensions." % spec.cshort_spec) + return + tty.msg("%s extensions:" % spec.cshort_spec) + colify(ext.name for ext in extensions) + + # List specs of installed extensions. + installed = [s.spec for s in spack.db.installed_extensions_for(spec)] + print + if not installed: + tty.msg("None activated.") + return + tty.msg("%d installed:" % len(installed)) + spack.cmd.find.display_specs(installed, mode=args.mode) + + # List specs of activated extensions. + activated = spack.install_layout.get_extensions(spec) + print + if not activated: + tty.msg("None activated.") + return + tty.msg("%d currently activated:" % len(exts)) + spack.cmd.find.display_specs(installed, mode=args.mode) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index b3049e812f..7ef8135c1a 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -117,6 +117,12 @@ class PackageDB(object): return [p for p in self.all_packages() if p.extends(extendee_spec)] + @_autospec + def installed_extensions_for(self, extendee_spec): + return [s.package for s in self.installed_package_specs() + if s.package.extends(extendee_spec)] + + def dirname_for_package_name(self, pkg_name): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" -- cgit v1.2.3-70-g09d2