diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/docs/basic_usage.rst | 39 | ||||
| -rw-r--r-- | lib/spack/docs/packaging_guide.rst | 34 | ||||
| -rw-r--r-- | lib/spack/docs/workflows.rst | 50 | ||||
| -rw-r--r-- | lib/spack/llnl/util/filesystem.py | 23 | ||||
| -rw-r--r-- | lib/spack/llnl/util/link_tree.py | 143 | ||||
| -rw-r--r-- | lib/spack/spack/binary_distribution.py | 4 | ||||
| -rw-r--r-- | lib/spack/spack/build_systems/aspell_dict.py | 12 | ||||
| -rw-r--r-- | lib/spack/spack/build_systems/python.py | 70 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/activate.py | 19 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/deactivate.py | 33 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/extensions.py | 14 | ||||
| -rw-r--r-- | lib/spack/spack/database.py | 4 | ||||
| -rw-r--r-- | lib/spack/spack/directory_layout.py | 48 | ||||
| -rw-r--r-- | lib/spack/spack/filesystem_view.py | 103 | ||||
| -rw-r--r-- | lib/spack/spack/hooks/extensions.py | 7 | ||||
| -rw-r--r-- | lib/spack/spack/package.py | 164 | ||||
| -rw-r--r-- | lib/spack/spack/relocate.py | 11 | ||||
| -rw-r--r-- | lib/spack/spack/store.py | 3 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/activate.py | 61 | ||||
| -rw-r--r-- | lib/spack/spack/test/test_activations.py | 211 | ||||
| -rw-r--r-- | lib/spack/spack/test/views.py | 48 |
21 files changed, 794 insertions, 307 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 1a7576c74e..ae180c0659 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -950,11 +950,11 @@ directly when you run ``python``: ImportError: No module named numpy >>> -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Extensions & Environment Modules -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^ +Using Extensions +^^^^^^^^^^^^^^^^ -There are two ways to get ``numpy`` working in Python. The first is +There are three ways to get ``numpy`` working in Python. The first is to use :ref:`shell-support`. You can simply ``use`` or ``load`` the module for the extension, and it will be added to the ``PYTHONPATH`` in your current shell. @@ -976,15 +976,26 @@ or, for dotkit: Now ``import numpy`` will succeed for as long as you keep your current session open. -^^^^^^^^^^^^^^^^^^^^^ -Activating Extensions -^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Activating Extensions in a View +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -It is often desirable to have certain packages *always* available as -part of a Python installation. Spack offers a more permanent solution -for this case. Instead of requiring users to load particular -environment modules, you can *activate* the package within the Python -installation: +The second way to use extensions is to create a view, which merges the +python installation along with the extensions into a single prefix. +See :ref:`filesystem-views` for a more in-depth description of views and +:ref:`cmd-spack-view` for usage of the ``spack view`` command. + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Activating Extensions Globally +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As an alternative to creating a merged prefix with Python and its extensions, +and prior to support for views, Spack has provided a means to install the +extension into the Spack installation prefix for the extendee. This has +typically been useful since extendable packages typically search their own +installation path for addons by default. + +Global activations are performed with the ``spack activate`` command: .. _cmd-spack-activate: @@ -1044,11 +1055,11 @@ the ``py-numpy`` into the prefix of the ``python`` package. To the python interpreter, it looks like ``numpy`` is installed in the ``site-packages`` directory. -The only limitation of activation is that you can only have a *single* +The only limitation of global activation is that you can only have a *single* version of an extension activated at a time. This is because multiple versions of the same extension would conflict if symbolically linked into the same prefix. Users who want a different version of a package -can still get it by using environment modules, but they will have to +can still get it by using environment modules or views, but they will have to explicitly load their preferred version. ^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 34be562c29..706144403d 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1854,18 +1854,38 @@ from being linked in at activation time. ``depends_on('python')`` and ``extends(python)`` in the same package. ``extends`` implies ``depends_on``. +----- +Views +----- + +As covered in :ref:`filesystem-views`, the ``spack view`` command can be +used to symlink a number of packages into a merged prefix. The methods of +``PackageViewMixin`` can be overridden to customize how packages are added +to views. Generally this can be used to create copies of specific files rather +than symlinking them when symlinking does not work. For example, ``Python`` +overrides ``add_files_to_view`` in order to create a copy of the ``python`` +binary since the real path of the Python executable is used to detect +extensions; as a consequence python extension packages (those inheriting from +``PythonPackage``) likewise override ``add_files_to_view`` in order to rewrite +shebang lines which point to the Python interpreter. + ^^^^^^^^^^^^^^^^^^^^^^^^^ Activation & deactivation ^^^^^^^^^^^^^^^^^^^^^^^^^ +Adding an extension to a view is referred to as an activation. If the view is +maintained in the Spack installation prefix of the extendee this is called a +global activation. Activations may involve updating some centralized state +that is maintained by the extendee package, so there can be additional work +for adding extensions compared with non-extension packages. + Spack's ``Package`` class has default ``activate`` and ``deactivate`` implementations that handle symbolically linking extensions' prefixes -into the directory of the parent package. However, extendable -packages can override these methods to add custom activate/deactivate -logic of their own. For example, the ``activate`` and ``deactivate`` -methods in the Python class use the symbolic linking, but they also -handle details surrounding Python's ``.pth`` files, and other aspects -of Python packaging. +into a specified view. Extendable packages can override these methods +to add custom activate/deactivate logic of their own. For example, +the ``activate`` and ``deactivate`` methods in the Python class handle +symbolic linking of extensions, but they also handle details surrounding +Python's ``.pth`` files, and other aspects of Python packaging. Spack's extensions mechanism is designed to be extensible, so that other packages (like Ruby, R, Perl, etc.) can provide their own @@ -1880,7 +1900,7 @@ Let's look at Python's activate function: This function is called on the *extendee* (Python). It first calls ``activate`` in the superclass, which handles symlinking the -extension package's prefix into this package's prefix. It then does +extension package's prefix into the specified view. It then does some special handling of the ``easy-install.pth`` file, part of Python's setuptools. diff --git a/lib/spack/docs/workflows.rst b/lib/spack/docs/workflows.rst index 78fc6e52a9..b7d5a0ce43 100644 --- a/lib/spack/docs/workflows.rst +++ b/lib/spack/docs/workflows.rst @@ -402,31 +402,6 @@ Numpy, core Python, BLAS/LAPACK and anything else needed: spack module loads --dependencies py-scipy -^^^^^^^^^^^^^^^^^^ -Extension Packages -^^^^^^^^^^^^^^^^^^ - -:ref:`packaging_extensions` may be used as an alternative to loading -Python (and similar systems) packages directly. If extensions are -activated, then ``spack load python`` will also load all the -extensions activated for the given ``python``. This reduces the need -for users to load a large number of modules. - -However, Spack extensions have two potential drawbacks: - -#. Activated packages that involve compiled C extensions may still - need their dependencies to be loaded manually. For example, - ``spack load openblas`` might be required to make ``py-numpy`` - work. - -#. Extensions "break" a core feature of Spack, which is that multiple - versions of a package can co-exist side-by-side. For example, - suppose you wish to run a Python package in two different - environments but the same basic Python --- one with - ``py-numpy@1.7`` and one with ``py-numpy@1.8``. Spack extensions - will not support this potential debugging use case. - - ^^^^^^^^^^^^^^ Dummy Packages ^^^^^^^^^^^^^^ @@ -447,6 +422,8 @@ it. A disadvantage is the set of packages will be consistent; this means you cannot load up two applications this way if they are not consistent with each other. +.. _filesystem-views: + ^^^^^^^^^^^^^^^^ Filesystem Views ^^^^^^^^^^^^^^^^ @@ -587,6 +564,29 @@ symlinks. At any time one can delete ``/path/to/MYVIEW`` or use ``spack view`` to manage it surgically. None of this will affect the real Spack install area. +^^^^^^^^^^^^^^^^^^ +Global Activations +^^^^^^^^^^^^^^^^^^ + +:ref:`cmd-spack-activate` may be used as an alternative to loading +Python (and similar systems) packages directly or creating a view. +If extensions are globally activated, then ``spack load python`` will +also load all the extensions activated for the given ``python``. +This reduces the need for users to load a large number of modules. + +However, Spack global activations have two potential drawbacks: + +#. Activated packages that involve compiled C extensions may still + need their dependencies to be loaded manually. For example, + ``spack load openblas`` might be required to make ``py-numpy`` + work. + +#. Global activations "break" a core feature of Spack, which is that + multiple versions of a package can co-exist side-by-side. For example, + suppose you wish to run a Python package in two different + environments but the same basic Python --- one with + ``py-numpy@1.7`` and one with ``py-numpy@1.8``. Spack extensions + will not support this potential debugging use case. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Discussion: Running Binaries diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index b2591c2cf6..f43745a551 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -79,6 +79,18 @@ __all__ = [ ] +def path_contains_subdirectory(path, root): + norm_root = os.path.abspath(root).rstrip(os.path.sep) + os.path.sep + norm_path = os.path.abspath(path).rstrip(os.path.sep) + os.path.sep + return norm_path.startswith(norm_root) + + +def same_path(path1, path2): + norm1 = os.path.abspath(path1).rstrip(os.path.sep) + norm2 = os.path.abspath(path2).rstrip(os.path.sep) + return norm1 == norm2 + + def filter_file(regex, repl, *filenames, **kwargs): r"""Like sed, but uses python regular expressions. @@ -281,6 +293,17 @@ def is_exe(path): return os.path.isfile(path) and os.access(path, os.X_OK) +def get_filetype(path_name): + """ + Return the output of file path_name as a string to identify file type. + """ + file = Executable('file') + file.add_default_env('LC_ALL', 'C') + output = file('-b', '-h', '%s' % path_name, + output=str, error=str) + return output.strip() + + def mkdirp(*paths): """Creates a directory, as well as parent directories if needed.""" for path in paths: diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py index b90661ccce..6deadf8109 100644 --- a/lib/spack/llnl/util/link_tree.py +++ b/lib/spack/llnl/util/link_tree.py @@ -29,6 +29,7 @@ import shutil import filecmp from llnl.util.filesystem import traverse_tree, mkdirp, touch +import llnl.util.tty as tty __all__ = ['LinkTree'] @@ -44,37 +45,49 @@ class LinkTree(object): Trees comprise symlinks only to files; directries are never symlinked to, to prevent the source directory from ever being modified. - """ - def __init__(self, source_root): if not os.path.exists(source_root): raise IOError("No such file or directory: '%s'", source_root) self._root = source_root - def find_conflict(self, dest_root, **kwargs): + def find_conflict(self, dest_root, ignore=None, + ignore_file_conflicts=False): """Returns the first file in dest that conflicts with src""" - kwargs['follow_nonexisting'] = False + ignore = ignore or (lambda x: False) + conflicts = self.find_dir_conflicts(dest_root, ignore) + + if not ignore_file_conflicts: + conflicts.extend( + dst for src, dst + in self.get_file_map(dest_root, ignore).items() + if os.path.exists(dst)) + + if conflicts: + return conflicts[0] + + def find_dir_conflicts(self, dest_root, ignore): + conflicts = [] + kwargs = {'follow_nonexisting': False, 'ignore': ignore} for src, dest in traverse_tree(self._root, dest_root, **kwargs): if os.path.isdir(src): if os.path.exists(dest) and not os.path.isdir(dest): - return dest - elif os.path.exists(dest): - return dest - return None - - def merge(self, dest_root, link=os.symlink, **kwargs): - """Link all files in src into dest, creating directories - if necessary. - If ignore_conflicts is True, do not break when the target exists but - rather return a list of files that could not be linked. - Note that files blocking directories will still cause an error. - """ - kwargs['order'] = 'pre' - ignore_conflicts = kwargs.get("ignore_conflicts", False) - existing = [] + conflicts.append("File blocks directory: %s" % dest) + elif os.path.exists(dest) and os.path.isdir(dest): + conflicts.append("Directory blocks directory: %s" % dest) + return conflicts + + def get_file_map(self, dest_root, ignore): + merge_map = {} + kwargs = {'follow_nonexisting': True, 'ignore': ignore} for src, dest in traverse_tree(self._root, dest_root, **kwargs): + if not os.path.isdir(src): + merge_map[src] = dest + return merge_map + + def merge_directories(self, dest_root, ignore): + for src, dest in traverse_tree(self._root, dest_root, ignore=ignore): if os.path.isdir(src): if not os.path.exists(dest): mkdirp(dest) @@ -88,31 +101,13 @@ class LinkTree(object): marker = os.path.join(dest, empty_file_name) touch(marker) - else: - if os.path.exists(dest): - if ignore_conflicts: - existing.append(src) - else: - raise AssertionError("File already exists: %s" % dest) - else: - link(src, dest) - if ignore_conflicts: - return existing - - def unmerge(self, dest_root, **kwargs): - """Unlink all files in dest that exist in src. - - Unlinks directories in dest if they are empty. - - """ - kwargs['order'] = 'post' - for src, dest in traverse_tree(self._root, dest_root, **kwargs): + def unmerge_directories(self, dest_root, ignore): + for src, dest in traverse_tree( + self._root, dest_root, ignore=ignore, order='post'): if os.path.isdir(src): - # Skip non-existing links. if not os.path.exists(dest): continue - - if not os.path.isdir(dest): + elif not os.path.isdir(dest): raise ValueError("File blocks directory: %s" % dest) # remove directory if it is empty. @@ -124,11 +119,61 @@ class LinkTree(object): if os.path.exists(marker): os.remove(marker) - elif os.path.exists(dest): - if not os.path.islink(dest): - raise ValueError("%s is not a link tree!" % dest) - # remove if dest is a hardlink/symlink to src; this will only - # be false if two packages are merged into a prefix and have a - # conflicting file - if filecmp.cmp(src, dest, shallow=True): - os.remove(dest) + def merge(self, dest_root, **kwargs): + """Link all files in src into dest, creating directories + if necessary. + If ignore_conflicts is True, do not break when the target exists but + rather return a list of files that could not be linked. + Note that files blocking directories will still cause an error. + """ + ignore_conflicts = kwargs.get("ignore_conflicts", False) + + ignore = kwargs.get('ignore', lambda x: False) + conflict = self.find_conflict( + dest_root, ignore=ignore, ignore_file_conflicts=ignore_conflicts) + if conflict: + raise MergeConflictError(conflict) + + self.merge_directories(dest_root, ignore) + existing = [] + merge_file = kwargs.get('merge_file', merge_link) + for src, dst in self.get_file_map(dest_root, ignore).items(): + if os.path.exists(dst): + existing.append(dst) + else: + merge_file(src, dst) + + for c in existing: + tty.warn("Could not merge: %s" % c) + + def unmerge(self, dest_root, **kwargs): + """Unlink all files in dest that exist in src. + + Unlinks directories in dest if they are empty. + """ + remove_file = kwargs.get('remove_file', remove_link) + ignore = kwargs.get('ignore', lambda x: False) + for src, dst in self.get_file_map(dest_root, ignore).items(): + remove_file(src, dst) + self.unmerge_directories(dest_root, ignore) + + +def merge_link(src, dest): + os.symlink(src, dest) + + +def remove_link(src, dest): + if not os.path.islink(dest): + raise ValueError("%s is not a link tree!" % dest) + # remove if dest is a hardlink/symlink to src; this will only + # be false if two packages are merged into a prefix and have a + # conflicting file + if filecmp.cmp(src, dest, shallow=True): + os.remove(dest) + + +class MergeConflictError(Exception): + + def __init__(self, path): + super(MergeConflictError, self).__init__( + "Package merge blocked by file: %s" % path) diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 1ae068e897..b96237ed99 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -34,7 +34,7 @@ from contextlib import closing import yaml import llnl.util.tty as tty -from llnl.util.filesystem import mkdirp, install_tree +from llnl.util.filesystem import mkdirp, install_tree, get_filetype import spack.cmd import spack.fetch_strategy as fs @@ -148,7 +148,7 @@ def write_buildinfo_file(prefix, workdir, rel=False): # of files potentially needing relocation if relocate.strings_contains_installroot( path_name, spack.store.layout.root): - filetype = relocate.get_filetype(path_name) + filetype = get_filetype(path_name) if relocate.needs_binary_relocation(filetype, os_id): rel_path_name = os.path.relpath(path_name, prefix) binary_to_relocate.append(rel_path_name) diff --git a/lib/spack/spack/build_systems/aspell_dict.py b/lib/spack/spack/build_systems/aspell_dict.py index c9369ca9a8..99039fcf28 100644 --- a/lib/spack/spack/build_systems/aspell_dict.py +++ b/lib/spack/spack/build_systems/aspell_dict.py @@ -27,6 +27,7 @@ from llnl.util.filesystem import filter_file from spack.build_systems.autotools import AutotoolsPackage from spack.directives import extends +from spack.package import ExtensionError from spack.util.executable import which @@ -42,6 +43,17 @@ class AspellDictPackage(AutotoolsPackage): extends('aspell') + def view_destination(self, view): + aspell_spec = self.spec['aspell'] + if view.root != aspell_spec.prefix: + raise ExtensionError( + 'aspell does not support non-global extensions') + aspell = aspell_spec.command + return aspell('dump', 'config', 'dict-dir', output=str).strip() + + def view_source(self): + return self.prefix.lib + def patch(self): filter_file(r'^dictdir=.*$', 'dictdir=/lib', 'configure') filter_file(r'^datadir=.*$', 'datadir=/lib', 'configure') diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py index 98fdab3be5..7933253eed 100644 --- a/lib/spack/spack/build_systems/python.py +++ b/lib/spack/spack/build_systems/python.py @@ -25,11 +25,14 @@ import inspect import os +import shutil from spack.directives import depends_on, extends from spack.package import PackageBase, run_after -from llnl.util.filesystem import working_dir +from llnl.util.filesystem import (working_dir, get_filetype, filter_file, + path_contains_subdirectory, same_path) +from llnl.util.lang import match_predicate class PythonPackage(PackageBase): @@ -116,6 +119,8 @@ class PythonPackage(PackageBase): depends_on('python', type=('build', 'run')) + py_namespace = None + def setup_file(self): """Returns the name of the setup file to use.""" return 'setup.py' @@ -403,3 +408,66 @@ class PythonPackage(PackageBase): # Check that self.prefix is there after installation run_after('install')(PackageBase.sanity_check_prefix) + + def view_file_conflicts(self, view, merge_map): + """Report all file conflicts, excepting special cases for python. + Specifically, this does not report errors for duplicate + __init__.py files for packages in the same namespace. + """ + conflicts = list(dst for src, dst in merge_map.items() + if os.path.exists(dst)) + + if conflicts and self.py_namespace: + ext_map = view.extensions_layout.extension_map(self.extendee_spec) + namespaces = set( + x.package.py_namespace for x in ext_map.values()) + namespace_re = ( + r'site-packages/{0}/__init__.py'.format(self.py_namespace)) + find_namespace = match_predicate(namespace_re) + if self.py_namespace in namespaces: + conflicts = list( + x for x in conflicts if not find_namespace(x)) + + return conflicts + + def add_files_to_view(self, view, merge_map): + bin_dir = self.spec.prefix.bin + python_prefix = self.extendee_spec.prefix + global_view = same_path(python_prefix, view.root) + for src, dst in merge_map.items(): + if os.path.exists(dst): + continue + elif global_view or not path_contains_subdirectory(src, bin_dir): + view.link(src, dst) + elif not os.path.islink(src): + shutil.copy2(src, dst) + if 'script' in get_filetype(src): + filter_file( + python_prefix, os.path.abspath(view.root), dst) + else: + orig_link_target = os.path.realpath(src) + new_link_target = os.path.abspath(merge_map[orig_link_target]) + view.link(new_link_target, dst) + + def remove_files_from_view(self, view, merge_map): + ignore_namespace = False + if self.py_namespace: + ext_map = view.extensions_layout.extension_map(self.extendee_spec) + remaining_namespaces = set( + spec.package.py_namespace for name, spec in ext_map.items() + if name != self.name) + if self.py_namespace in remaining_namespaces: + namespace_init = match_predicate( + r'site-packages/{0}/__init__.py'.format(self.py_namespace)) + ignore_namespace = True + + bin_dir = self.spec.prefix.bin + global_view = self.extendee_spec.prefix == view.root + for src, dst in merge_map.items(): + if ignore_namespace and namespace_init(dst): + continue + + if global_view or not path_contains_subdirectory(src, bin_dir): + view.remove_file(src, dst) + else: + os.remove(dst) diff --git a/lib/spack/spack/cmd/activate.py b/lib/spack/spack/cmd/activate.py index b0e4be91f1..73c4c63bce 100644 --- a/lib/spack/spack/cmd/activate.py +++ b/lib/spack/spack/cmd/activate.py @@ -27,7 +27,7 @@ import argparse import llnl.util.tty as tty import spack.cmd -from spack.directory_layout import YamlViewExtensionsLayout +from spack.filesystem_view import YamlFilesystemView description = "activate a package extension" section = "extensions" @@ -55,14 +55,17 @@ def activate(parser, args): if not spec.package.is_extension: tty.die("%s is not an extension." % spec.name) - layout = spack.store.extensions - if args.view is not None: - layout = YamlViewExtensionsLayout( - args.view, spack.store.layout) + if args.view: + target = args.view + else: + target = spec.package.extendee_spec.prefix - if spec.package.is_activated(extensions_layout=layout): + view = YamlFilesystemView(target, spack.store.layout) + + if spec.package.is_activated(view): tty.msg("Package %s is already activated." % specs[0].short_spec) return - spec.package.do_activate(extensions_layout=layout, - with_dependencies=not args.force) + # TODO: refactor FilesystemView.add_extension and use that here (so there + # aren't two ways of activating extensions) + spec.package.do_activate(view, with_dependencies=not args.force) diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index 30eb271412..fdf64ee0b7 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -27,7 +27,7 @@ import llnl.util.tty as tty import spack.cmd import spack.store -from spack.directory_layout import YamlViewExtensionsLayout +from spack.filesystem_view import YamlFilesystemView from spack.graph import topological_sort description = "deactivate a package extension" @@ -59,25 +59,29 @@ def deactivate(parser, args): spec = spack.cmd.disambiguate_spec(specs[0]) pkg = spec.package - layout = spack.store.extensions - if args.view is not None: - layout = YamlViewExtensionsLayout( - args.view, spack.store.layout) + if args.view: + target = args.view + elif pkg.is_extension: + target = pkg.extendee_spec.prefix + elif pkg.extendable: + target = spec.prefix + + view = YamlFilesystemView(target, spack.store.layout) if args.all: if pkg.extendable: tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec) ext_pkgs = spack.store.db.activated_extensions_for( - spec, extensions_layout=layout) + spec, view.extensions_layout) for ext_pkg in ext_pkgs: ext_pkg.spec.normalize() - if ext_pkg.is_activated(): - ext_pkg.do_deactivate(force=True, extensions_layout=layout) + if ext_pkg.is_activated(view): + ext_pkg.do_deactivate(view, force=True) elif pkg.is_extension: if not args.force and \ - not spec.package.is_activated(extensions_layout=layout): + not spec.package.is_activated(view): tty.die("%s is not activated." % pkg.spec.short_spec) tty.msg("Deactivating %s and all dependencies." % @@ -90,11 +94,8 @@ def deactivate(parser, args): espec = index[name] epkg = espec.package if epkg.extends(pkg.extendee_spec): - if epkg.is_activated(extensions_layout=layout) or \ - args.force: - - epkg.do_deactivate( - force=args.force, extensions_layout=layout) + if epkg.is_activated(view) or args.force: + epkg.do_deactivate(view, force=args.force) else: tty.die( @@ -107,7 +108,7 @@ def deactivate(parser, args): "Did you mean 'spack deactivate --all'?") if not args.force and \ - not spec.package.is_activated(extensions_layout=layout): + not spec.package.is_activated(view): tty.die("Package %s is not activated." % specs[0].short_spec) - spec.package.do_deactivate(force=args.force, extensions_layout=layout) + spec.package.do_deactivate(view, force=args.force) diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index a428e85035..9e17dbb81a 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -31,7 +31,7 @@ import spack.cmd import spack.cmd.find import spack.repo import spack.store -from spack.directory_layout import YamlViewExtensionsLayout +from spack.filesystem_view import YamlFilesystemView description = "list extensions for package" section = "extensions" @@ -113,10 +113,12 @@ def extensions(parser, args): tty.msg("%d extensions:" % len(extensions)) colify(ext.name for ext in extensions) - layout = spack.store.extensions - if args.view is not None: - layout = YamlViewExtensionsLayout( - args.view, spack.store.layout) + if args.view: + target = args.view + else: + target = spec.prefix + + view = YamlFilesystemView(target, spack.store.layout) if show_installed: # @@ -137,7 +139,7 @@ def extensions(parser, args): # # List specs of activated extensions. # - activated = layout.extension_map(spec) + activated = view.extensions_layout.extension_map(spec) if show_all: print if not activated: diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 4bc6891ec8..4d82643b38 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -58,6 +58,7 @@ import spack.repo import spack.spec import spack.util.spack_yaml as syaml import spack.util.spack_json as sjson +from spack.filesystem_view import YamlFilesystemView from spack.util.crypto import bit_length from spack.directory_layout import DirectoryLayoutError from spack.error import SpackError @@ -823,7 +824,8 @@ class Database(object): the given spec """ if extensions_layout is None: - extensions_layout = spack.store.extensions + view = YamlFilesystemView(extendee_spec.prefix, spack.store.layout) + extensions_layout = view.extensions_layout for spec in self.query(): try: extensions_layout.check_activated(extendee_spec, spec) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index b392af01eb..6b4e9c7336 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -128,7 +128,6 @@ class ExtensionsLayout(object): """ def __init__(self, root, **kwargs): self.root = root - self.link = kwargs.get("link", os.symlink) def add_extension(self, spec, ext_spec): """Add to the list of currently installed extensions.""" @@ -313,14 +312,14 @@ class YamlDirectoryLayout(DirectoryLayout): return by_hash -class YamlExtensionsLayout(ExtensionsLayout): - """Implements globally activated extensions within a YamlDirectoryLayout. +class YamlViewExtensionsLayout(ExtensionsLayout): + """Maintain extensions within a view. """ def __init__(self, root, layout): """layout is the corresponding YamlDirectoryLayout object for which we implement extensions. """ - super(YamlExtensionsLayout, self).__init__(root) + super(YamlViewExtensionsLayout, self).__init__(root) self.layout = layout self.extension_file_name = 'extensions.yaml' @@ -354,19 +353,29 @@ class YamlExtensionsLayout(ExtensionsLayout): raise NoSuchExtensionError(spec, ext_spec) def extension_file_path(self, spec): - """Gets full path to an installed package's extension file""" + """Gets full path to an installed package's extension file, which + keeps track of all the extensions for that package which have been + added to this view. + """ _check_concrete(spec) - return os.path.join(self.layout.metadata_path(spec), - self.extension_file_name) + normalize_path = lambda p: ( + os.path.abspath(p).rstrip(os.path.sep)) + if normalize_path(spec.prefix) == normalize_path(self.root): + # For backwards compatibility, when the root is the extended + # package's installation directory, do not include the spec name + # as a subdirectory. + components = [self.root, self.layout.metadata_dir, + self.extension_file_name] + else: + components = [self.root, self.layout.metadata_dir, spec.name, + self.extension_file_name] + return os.path.join(*components) def extension_map(self, spec): """Defensive copying version of _extension_map() for external API.""" _check_concrete(spec) return self._extension_map(spec).copy() - def extendee_target_directory(self, extendee): - return extendee.prefix - def remove_extension(self, spec, ext_spec): _check_concrete(spec) _check_concrete(ext_spec) @@ -419,6 +428,8 @@ class YamlExtensionsLayout(ExtensionsLayout): # Create a temp file in the same directory as the actual file. dirname, basename = os.path.split(path) + mkdirp(dirname) + tmp = tempfile.NamedTemporaryFile( prefix=basename, dir=dirname, delete=False) @@ -436,23 +447,6 @@ class YamlExtensionsLayout(ExtensionsLayout): os.rename(tmp.name, path) -class YamlViewExtensionsLayout(YamlExtensionsLayout): - """Governs the directory layout present when creating filesystem views in a - certain root folder. - - Meant to replace YamlDirectoryLayout when working with filesystem views. - """ - - def extension_file_path(self, spec): - """Gets the full path to an installed package's extension file.""" - _check_concrete(spec) - return os.path.join(self.root, self.layout.metadata_dir, spec.name, - self.extension_file_name) - - def extendee_target_directory(self, extendee): - return self.root - - class DirectoryLayoutError(SpackError): """Superclass for directory layout errors.""" diff --git a/lib/spack/spack/filesystem_view.py b/lib/spack/spack/filesystem_view.py index ead18c5836..0f8c8c4282 100644 --- a/lib/spack/spack/filesystem_view.py +++ b/lib/spack/spack/filesystem_view.py @@ -22,14 +22,16 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import filecmp import functools as ft import os import re import shutil import sys -from llnl.util.link_tree import LinkTree +from llnl.util.link_tree import LinkTree, MergeConflictError from llnl.util import tty +from llnl.util.lang import match_predicate import spack.spec import spack.store @@ -223,20 +225,9 @@ class YamlFilesystemView(FilesystemView): % colorize_spec(spec)) return True - try: - if not spec.package.is_activated(self.extensions_layout): - spec.package.do_activate( - ignore_conflicts=self.ignore_conflicts, - with_dependencies=False, # already taken care of - # in add_specs() - verbose=self.verbose, - extensions_layout=self.extensions_layout) - - except ExtensionAlreadyInstalledError: - # As we use sets in add_specs(), the order in which packages get - # activated is essentially random. So this spec might have already - # been activated as dependency of another package -> fail silently - pass + if not spec.package.is_activated(self): + spec.package.do_activate( + self, verbose=self.verbose, with_dependencies=False) # make sure the meta folder is linked as well (this is not done by the # extension-activation mechnism) @@ -274,29 +265,66 @@ class YamlFilesystemView(FilesystemView): long=False) return False - tree = LinkTree(spec.prefix) - - if not self.ignore_conflicts: - conflict = tree.find_conflict(self.root) - if conflict is not None: - tty.error(self._croot + - "Cannot link package %s, file already exists: %s" - % (spec.name, conflict)) - return False + self.merge(spec) - conflicts = tree.merge(self.root, link=self.link, - ignore=ignore_metadata_dir, - ignore_conflicts=self.ignore_conflicts) self.link_meta_folder(spec) - if self.ignore_conflicts: - for c in conflicts: - tty.warn(self._croot + "Could not link: %s" % c) - if self.verbose: tty.info(self._croot + 'Linked package: %s' % colorize_spec(spec)) return True + def merge(self, spec, ignore=None): + pkg = spec.package + view_source = pkg.view_source() + view_dst = pkg.view_destination(self) + + tree = LinkTree(view_source) + + ignore = ignore or (lambda f: False) + ignore_file = match_predicate( + self.layout.hidden_file_paths, ignore) + + # check for dir conflicts + conflicts = tree.find_dir_conflicts(view_dst, ignore_file) + + merge_map = tree.get_file_map(view_dst, ignore_file) + if not self.ignore_conflicts: + conflicts.extend(pkg.view_file_conflicts(self, merge_map)) + + if conflicts: + raise MergeConflictError(conflicts[0]) + + # merge directories with the tree + tree.merge_directories(view_dst, ignore_file) + + pkg.add_files_to_view(self, merge_map) + + def unmerge(self, spec, ignore=None): + pkg = spec.package + view_source = pkg.view_source() + view_dst = pkg.view_destination(self) + + tree = LinkTree(view_source) + + ignore = ignore or (lambda f: False) + ignore_file = match_predicate( + self.layout.hidden_file_paths, ignore) + + merge_map = tree.get_file_map(view_dst, ignore_file) + pkg.remove_files_from_view(self, merge_map) + + # now unmerge the directory tree + tree.unmerge_directories(view_dst, ignore_file) + + def remove_file(self, src, dest): + if not os.path.islink(dest): + raise ValueError("%s is not a link tree!" % dest) + # remove if dest is a hardlink/symlink to src; this will only + # be false if two packages are merged into a prefix and have a + # conflicting file + if filecmp.cmp(src, dest, shallow=True): + os.remove(dest) + def check_added(self, spec): assert spec.concrete return spec == self.get_spec(spec) @@ -364,11 +392,11 @@ class YamlFilesystemView(FilesystemView): # The spec might have been deactivated as depdency of another package # already - if spec.package.is_activated(self.extensions_layout): + if spec.package.is_activated(self): spec.package.do_deactivate( + self, verbose=self.verbose, - remove_dependents=with_dependents, - extensions_layout=self.extensions_layout) + remove_dependents=with_dependents) self.unlink_meta_folder(spec) def remove_standalone(self, spec): @@ -380,8 +408,7 @@ class YamlFilesystemView(FilesystemView): 'Skipping package not linked in view: %s' % spec.name) return - tree = LinkTree(spec.prefix) - tree.unmerge(self.root, ignore=ignore_metadata_dir) + self.unmerge(spec) self.unlink_meta_folder(spec) if self.verbose: @@ -545,7 +572,3 @@ def get_dependencies(specs): retval = set() set(map(retval.update, (set(s.traverse()) for s in specs))) return retval - - -def ignore_metadata_dir(f): - return f in spack.store.layout.hidden_file_paths diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py index 94fe9f3cf4..109a23ab1b 100644 --- a/lib/spack/spack/hooks/extensions.py +++ b/lib/spack/spack/hooks/extensions.py @@ -22,6 +22,8 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import spack +from spack.filesystem_view import YamlFilesystemView def pre_uninstall(spec): @@ -29,6 +31,9 @@ def pre_uninstall(spec): assert spec.concrete if pkg.is_extension: - if pkg.is_activated(): + target = pkg.extendee_spec.prefix + view = YamlFilesystemView(target, spack.store.layout) + + if pkg.is_activated(view): # deactivate globally pkg.do_deactivate(force=True) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 73a872545c..dab963740f 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -75,6 +75,7 @@ from llnl.util.lang import memoized from llnl.util.link_tree import LinkTree from llnl.util.tty.log import log_output from llnl.util.tty.color import colorize +from spack.filesystem_view import YamlFilesystemView from spack.util.executable import which from spack.stage import Stage, ResourceStage, StageComposite from spack.util.environment import dump_environment @@ -260,7 +261,54 @@ def on_package_attributes(**attr_dict): return _execute_under_condition -class PackageBase(with_metaclass(PackageMeta, object)): +class PackageViewMixin(object): + """This collects all functionality related to adding installed Spack + package to views. Packages can customize how they are added to views by + overriding these functions. + """ + def view_source(self): + """The source root directory that will be added to the view: files are + added such that their path relative to the view destination matches + their path relative to the view source. + """ + return self.spec.prefix + + def view_destination(self, view): + """The target root directory: each file is added relative to this + directory. + """ + return view.root + + def view_file_conflicts(self, view, merge_map): + """Report any files which prevent adding this package to the view. The + default implementation looks for any files which already exist. + Alternative implementations may allow some of the files to exist in + the view (in this case they would be omitted from the results). + """ + return set(dst for dst in merge_map.values() if os.path.exists(dst)) + + def add_files_to_view(self, view, merge_map): + """Given a map of package files to destination paths in the view, add + the files to the view. By default this adds all files. Alternative + implementations may skip some files, for example if other packages + linked into the view already include the file. + """ + for src, dst in merge_map.items(): + if not os.path.exists(dst): + view.link(src, dst) + + def remove_files_from_view(self, view, merge_map): + """Given a map of package files to files currently linked in the view, + remove the files from the view. The default implementation removes all + files. Alternative implementations may not remove all files. For + example if two packages include the same file, it should only be + removed when both packages are removed. + """ + for src, dst in merge_map.items(): + view.remove_file(src, dst) + + +class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): """This is the superclass for all spack packages. ***The Package class*** @@ -942,13 +990,12 @@ class PackageBase(with_metaclass(PackageMeta, object)): s = self.extendee_spec return s and spec.satisfies(s) - def is_activated(self, extensions_layout=None): + def is_activated(self, view): """Return True if package is activated.""" if not self.is_extension: raise ValueError( - "is_extension called on package that is not an extension.") - if extensions_layout is None: - extensions_layout = spack.store.extensions + "is_activated called on package that is not an extension.") + extensions_layout = view.extensions_layout exts = extensions_layout.extension_map(self.extendee_spec) return (self.name in exts) and (exts[self.name] == self.spec) @@ -1979,8 +2026,7 @@ class PackageBase(with_metaclass(PackageMeta, object)): raise ActivationError("%s does not extend %s!" % (self.name, self.extendee.name)) - def do_activate(self, with_dependencies=True, ignore_conflicts=False, - verbose=True, extensions_layout=None): + def do_activate(self, view=None, with_dependencies=True, verbose=True): """Called on an extension to invoke the extendee's activate method. Commands should call this routine, and should not call @@ -1991,9 +2037,11 @@ class PackageBase(with_metaclass(PackageMeta, object)): (self.spec.cshort_spec, self.extendee_spec.cshort_spec)) self._sanity_check_extension() + if not view: + view = YamlFilesystemView( + self.extendee_spec.prefix, spack.store.layout) - if extensions_layout is None: - extensions_layout = spack.store.extensions + extensions_layout = view.extensions_layout extensions_layout.check_extension_conflict( self.extendee_spec, self.spec) @@ -2001,17 +2049,13 @@ class PackageBase(with_metaclass(PackageMeta, object)): # Activate any package dependencies that are also extensions. if with_dependencies: for spec in self.dependency_activations(): - if not spec.package.is_activated( - extensions_layout=extensions_layout): + if not spec.package.is_activated(view): spec.package.do_activate( - with_dependencies=with_dependencies, - ignore_conflicts=ignore_conflicts, - verbose=verbose, - extensions_layout=extensions_layout) + view, with_dependencies=with_dependencies, + verbose=verbose) self.extendee_spec.package.activate( - self, extensions_layout=extensions_layout, - ignore_conflicts=ignore_conflicts, **self.extendee_args) + self, view, **self.extendee_args) extensions_layout.add_extension(self.extendee_spec, self.spec) @@ -2025,41 +2069,22 @@ class PackageBase(with_metaclass(PackageMeta, object)): return (spec for spec in self.spec.traverse(root=False, deptype='run') if spec.package.extends(self.extendee_spec)) - def activate(self, extension, ignore_conflicts=False, **kwargs): - """Make extension package usable by linking all its files to a target - provided by the directory layout (depending if the user wants to - activate globally or in a specified file system view). - - Package authors can override this method to support other - extension mechanisms. Spack internals (commands, hooks, etc.) - should call do_activate() method so that proper checks are - always executed. - + def activate(self, extension, view, **kwargs): """ - extensions_layout = kwargs.get("extensions_layout", - spack.store.extensions) - target = extensions_layout.extendee_target_directory(self) - - def ignore(filename): - return (filename in spack.store.layout.hidden_file_paths or - kwargs.get('ignore', lambda f: False)(filename)) - - tree = LinkTree(extension.prefix) + Add the extension to the specified view. - conflict = tree.find_conflict(target, ignore=ignore) - if not conflict: - pass - elif ignore_conflicts: - tty.warn("While activating %s, found conflict %s" % - (self.spec.cshort_spec, conflict)) - else: - raise ExtensionConflictError(conflict) + Package authors can override this function to maintain some + centralized state related to the set of activated extensions + for a package. - tree.merge(target, ignore=ignore, link=extensions_layout.link, - ignore_conflicts=ignore_conflicts) + Spack internals (commands, hooks, etc.) should call + do_activate() method so that proper checks are always executed. + """ + view.merge(extension.spec, ignore=kwargs.get('ignore', None)) - def do_deactivate(self, **kwargs): - """Called on the extension to invoke extendee's deactivate() method. + def do_deactivate(self, view=None, **kwargs): + """Remove this extension package from the specified view. Called + on the extension to invoke extendee's deactivate() method. `remove_dependents=True` deactivates extensions depending on this package instead of raising an error. @@ -2068,8 +2093,11 @@ class PackageBase(with_metaclass(PackageMeta, object)): force = kwargs.get('force', False) verbose = kwargs.get("verbose", True) remove_dependents = kwargs.get("remove_dependents", False) - extensions_layout = kwargs.get("extensions_layout", - spack.store.extensions) + + if not view: + view = YamlFilesystemView( + self.extendee_spec.prefix, spack.store.layout) + extensions_layout = view.extensions_layout # Allow a force deactivate to happen. This can unlink # spurious files if something was corrupted. @@ -2094,13 +2122,11 @@ class PackageBase(with_metaclass(PackageMeta, object)): aspec.cshort_spec)) self.extendee_spec.package.deactivate( - self, - extensions_layout=extensions_layout, - **self.extendee_args) + self, view, **self.extendee_args) # redundant activation check -- makes SURE the spec is not # still activated even if something was wrong above. - if self.is_activated(extensions_layout): + if self.is_activated(view): extensions_layout.remove_extension( self.extendee_spec, self.spec) @@ -2110,26 +2136,23 @@ class PackageBase(with_metaclass(PackageMeta, object)): (self.spec.short_spec, self.extendee_spec.cformat("$_$@$+$%@"))) - def deactivate(self, extension, **kwargs): - """Unlinks all files from extension out of this package's install dir - or the corresponding filesystem view. + def deactivate(self, extension, view, **kwargs): + """ + Remove all extension files from the specified view. Package authors can override this method to support other extension mechanisms. Spack internals (commands, hooks, etc.) should call do_deactivate() method so that proper checks are always executed. - """ - extensions_layout = kwargs.get("extensions_layout", - spack.store.extensions) - target = extensions_layout.extendee_target_directory(self) + view.unmerge(extension.spec, ignore=kwargs.get('ignore', None)) - def ignore(filename): - return (filename in spack.store.layout.hidden_file_paths or - kwargs.get('ignore', lambda f: False)(filename)) - - tree = LinkTree(extension.prefix) - tree.unmerge(target, ignore=ignore) + def view(self): + """Create a view with the prefix of this package as the root. + Extensions added to this view will modify the installation prefix of + this package. + """ + return YamlFilesystemView(self.prefix, spack.store.layout) def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" @@ -2409,13 +2432,6 @@ class ExtensionError(PackageError): pass -class ExtensionConflictError(ExtensionError): - - def __init__(self, path): - super(ExtensionConflictError, self).__init__( - "Extension blocked by file: %s" % path) - - class ActivationError(ExtensionError): def __init__(self, msg, long_msg=None): diff --git a/lib/spack/spack/relocate.py b/lib/spack/spack/relocate.py index 9800718dff..6996cc1d5c 100644 --- a/lib/spack/spack/relocate.py +++ b/lib/spack/spack/relocate.py @@ -256,17 +256,6 @@ def modify_macho_object(cur_path, rpaths, deps, idpath, return -def get_filetype(path_name): - """ - Return the output of file path_name as a string to identify file type. - """ - file = Executable('file') - file.add_default_env('LC_ALL', 'C') - output = file('-b', '-h', '%s' % path_name, - output=str, error=str) - return output.strip() - - def strings_contains_installroot(path_name, root_dir): """ Check if the file contain the install root string. diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index f5111e7f46..189bef4bd9 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -80,8 +80,6 @@ class Store(object): self.db = spack.database.Database(root) self.layout = spack.directory_layout.YamlDirectoryLayout( root, hash_len=hash_length, path_scheme=path_scheme) - self.extensions = spack.directory_layout.YamlExtensionsLayout( - root, self.layout) def reindex(self): """Convenience function to reindex the store DB with its own layout.""" @@ -105,4 +103,3 @@ store = llnl.util.lang.Singleton(_store) root = llnl.util.lang.LazyReference(lambda: store.root) db = llnl.util.lang.LazyReference(lambda: store.db) layout = llnl.util.lang.LazyReference(lambda: store.layout) -extensions = llnl.util.lang.LazyReference(lambda: store.extensions) diff --git a/lib/spack/spack/test/cmd/activate.py b/lib/spack/spack/test/cmd/activate.py new file mode 100644 index 0000000000..59a25ff51d --- /dev/null +++ b/lib/spack/spack/test/cmd/activate.py @@ -0,0 +1,61 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack.main import SpackCommand + +activate = SpackCommand('activate') +deactivate = SpackCommand('deactivate') +install = SpackCommand('install') +extensions = SpackCommand('extensions') + + +def test_activate( + mock_packages, mock_archive, mock_fetch, config, + install_mockery): + install('extension1') + activate('extension1') + output = extensions('--show', 'activated', 'extendee') + assert 'extension1' in output + + +def test_deactivate( + mock_packages, mock_archive, mock_fetch, config, + install_mockery): + install('extension1') + activate('extension1') + deactivate('extension1') + output = extensions('--show', 'activated', 'extendee') + assert 'extension1' not in output + + +def test_deactivate_all( + mock_packages, mock_archive, mock_fetch, config, + install_mockery): + install('extension1') + install('extension2') + activate('extension1') + activate('extension2') + deactivate('--all', 'extendee') + output = extensions('--show', 'activated', 'extendee') + assert 'extension1' not in output diff --git a/lib/spack/spack/test/test_activations.py b/lib/spack/spack/test/test_activations.py index da20a46bdc..04f5580cd9 100644 --- a/lib/spack/spack/test/test_activations.py +++ b/lib/spack/spack/test/test_activations.py @@ -24,16 +24,25 @@ ############################################################################## import os import pytest +import sys import spack.spec +import spack.package +from llnl.util.link_tree import MergeConflictError +from spack.build_systems.python import PythonPackage from spack.directory_layout import YamlDirectoryLayout from spack.filesystem_view import YamlFilesystemView +from spack.util.prefix import Prefix +"""This includes tests for customized activation logic for specific packages + (e.g. python and perl). +""" -class FakeExtensionPackage(object): + +class FakeExtensionPackage(spack.package.PackageViewMixin): def __init__(self, name, prefix): self.name = name - self.prefix = prefix + self.prefix = Prefix(prefix) self.spec = FakeSpec(self) @@ -42,10 +51,43 @@ class FakeSpec(object): self.name = package.name self.prefix = package.prefix self.hash = self.name + self.package = package + self.concrete = True def dag_hash(self): return self.hash + def __lt__(self, other): + return self.name < other.name + + +class FakePythonExtensionPackage(FakeExtensionPackage): + def __init__(self, name, prefix, py_namespace, python_spec): + self.py_namespace = py_namespace + self.extendee_spec = python_spec + super(FakePythonExtensionPackage, self).__init__(name, prefix) + + def add_files_to_view(self, view, merge_map): + if sys.version_info >= (3, 0): + add_fn = PythonPackage.add_files_to_view + else: + add_fn = PythonPackage.add_files_to_view.im_func + return add_fn(self, view, merge_map) + + def view_file_conflicts(self, view, merge_map): + if sys.version_info >= (3, 0): + conflicts_fn = PythonPackage.view_file_conflicts + else: + conflicts_fn = PythonPackage.view_file_conflicts.im_func + return conflicts_fn(self, view, merge_map) + + def remove_files_from_view(self, view, merge_map): + if sys.version_info >= (3, 0): + remove_fn = PythonPackage.remove_files_from_view + else: + remove_fn = PythonPackage.remove_files_from_view.im_func + return remove_fn(self, view, merge_map) + def create_dir_structure(tmpdir, dir_structure): for fname, children in dir_structure.items(): @@ -102,25 +144,49 @@ path/to/setuptools.egg""") return str(python_prefix), str(ext_prefix) -def test_python_activation(tmpdir): - # Note the lib directory is based partly on the python version - python_spec = spack.spec.Spec('python@2.7.12') - python_spec._concrete = True +@pytest.fixture() +def namespace_extensions(tmpdir): + ext1_dirs = { + 'bin/': { + 'py-ext-tool1': None + }, + 'lib/': { + 'python2.7/': { + 'site-packages/': { + 'examplenamespace/': { + '__init__.py': None, + 'ext1_sample.py': None + } + } + } + } + } - python_name = 'python' - tmpdir.ensure(python_name, dir=True) + ext2_dirs = { + 'bin/': { + 'py-ext-tool2': None + }, + 'lib/': { + 'python2.7/': { + 'site-packages/': { + 'examplenamespace/': { + '__init__.py': None, + 'ext2_sample.py': None + } + } + } + } + } - python_prefix = str(tmpdir.join(python_name)) - # Set the prefix on the package's spec reference because that is a copy of - # the original spec - python_spec.package.spec.prefix = python_prefix + ext1_name = 'py-extension1' + ext1_prefix = tmpdir.join(ext1_name) + create_dir_structure(ext1_prefix, ext1_dirs) - ext_name = 'py-extension' - tmpdir.ensure(ext_name, dir=True) - ext_pkg = FakeExtensionPackage(ext_name, str(tmpdir.join(ext_name))) + ext2_name = 'py-extension2' + ext2_prefix = tmpdir.join(ext2_name) + create_dir_structure(ext2_prefix, ext2_dirs) - python_pkg = python_spec.package - python_pkg.activate(ext_pkg) + return str(ext1_prefix), str(ext2_prefix), 'examplenamespace' def test_python_activation_with_files(tmpdir, python_and_extension_dirs): @@ -133,7 +199,7 @@ def test_python_activation_with_files(tmpdir, python_and_extension_dirs): ext_pkg = FakeExtensionPackage('py-extension', ext_prefix) python_pkg = python_spec.package - python_pkg.activate(ext_pkg) + python_pkg.activate(ext_pkg, python_pkg.view()) assert os.path.exists(os.path.join(python_prefix, 'bin/py-ext-tool')) @@ -159,13 +225,114 @@ def test_python_activation_view(tmpdir, python_and_extension_dirs): view = YamlFilesystemView(view_dir, layout) python_pkg = python_spec.package - python_pkg.activate(ext_pkg, extensions_layout=view.extensions_layout) + python_pkg.activate(ext_pkg, view) assert not os.path.exists(os.path.join(python_prefix, 'bin/py-ext-tool')) assert os.path.exists(os.path.join(view_dir, 'bin/py-ext-tool')) +def test_python_ignore_namespace_init_conflict(tmpdir, namespace_extensions): + """Test the view update logic in PythonPackage ignores conflicting + instances of __init__ for packages which are in the same namespace. + """ + ext1_prefix, ext2_prefix, py_namespace = namespace_extensions + + python_spec = spack.spec.Spec('python@2.7.12') + python_spec._concrete = True + + ext1_pkg = FakePythonExtensionPackage( + 'py-extension1', ext1_prefix, py_namespace, python_spec) + ext2_pkg = FakePythonExtensionPackage( + 'py-extension2', ext2_prefix, py_namespace, python_spec) + + view_dir = str(tmpdir.join('view')) + layout = YamlDirectoryLayout(view_dir) + view = YamlFilesystemView(view_dir, layout) + + python_pkg = python_spec.package + python_pkg.activate(ext1_pkg, view) + # Normally handled by Package.do_activate, but here we activate directly + view.extensions_layout.add_extension(python_spec, ext1_pkg.spec) + python_pkg.activate(ext2_pkg, view) + + f1 = 'lib/python2.7/site-packages/examplenamespace/ext1_sample.py' + f2 = 'lib/python2.7/site-packages/examplenamespace/ext2_sample.py' + init_file = 'lib/python2.7/site-packages/examplenamespace/__init__.py' + + assert os.path.exists(os.path.join(view_dir, f1)) + assert os.path.exists(os.path.join(view_dir, f2)) + assert os.path.exists(os.path.join(view_dir, init_file)) + + +def test_python_keep_namespace_init(tmpdir, namespace_extensions): + """Test the view update logic in PythonPackage keeps the namespace + __init__ file as long as one package in the namespace still + exists. + """ + ext1_prefix, ext2_prefix, py_namespace = namespace_extensions + + python_spec = spack.spec.Spec('python@2.7.12') + python_spec._concrete = True + + ext1_pkg = FakePythonExtensionPackage( + 'py-extension1', ext1_prefix, py_namespace, python_spec) + ext2_pkg = FakePythonExtensionPackage( + 'py-extension2', ext2_prefix, py_namespace, python_spec) + + view_dir = str(tmpdir.join('view')) + layout = YamlDirectoryLayout(view_dir) + view = YamlFilesystemView(view_dir, layout) + + python_pkg = python_spec.package + python_pkg.activate(ext1_pkg, view) + # Normally handled by Package.do_activate, but here we activate directly + view.extensions_layout.add_extension(python_spec, ext1_pkg.spec) + python_pkg.activate(ext2_pkg, view) + view.extensions_layout.add_extension(python_spec, ext2_pkg.spec) + + f1 = 'lib/python2.7/site-packages/examplenamespace/ext1_sample.py' + init_file = 'lib/python2.7/site-packages/examplenamespace/__init__.py' + + python_pkg.deactivate(ext1_pkg, view) + view.extensions_layout.remove_extension(python_spec, ext1_pkg.spec) + + assert not os.path.exists(os.path.join(view_dir, f1)) + assert os.path.exists(os.path.join(view_dir, init_file)) + + python_pkg.deactivate(ext2_pkg, view) + view.extensions_layout.remove_extension(python_spec, ext2_pkg.spec) + + assert not os.path.exists(os.path.join(view_dir, init_file)) + + +def test_python_namespace_conflict(tmpdir, namespace_extensions): + """Test the view update logic in PythonPackage reports an error when two + python extensions with different namespaces have a conflicting __init__ + file. + """ + ext1_prefix, ext2_prefix, py_namespace = namespace_extensions + other_namespace = py_namespace + 'other' + + python_spec = spack.spec.Spec('python@2.7.12') + python_spec._concrete = True + + ext1_pkg = FakePythonExtensionPackage( + 'py-extension1', ext1_prefix, py_namespace, python_spec) + ext2_pkg = FakePythonExtensionPackage( + 'py-extension2', ext2_prefix, other_namespace, python_spec) + + view_dir = str(tmpdir.join('view')) + layout = YamlDirectoryLayout(view_dir) + view = YamlFilesystemView(view_dir, layout) + + python_pkg = python_spec.package + python_pkg.activate(ext1_pkg, view) + view.extensions_layout.add_extension(python_spec, ext1_pkg.spec) + with pytest.raises(MergeConflictError): + python_pkg.activate(ext2_pkg, view) + + @pytest.fixture() def perl_and_extension_dirs(tmpdir): perl_dirs = { @@ -230,7 +397,7 @@ def test_perl_activation(tmpdir): ext_pkg = FakeExtensionPackage(ext_name, str(tmpdir.join(ext_name))) perl_pkg = perl_spec.package - perl_pkg.activate(ext_pkg) + perl_pkg.activate(ext_pkg, perl_pkg.view()) def test_perl_activation_with_files(tmpdir, perl_and_extension_dirs): @@ -243,7 +410,7 @@ def test_perl_activation_with_files(tmpdir, perl_and_extension_dirs): ext_pkg = FakeExtensionPackage('perl-extension', ext_prefix) perl_pkg = perl_spec.package - perl_pkg.activate(ext_pkg) + perl_pkg.activate(ext_pkg, perl_pkg.view()) assert os.path.exists(os.path.join(perl_prefix, 'bin/perl-ext-tool')) @@ -262,7 +429,7 @@ def test_perl_activation_view(tmpdir, perl_and_extension_dirs): view = YamlFilesystemView(view_dir, layout) perl_pkg = perl_spec.package - perl_pkg.activate(ext_pkg, extensions_layout=view.extensions_layout) + perl_pkg.activate(ext_pkg, view) assert not os.path.exists(os.path.join(perl_prefix, 'bin/perl-ext-tool')) diff --git a/lib/spack/spack/test/views.py b/lib/spack/spack/test/views.py new file mode 100644 index 0000000000..0981d05015 --- /dev/null +++ b/lib/spack/spack/test/views.py @@ -0,0 +1,48 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os + +from spack.spec import Spec + + +def test_global_activation(install_mockery, mock_fetch): + """This test ensures that views which are maintained inside of an extendee + package's prefix are maintained as expected and are compatible with + global activations prior to #7152. + """ + spec = Spec('extension1').concretized() + pkg = spec.package + pkg.do_install() + pkg.do_activate() + + extendee_spec = spec['extendee'] + extendee_pkg = spec['extendee'].package + view = extendee_pkg.view() + assert pkg.is_activated(view) + + expected_path = os.path.join( + extendee_spec.prefix, '.spack', 'extensions.yaml') + assert (view.extensions_layout.extension_file_path(extendee_spec) == + expected_path) |