diff options
author | scheibelp <scheibel1@llnl.gov> | 2018-06-26 16:14:05 -0700 |
---|---|---|
committer | Todd Gamblin <tgamblin@llnl.gov> | 2018-06-26 16:14:05 -0700 |
commit | 3560f6dbe9823c4b36783dfb8244d01672074760 (patch) | |
tree | ed5843c06d12c8c3fba8e26ae3e0fb04758c22a5 /var | |
parent | 1276ce0585187d56d3f4fe3da9ca0753d04441fa (diff) | |
download | spack-3560f6dbe9823c4b36783dfb8244d01672074760.tar.gz spack-3560f6dbe9823c4b36783dfb8244d01672074760.tar.bz2 spack-3560f6dbe9823c4b36783dfb8244d01672074760.tar.xz spack-3560f6dbe9823c4b36783dfb8244d01672074760.zip |
views: packages can customize how they're added to views (#7152)
Functional updates:
- `python` now creates a copy of the `python` binaries when it is added
to a view
- Python extensions (packages which subclass `PythonPackage`) rewrite
their shebang lines to refer to python in the view
- Python packages in the same namespace will not generate conflicts if
both have `...lib/site-packages/namespace-example/__init__.py`
- These `__init__` files will also remain when removing any package in
the namespace until the last package in the namespace is removed
Generally (Updated 2/16):
- Any package can define `add_files_to_view` to customize how it is added
to a view (and at the moment custom definitions are included for
`python` and `PythonPackage`)
- Likewise any package can define `remove_files_from_view` to customize
which files are removed (e.g. you don't always want to remove the
namespace `__init__`)
- Any package can define `view_file_conflicts` to customize what it
considers a merge conflict
- Global activations are handled like views (where the view root is the
spec prefix of the extendee)
- Benefit: filesystem-management aspects of activating extensions are
now placed in views (e.g. now one can hardlink a global activation)
- Benefit: overriding `Package.activate` is more straightforward (see
`Python.activate`)
- Complication: extension packages which have special-purpose logic
*only* when activated outside of the extendee prefix must check for
this in their `add_files_to_view` method (see `PythonPackage`)
- `LinkTree` is refactored to have separate methods for copying a
directory structure and for copying files (since it was found that
generally packages may want to alter how files are copied but still
wanted to copy directories in the same way)
TODOs (updated 2/20):
- [x] additional testing (there is some unit testing added at this point
but more would be useful)
- [x] refactor or reorganize `LinkTree` methods: currently there is a
separate set of methods for replicating just the directory structure
without the files, and a set for replicating everything
- [x] Right now external views (i.e. those not used for global
activations) call `view.add_extension`, but global activations do not
to avoid some extra work that goes into maintaining external views. I'm
not sure if addressing that needs to be done here but I'd like to
clarify it in the comments (UPDATE: for now I have added a TODO and in
my opinion this can be merged now and the refactor handled later)
- [x] Several method descriptions (e.g. for `Package.activate`) are out
of date and reference a distinction between global activations and
views, they need to be updated
- [x] Update aspell package activations
Diffstat (limited to 'var')
5 files changed, 44 insertions, 74 deletions
diff --git a/var/spack/repos/builtin/packages/aspell/package.py b/var/spack/repos/builtin/packages/aspell/package.py index 66eeb9e7cb..30cd8b9a23 100644 --- a/var/spack/repos/builtin/packages/aspell/package.py +++ b/var/spack/repos/builtin/packages/aspell/package.py @@ -23,9 +23,6 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -from llnl.util.link_tree import LinkTree -import spack.store -from spack.package import ExtensionError, ExtensionConflictError # See also: AspellDictPackage @@ -41,47 +38,3 @@ class Aspell(AutotoolsPackage): version('0.60.6.1', 'e66a9c9af6a60dc46134fdacf6ce97d7') patch('darwin.patch', when='platform=darwin') - - # The dictionaries install all their bits into their prefix.lib dir, - # we want to link them into aspell's dict-dir. - # These are identical to what's in spack/package.py except - # for using: - # - extension.prefix.lib instead of extension.prefix in LinkTree() - # - dest_dir instead of self.prefix in tree.(find_conflict|merge)() - def activate(self, extension, **kwargs): - extensions_layout = kwargs.get("extensions_layout", - spack.store.extensions) - if extensions_layout is not spack.store.extensions: - raise ExtensionError( - 'aspell does not support non-global extensions') - - aspell = which(self.prefix.bin.aspell) - dest_dir = aspell('dump', 'config', 'dict-dir', output=str).strip() - tree = LinkTree(extension.prefix.lib) - - def ignore(filename): - return (filename in spack.store.layout.hidden_file_paths or - kwargs.get('ignore', lambda f: False)(filename)) - - conflict = tree.find_conflict(dest_dir, ignore=ignore) - if conflict: - raise ExtensionConflictError(conflict) - - tree.merge(dest_dir, ignore=ignore) - - def deactivate(self, extension, **kwargs): - extensions_layout = kwargs.get("extensions_layout", - spack.store.extensions) - if extensions_layout is not spack.store.extensions: - raise ExtensionError( - 'aspell does not support non-global extensions') - - aspell = which(self.prefix.bin.aspell) - dest_dir = aspell('dump', 'config', 'dict-dir', output=str).strip() - - def ignore(filename): - return (filename in spack.store.layout.hidden_file_paths or - kwargs.get('ignore', lambda f: False)(filename)) - - tree = LinkTree(extension.prefix.lib) - tree.unmerge(dest_dir, ignore=ignore) diff --git a/var/spack/repos/builtin/packages/perl/package.py b/var/spack/repos/builtin/packages/perl/package.py index 2cacb5a03c..3e40c68349 100644 --- a/var/spack/repos/builtin/packages/perl/package.py +++ b/var/spack/repos/builtin/packages/perl/package.py @@ -35,7 +35,6 @@ from contextlib import contextmanager from llnl.util.lang import match_predicate -import spack.store from spack import * @@ -267,27 +266,23 @@ class Perl(Package): # Perl doesn't use Autotools, it should subclass Package return match_predicate(ignore_arg, patterns) - def activate(self, ext_pkg, **args): + def activate(self, ext_pkg, view, **args): ignore = self.perl_ignore(ext_pkg, args) args.update(ignore=ignore) - super(Perl, self).activate(ext_pkg, **args) - - extensions_layout = args.get("extensions_layout", - spack.store.extensions) + super(Perl, self).activate(ext_pkg, view, **args) + extensions_layout = view.extensions_layout exts = extensions_layout.extension_map(self.spec) exts[ext_pkg.name] = ext_pkg.spec - def deactivate(self, ext_pkg, **args): + def deactivate(self, ext_pkg, view, **args): ignore = self.perl_ignore(ext_pkg, args) args.update(ignore=ignore) - super(Perl, self).deactivate(ext_pkg, **args) - - extensions_layout = args.get("extensions_layout", - spack.store.extensions) + super(Perl, self).deactivate(ext_pkg, view, **args) + extensions_layout = view.extensions_layout exts = extensions_layout.extension_map(self.spec) # Make deactivate idempotent if ext_pkg.name in exts: diff --git a/var/spack/repos/builtin/packages/py-backports-shutil-get-terminal-size/package.py b/var/spack/repos/builtin/packages/py-backports-shutil-get-terminal-size/package.py index e30ef3770b..350eb793e6 100644 --- a/var/spack/repos/builtin/packages/py-backports-shutil-get-terminal-size/package.py +++ b/var/spack/repos/builtin/packages/py-backports-shutil-get-terminal-size/package.py @@ -32,6 +32,8 @@ class PyBackportsShutilGetTerminalSize(PythonPackage): homepage = "https://pypi.python.org/pypi/backports.shutil_get_terminal_size" url = "https://pypi.io/packages/source/b/backports.shutil_get_terminal_size/backports.shutil_get_terminal_size-1.0.0.tar.gz" + py_namespace = 'backports' + version('1.0.0', '03267762480bd86b50580dc19dff3c66') # newer setuptools version mess with "namespace" packages in an diff --git a/var/spack/repos/builtin/packages/py-backports-ssl-match-hostname/package.py b/var/spack/repos/builtin/packages/py-backports-ssl-match-hostname/package.py index 08ee1abbbb..df7c2dc84b 100644 --- a/var/spack/repos/builtin/packages/py-backports-ssl-match-hostname/package.py +++ b/var/spack/repos/builtin/packages/py-backports-ssl-match-hostname/package.py @@ -31,4 +31,6 @@ class PyBackportsSslMatchHostname(PythonPackage): homepage = "https://pypi.python.org/pypi/backports.ssl_match_hostname" url = "https://pypi.io/packages/source/b/backports.ssl_match_hostname/backports.ssl_match_hostname-3.5.0.1.tar.gz" + py_namespace = 'backports' + version('3.5.0.1', 'c03fc5e2c7b3da46b81acf5cbacfe1e6') diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 0c913422ef..7ce53903fb 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -27,10 +27,12 @@ import os import platform import re import sys +import shutil import llnl.util.tty as tty from llnl.util.lang import match_predicate -from llnl.util.filesystem import force_remove +from llnl.util.filesystem import (force_remove, get_filetype, + path_contains_subdirectory) import spack.store import spack.util.spack_json as sjson @@ -676,33 +678,49 @@ class Python(AutotoolsPackage): "sys.path[p:p]=new; " "sys.__egginsert = p+len(new)\n") - def activate(self, ext_pkg, **args): + def activate(self, ext_pkg, view, **args): ignore = self.python_ignore(ext_pkg, args) args.update(ignore=ignore) - extensions_layout = args.get("extensions_layout", - spack.store.extensions) - - super(Python, self).activate(ext_pkg, **args) + super(Python, self).activate(ext_pkg, view, **args) + extensions_layout = view.extensions_layout exts = extensions_layout.extension_map(self.spec) exts[ext_pkg.name] = ext_pkg.spec - self.write_easy_install_pth( - exts, - prefix=extensions_layout.extendee_target_directory(self)) + self.write_easy_install_pth(exts, prefix=view.root) - def deactivate(self, ext_pkg, **args): + def deactivate(self, ext_pkg, view, **args): args.update(ignore=self.python_ignore(ext_pkg, args)) - super(Python, self).deactivate(ext_pkg, **args) - extensions_layout = args.get("extensions_layout", - spack.store.extensions) + super(Python, self).deactivate(ext_pkg, view, **args) + extensions_layout = view.extensions_layout exts = extensions_layout.extension_map(self.spec) # Make deactivate idempotent if ext_pkg.name in exts: del exts[ext_pkg.name] - self.write_easy_install_pth( - exts, - prefix=extensions_layout.extendee_target_directory(self)) + self.write_easy_install_pth(exts, prefix=view.root) + + def add_files_to_view(self, view, merge_map): + bin_dir = self.spec.prefix.bin + for src, dst in merge_map.items(): + if not path_contains_subdirectory(src, bin_dir): + view.link(src, dst) + elif not os.path.islink(src): + shutil.copy2(src, dst) + if 'script' in get_filetype(src): + filter_file( + self.spec.prefix, os.path.abspath(view.root), dst) + else: + orig_link_target = os.path.realpath(src) + new_link_target = os.path.abspath(merge_map[orig_link_target]) + view.link(new_link_target, dst) + + def remove_files_from_view(self, view, merge_map): + bin_dir = self.spec.prefix.bin + for src, dst in merge_map.items(): + if not path_contains_subdirectory(src, bin_dir): + view.remove_file(src, dst) + else: + os.remove(dst) |