diff options
| -rw-r--r-- | lib/spack/docs/basic_usage.rst | 115 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/view.py | 295 |
2 files changed, 410 insertions, 0 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 2eed9dddd4..50c48b802b 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -342,6 +342,7 @@ will find every installed package with a 'debug' compile-time option enabled. The full spec syntax is discussed in detail in :ref:`sec-specs`. + Compiler configuration ----------------------------------- @@ -1320,6 +1321,120 @@ regenerate all module and dotkit files from scratch: .. _extensions: +Filesystem Views +------------------------------- + +.. Maybe this is not the right location for this documentation. + +The Spack installation area allows for many package installation trees +to coexist and gives the user choices as to what versions and variants +of packages to use. To use them, the user must rely on a way to +aggregate a subset of those packages. The section on Environment +Modules gives one good way to do that which relies on setting various +environment variables. An alternative way to aggregate is through +**filesystem views**. + +A filesystem view is a single directory tree which is the union of the +directory hierarchies of the individual package installation trees +that have been included. The files of the view's installed packages +are brought into the view by symbolic or hard links back to their +location in the original Spack installation area. As the view is +formed, any clashes due to a file having the exact same path in its +package installation tree are handled in a first-come-first-served +basis and a warning is printed. Packages and their dependencies can +be both added and removed. During removal, empty directories will be +purged. These operations can be limited to pertain to just the +packages listed by the user or to exclude specific dependencies and +they allow for software installed outside of Spack to coexist inside +the filesystem view tree. + +By its nature, a filesystem view represents a particular choice of one +set of packages among all the versions and variants that are available +in the Spack installation area. It is thus equivalent to the +directory hiearchy that might exist under ``/usr/local``. While this +limits a view to including only one version/variant of any package, it +provides the benefits of having a simpler and traditional layout which +may be used without any particular knowledge that its packages were +built by Spack. + +Views can be used for a variety of purposes including: + +- A central installation in a traditional layout, eg ``/usr/local`` maintained over time by the sysadmin. +- A self-contained installation area which may for the basis of a top-level atomic versioning scheme, eg ``/opt/pro`` vs ``/opt/dev``. +- Providing an atomic and monolithic binary distribution, eg for delivery as a single tarball. +- Producing ephemeral testing or developing environments. + +Using Filesystem Views +~~~~~~~~~~~~~~~~~~~~~~ + +A filesystem view is created and packages are linked in by the ``spack +view`` command's ``symlink`` and ``hardlink`` sub-commands. The +``spack view remove`` command can be used to unlink some or all of the +filesystem view. + +The following example creates a filesystem view based +on an installed ``cmake`` package and then removes from the view the +files in the ``cmake`` package while retaining its dependencies. + +.. code-block:: sh + + + $ spack view -v symlink myview cmake@3.5.2 + ==> Linking package: "ncurses" + ==> Linking package: "zlib" + ==> Linking package: "openssl" + ==> Linking package: "cmake" + + $ ls myview/ + bin doc etc include lib share + + $ ls myview/bin/ + captoinfo clear cpack ctest infotocap openssl tabs toe tset + ccmake cmake c_rehash infocmp ncurses6-config reset tic tput + + $ spack view -v -d false rm myview cmake@3.5.2 + ==> Removing package: "cmake" + + $ ls myview/bin/ + captoinfo c_rehash infotocap openssl tabs toe tset + clear infocmp ncurses6-config reset tic tput + + +Limitations of Filesystem Views +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This section describes some limitations that should be considered in +using filesystems views. + +Filesystem views are merely organizational. The binary executable +programs, shared libraries and other build products found in a view +are mere links into the "real" Spack installation area. If a view is +built with symbolic links it requires the Spack-installed package to +be kept in place. Building a view with hardlinks removes this +requirement but any internal paths (eg, rpath or ``#!`` interpreter +specifications) will still require the Spack-installed package files +to be in place. + +.. FIXME: reference the relocation work of Hegner and Gartung. + +As described above, when a view is built only a single instance of a +file may exist in the unified filesystem tree. If more than one +package provides a file at the same path (relative to its own root) +then it is the first package added to the view that "wins". A warning +is printed and it is up to the user to determine if the conflict +matters. + +It is up to the user to assure a consistent view is produced. In +particular if the user excludes packages, limits the following of +dependencies or removes packages the view may become inconsistent. In +particular, if two packages require the same sub-tree of dependencies, +removing one package (recursively) will remove its dependencies and +leave the other package broken. + + + + + Extensions & Python support ------------------------------------ diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py new file mode 100644 index 0000000000..8f1fc9be74 --- /dev/null +++ b/lib/spack/spack/cmd/view.py @@ -0,0 +1,295 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +'''Produce a "view" of a Spack DAG. + +A "view" is file hierarchy representing the union of a number of +Spack-installed package file hierarchies. The union is formed from: + +- specs resolved from the package names given by the user (the seeds) + +- all depenencies of the seeds unless user specifies `--no-depenencies` + +- less any specs with names matching the regular expressions given by + `--exclude` + +The `view` can be built and tore down via a number of methods (the "actions"): + +- symlink :: a file system view which is a directory hierarchy that is + the union of the hierarchies of the installed packages in the DAG + where installed files are referenced via symlinks. + +- hardlink :: like the symlink view but hardlinks are used. + +- statlink :: a view producing a status report of a symlink or + hardlink view. + +The file system view concept is imspired by Nix, implemented by +brett.viren@gmail.com ca 2016. + +''' +# Implementation notes: +# +# This is implemented as a visitor pattern on the set of package specs. +# +# The command line ACTION maps to a visitor_*() function which takes +# the set of package specs and any args which may be specific to the +# ACTION. +# +# To add a new view: +# 1. add a new cmd line args sub parser ACTION +# 2. add any action-specific options/arguments, most likely a list of specs. +# 3. add a visitor_MYACTION() function +# 4. add any visitor_MYALIAS assignments to match any command line aliases + +import os +import re +import spack +import spack.cmd +import llnl.util.tty as tty + +description = "Produce a single-rooted directory view of a spec." + + +def setup_parser(sp): + setup_parser.parser = sp + + sp.add_argument( + '-v', '--verbose', action='store_true', default=False, + help="Display verbose output.") + sp.add_argument( + '-e', '--exclude', action='append', default=[], + help="Exclude packages with names matching the given regex pattern.") + sp.add_argument( + '-d', '--dependencies', choices=['true', 'false', 'yes', 'no'], + default='true', + help="Follow dependencies.") + + ssp = sp.add_subparsers(metavar='ACTION', dest='action') + + specs_opts = dict(metavar='spec', nargs='+', + help="Seed specs of the packages to view.") + + # The action parameterizes the command but in keeping with Spack + # patterns we make it a subcommand. + file_system_view_actions = [ + ssp.add_parser( + 'symlink', aliases=['add', 'soft'], + help='Add package files to a filesystem view via symbolic links.'), + ssp.add_parser( + 'hardlink', aliases=['hard'], + help='Add packages files to a filesystem via via hard links.'), + ssp.add_parser( + 'remove', aliases=['rm'], + help='Remove packages from a filesystem view.'), + ssp.add_parser( + 'statlink', aliases=['status', 'check'], + help='Check status of packages in a filesystem view.') + ] + # All these options and arguments are common to every action. + for act in file_system_view_actions: + act.add_argument('path', nargs=1, + help="Path to file system view directory.") + act.add_argument('specs', **specs_opts) + + return + + +def assuredir(path): + 'Assure path exists as a directory' + if not os.path.exists(path): + os.makedirs(path) + + +def relative_to(prefix, path): + 'Return end of `path` relative to `prefix`' + assert 0 == path.find(prefix) + reldir = path[len(prefix):] + if reldir.startswith('/'): + reldir = reldir[1:] + return reldir + + +def transform_path(spec, path, prefix=None): + 'Return the a relative path corresponding to given path spec.prefix' + if os.path.isabs(path): + path = relative_to(spec.prefix, path) + subdirs = path.split(os.path.sep) + if subdirs[0] == '.spack': + lst = ['.spack', spec.name] + subdirs[1:] + path = os.path.join(*lst) + if prefix: + path = os.path.join(prefix, path) + return path + + +def purge_empty_directories(path): + '''Ascend up from the leaves accessible from `path` + and remove empty directories.''' + for dirpath, subdirs, files in os.walk(path, topdown=False): + for sd in subdirs: + sdp = os.path.join(dirpath, sd) + try: + os.rmdir(sdp) + except OSError: + pass + + +def filter_exclude(specs, exclude): + 'Filter specs given sequence of exclude regex' + to_exclude = [re.compile(e) for e in exclude] + + def exclude(spec): + for e in to_exclude: + if e.match(spec.name): + return True + return False + return [s for s in specs if not exclude(s)] + + +def flatten(seeds, descend=True): + 'Normalize and flattend seed specs and descend hiearchy' + flat = set() + for spec in seeds: + if not descend: + flat.add(spec) + continue + flat.update(spec.normalized().traverse()) + return flat + + +def check_one(spec, path, verbose=False): + 'Check status of view in path against spec' + dotspack = os.path.join(path, '.spack', spec.name) + if os.path.exists(os.path.join(dotspack)): + tty.info('Package in view: "%s"' % spec.name) + return + tty.info('Package not in view: "%s"' % spec.name) + return + + +def remove_one(spec, path, verbose=False): + 'Remove any files found in `spec` from `path` and purge empty directories.' + + if not os.path.exists(path): + return # done, short circuit + + dotspack = transform_path(spec, '.spack', path) + if not os.path.exists(dotspack): + if verbose: + tty.info('Skipping nonexistent package: "%s"' % spec.name) + return + + if verbose: + tty.info('Removing package: "%s"' % spec.name) + for dirpath, dirnames, filenames in os.walk(spec.prefix): + if not filenames: + continue + targdir = transform_path(spec, dirpath, path) + for fname in filenames: + dst = os.path.join(targdir, fname) + if not os.path.exists(dst): + continue + os.unlink(dst) + + +def link_one(spec, path, link=os.symlink, verbose=False): + 'Link all files in `spec` into directory `path`.' + + dotspack = transform_path(spec, '.spack', path) + if os.path.exists(dotspack): + tty.warn('Skipping existing package: "%s"' % spec.name) + return + + if verbose: + tty.info('Linking package: "%s"' % spec.name) + for dirpath, dirnames, filenames in os.walk(spec.prefix): + if not filenames: + continue # avoid explicitly making empty dirs + + targdir = transform_path(spec, dirpath, path) + assuredir(targdir) + + for fname in filenames: + src = os.path.join(dirpath, fname) + dst = os.path.join(targdir, fname) + if os.path.exists(dst): + if '.spack' in dst.split(os.path.sep): + continue # silence these + tty.warn("Skipping existing file: %s" % dst) + continue + link(src, dst) + + +def visitor_symlink(specs, args): + 'Symlink all files found in specs' + path = args.path[0] + assuredir(path) + for spec in specs: + link_one(spec, path, verbose=args.verbose) +visitor_add = visitor_symlink +visitor_soft = visitor_symlink + + +def visitor_hardlink(specs, args): + 'Hardlink all files found in specs' + path = args.path[0] + assuredir(path) + for spec in specs: + link_one(spec, path, os.link, verbose=args.verbose) +visitor_hard = visitor_hardlink + + +def visitor_remove(specs, args): + 'Remove all files and directories found in specs from args.path' + path = args.path[0] + for spec in specs: + remove_one(spec, path, verbose=args.verbose) + purge_empty_directories(path) +visitor_rm = visitor_remove + + +def visitor_statlink(specs, args): + 'Give status of view in args.path relative to specs' + path = args.path[0] + for spec in specs: + check_one(spec, path, verbose=args.verbose) +visitor_status = visitor_statlink +visitor_check = visitor_statlink + + +def view(parser, args): + 'Produce a view of a set of packages.' + + # Process common args + seeds = [spack.cmd.disambiguate_spec(s) for s in args.specs] + specs = flatten(seeds, args.dependencies.lower() in ['yes', 'true']) + specs = filter_exclude(specs, args.exclude) + + # Execute the visitation. + try: + visitor = globals()['visitor_' + args.action] + except KeyError: + tty.error('Unknown action: "%s"' % args.action) + visitor(specs, args) |