diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/docs/basic_usage.rst | 233 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/view.py | 361 |
2 files changed, 594 insertions, 0 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 15db2f7a16..4af4387846 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -330,6 +330,125 @@ of libelf would look like this: The full spec syntax is discussed in detail in :ref:`sec-specs`. +``spack view print`` +~~~~~~~~~~~~~~~~~~~~~~ + +``spack view print`` is a subcommand of ``spack view`` which displays information about one or more installed packages and their dependencies using a user-provided format string. The string can reverence variables in a shell-like manner for example ``$variable`` or ``${variable}text``. It can also include ``\t`` for tabs and ``\n`` for new lines. + +Some of the supported variables are: + +``name`` + The package name. + +``version`` + The package version. + +``spec`` + The package specification. + +``root`` + The root specification. + +``prefix`` + The installation directory. + +``variants`` + The collection of variants, if any. + +``namespace`` + The package repository name space. + +``compiler`` + The compiler \`name@versoin\` used to build the package. + +``architecture`` + The architecture targeted by the compiler. + +``dependencies`` + A comma-separated list of names of packages on which the package depends. + +``dependents`` + A comma-separated list of names of packages which depend on the package. + +``hash`` + The Spack hash for the package. + +``url`` + The source URL for the package. + +``stage`` + The directory for staging the build. + +``build_log`` + The path to the build log file. + +``rpath`` + The colon-separated library \`RPATH\` used in building the package. + +Here are some example uses of `spack view print`. A simple line-oriented report of information can be produced: + +.. code-block:: sh + + $ spack view print '$name\t$version\t$hash\n' cmake@3.5.2 + ncurses 6.0 bvbu4ixbnvtodpik4qzljlx3ukpyfrcz + zlib 1.2.8 ckki7zlryxrsetfqkgoxxahlhqqjni7n + openssl 1.0.2g 6zbar63sciso253nptxyrnhupymo7oyi + cmake 3.5.2 wprvmoczkpw4tiy5ybuk5zr7saus2d7g + +There are better ways to do this but a slow-and-dirty shell init procedure can be formed: + +.. code-block:: sh + + $ spack view print 'export PATH="${prefix}/bin:$$PATH"\n' m4 + export PATH="/spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2/bin:$PATH" + export PATH="/spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp/bin:$PATH" + $ eval $(spack view print 'export PATH="${prefix}/bin:$$PATH"\n' m4) + +Or, maybe you have some external application that can chew on Spack data in which case you can dump it to some easily parsed markup syntax such as YAML: + +.. code-block:: sh + + + $ spack view print '${name}:\n - version: ${version}\n - url : ${url}\n - spec: ${spec}\n - prefix: ${prefix}\n - root : ${root}\n - stage : ${stage}\n - log: ${build_log}\n' m4 + +Which might produce something like: + +.. code-block:: yaml + + libsigsegv: + - version: 2.10 + - url : ftp://ftp.gnu.org/gnu/libsigsegv/libsigsegv-2.10.tar.gz + - spec: libsigsegv@2.10%gcc@5.2.1=linux-x86_64-h6hsv76 + - prefix: /spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2 + - root : m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.2.1=linux-x86_64 + - stage : /spack/var/spack/stage/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2 + - log: /spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2/.spack/build.out + m4: + - version: 1.4.17 + - url : ftp://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz + - spec: m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64-a4ikhdd + - prefix: /spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp + - root : m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.2.1=linux-x86_64 + - stage : /spack/var/spack/stage/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp + - log: /spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp/.spack/build.out + +Or, maybe you want to do something with information about package dependencies by generating some ready-to-import Python code holding the tree: + +.. code-block:: sh + + $ spack view print '$name = dict(name = "$name", parents = [$dependencies], children = [$dependents])\n' cmake@3.5.2 + +Producing this Python code + +.. code-block:: python + + ncurses = dict(name = "ncurses", parents = []]) + zlib = dict(name = "zlib", parents = []]) + openssl = dict(name = "openssl", parents = [zlib]]) + cmake = dict(name = "cmake", parents = [ncurses,openssl]]) + + + Compiler configuration ----------------------------------- @@ -1245,6 +1364,120 @@ regenerate all module and dotkit files from scratch: .. _extensions: +Filesystem Views +------------------------------- + +.. Maybe this is not the right location for this documentation. + +The Spack installation area allows for many package installation trees +to coexist and gives the user choices as to what versions and variants +of packages to use. To use them, the user must rely on a way to +aggregate a subset of those packages. The section on Environment +Modules gives one good way to do that which relies on setting various +environment variables. An alternative way to aggregate is through +**filesystem views**. + +A filesystem view is a single directory tree which is the union of the +directory hierarchies of the individual package installation trees +that have been included. The files of the view's installed packages +are brought into the view by symbolic or hard links back to their +location in the original Spack installation area. As the view is +formed, any clashes due to a file having the exact same path in its +package installation tree are handled in a first-come-first-served +basis and a warning is printed. Packages and their dependencies can +be both added and removed. During removal, empty directories will be +purged. These operations can be limited to pertain to just the +packages listed by the user or to exclude specific dependencies and +they allow for software installed outside of Spack to coexist inside +the filesystem view tree. + +By its nature, a filesystem view represents a particular choice of one +set of packages among all the versions and variants that are available +in the Spack installation area. It is thus equivalent to the +directory hiearchy that might exist under ``/usr/local``. While this +limits a view to including only one version/variant of any package, it +provides the benefits of having a simpler and traditional layout which +may be used without any particular knowledge that its packages were +built by Spack. + +Views can be used for a variety of purposes including: + +- A central installation in a traditional layout, eg ``/usr/local`` maintained over time by the sysadmin. +- A self-contained installation area which may for the basis of a top-level atomic versioning scheme, eg ``/opt/pro`` vs ``/opt/dev``. +- Providing an atomic and monolithic binary distribution, eg for delivery as a single tarball. +- Producing ephemeral testing or developing environments. + +Using Filesystem Views +~~~~~~~~~~~~~~~~~~~~~~ + +A filesystem view is created and packages are linked in by the ``spack +view`` command's ``symlink`` and ``hardlink`` sub-commands. The +``spack view remove`` command can be used to unlink some or all of the +filesystem view. + +The following example creates a filesystem view based +on an installed ``cmake`` package and then removes from the view the +files in the ``cmake`` package while retaining its dependencies. + +.. code-block:: sh + + + $ spack view -v symlink myview cmake@3.5.2 + ==> Linking package: "ncurses" + ==> Linking package: "zlib" + ==> Linking package: "openssl" + ==> Linking package: "cmake" + + $ ls myview/ + bin doc etc include lib share + + $ ls myview/bin/ + captoinfo clear cpack ctest infotocap openssl tabs toe tset + ccmake cmake c_rehash infocmp ncurses6-config reset tic tput + + $ spack view -v -d false rm myview cmake@3.5.2 + ==> Removing package: "cmake" + + $ ls myview/bin/ + captoinfo c_rehash infotocap openssl tabs toe tset + clear infocmp ncurses6-config reset tic tput + + +Limitations of Filesystem Views +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This section describes some limitations that should be considered in +using filesystems views. + +Filesystem views are merely organizational. The binary executable +programs, shared libraries and other build products found in a view +are mere links into the "real" Spack installation area. If a view is +built with symbolic links it requires the Spack-installed package to +be kept in place. Building a view with hardlinks removes this +requirement but any internal paths (eg, rpath or ``#!`` interpreter +specifications) will still require the Spack-installed package files +to be in place. + +.. FIXME: reference the relocation work of Hegner and Gartung. + +As described above, when a view is built only a single instance of a +file may exist in the unified filesystem tree. If more than one +package provides a file at the same path (relative to its own root) +then it is the first package added to the view that "wins". A warning +is printed and it is up to the user to determine if the conflict +matters. + +It is up to the user to assure a consistent view is produced. In +particular if the user excludes packages, limits the following of +dependencies or removes packages the view may become inconsistent. In +particular, if two packages require the same sub-tree of dependencies, +removing one package (recursively) will remove its dependencies and +leave the other package broken. + + + + + Extensions & Python support ------------------------------------ diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py new file mode 100644 index 0000000000..4861383eaa --- /dev/null +++ b/lib/spack/spack/cmd/view.py @@ -0,0 +1,361 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +'''Produce a "view" of a Spack DAG. + +A "view" is the product of applying a function on a set of package specs. + +This set consists of: + +- specs resolved from the package names given by the user (the seeds) + +- all depenencies of the seeds unless user specifies `--no-depenencies` + +- less any specs with names matching the regular expressions given by + `--exclude` + +The `view` command provides a number of functions (the "actions"): + +- symlink :: a file system view which is a directory hierarchy that is + the union of the hierarchies of the installed packages in the DAG + where installed files are referenced via symlinks. + +- hardlink :: like the symlink view but hardlinks are used. + +- statlink :: a view producing a status report of a symlink or + hardlink view. + +- format :: a view printing one string per spec following a given format. + +The file system view concept is imspired by Nix, implemented by +brett.viren@gmail.com ca 2016. + +''' +# Implementation notes: +# +# This is implemented as a visitor pattern on the set of package specs. +# +# The command line ACTION maps to a visitor_*() function which takes +# the set of package specs and any args which may be specific to the +# ACTION. +# +# To add a new view: +# 1. add a new cmd line args sub parser ACTION +# 2. add any action-specific options/arguments, most likely a list of specs. +# 3. add a visitor_MYACTION() function +# 4. add any visitor_MYALIAS assignments to match any command line aliases + +import os +import re +import sys +import spack +import spack.cmd +import llnl.util.tty as tty + +description = "Produce a single-rooted directory view of a spec." + + +def setup_parser(sp): + setup_parser.parser = sp + + sp.add_argument( + '-v', '--verbose', action='store_true', default=False, + help="Display verbose output.") + sp.add_argument( + '-e', '--exclude', action='append', default=[], + help="Exclude packages with names matching the given regex pattern.") + sp.add_argument( + '-d', '--dependencies', choices=['true', 'false', 'yes', 'no'], + default='true', + help="Follow dependencies.") + + ssp = sp.add_subparsers(metavar='ACTION', dest='action') + + specs_opts = dict(metavar='spec', nargs='+', + help="Seed specs of the packages to view.") + + # The action parameterizes the command but in keeping with Spack + # patterns we make it a subcommand. + file_system_view_actions = [ + ssp.add_parser( + 'symlink', aliases=['add', 'soft'], + help='Add package files to a filesystem view via symbolic links.'), + ssp.add_parser( + 'hardlink', aliases=['hard'], + help='Add packages files to a filesystem via via hard links.'), + ssp.add_parser( + 'remove', aliases=['rm'], + help='Remove packages from a filesystem view.'), + ssp.add_parser( + 'statlink', aliases=['status', 'check'], + help='Check status of packages in a filesystem view.') + ] + # All these options and arguments are common to every action. + for act in file_system_view_actions: + act.add_argument('path', nargs=1, + help="Path to file system view directory.") + act.add_argument('specs', **specs_opts) + + # The formatted print action. + act = ssp.add_parser('print', + help="Print a string to stdout based on given format") + act.add_argument('format', nargs=1, + help="Format describing per-package printout.") + act.add_argument('specs', **specs_opts) + + return + + +def assuredir(path): + 'Assure path exists as a directory' + if not os.path.exists(path): + os.makedirs(path) + + +def relative_to(prefix, path): + 'Return end of `path` relative to `prefix`' + assert 0 == path.find(prefix) + reldir = path[len(prefix):] + if reldir.startswith('/'): + reldir = reldir[1:] + return reldir + + +def transform_path(spec, path, prefix=None): + 'Return the a relative path corresponding to given path spec.prefix' + if os.path.isabs(path): + path = relative_to(spec.prefix, path) + subdirs = path.split(os.path.sep) + if subdirs[0] == '.spack': + lst = ['.spack', spec.name] + subdirs[1:] + path = os.path.join(*lst) + if prefix: + path = os.path.join(prefix, path) + return path + + +def purge_empty_directories(path): + '''Ascend up from the leaves accessible from `path` + and remove empty directories.''' + for dirpath, subdirs, files in os.walk(path, topdown=False): + for sd in subdirs: + sdp = os.path.join(dirpath, sd) + try: + os.rmdir(sdp) + except OSError: + pass + + +def filter_exclude(specs, exclude): + 'Filter specs given sequence of exclude regex' + to_exclude = [re.compile(e) for e in exclude] + + def exclude(spec): + for e in to_exclude: + if e.match(spec.name): + return True + return False + return [s for s in specs if not exclude(s)] + + +def flatten(seeds, descend=True): + 'Normalize and flattend seed specs and descend hiearchy' + flat = set() + for spec in seeds: + if not descend: + flat.add(spec) + continue + flat.update(spec.normalized().traverse()) + return flat + + +def spec2dict(spec): + 'Convert info in a spec into a simple dictionary.' + + # Expclitly convert instead of just returning spec.__dict__ as + # some things need processing or are properties. + # + pkg = spec.package + inst_deps = ','.join([s.name for s in pkg.installed_dependents]), + ret = dict(name=spec.name, + spec=spec.short_spec, + colorspec=spec.cshort_spec, # color + root=spec.root, + prefix=spec.prefix, + version=spec.version, + variants=spec.variants, + namespace=spec.namespace, + compiler=spec.compiler, + architecture=spec.architecture, + dependencies=','.join(spec.dependencies.keys()), + dependents=','.join(spec.dependents.keys()), + external=spec.external or "False", + hash=spec.dag_hash(), + + # package related: + url=pkg.url, + stage=pkg.stage.path, + installed=pkg.installed, + installed_dependents=inst_deps, + build_log=pkg.build_log_path, + rpath=':'.join(pkg.rpath), + + # ... + ) + return ret + + +def check_one(spec, path, verbose=False): + 'Check status of view in path against spec' + dotspack = os.path.join(path, '.spack', spec.name) + if os.path.exists(os.path.join(dotspack)): + tty.info('Package in view: "%s"' % spec.name) + return + tty.info('Package not in view: "%s"' % spec.name) + return + + +def remove_one(spec, path, verbose=False): + 'Remove any files found in `spec` from `path` and purge empty directories.' + + if not os.path.exists(path): + return # done, short circuit + + dotspack = transform_path(spec, '.spack', path) + if not os.path.exists(dotspack): + if verbose: + tty.info('Skipping nonexistent package: "%s"' % spec.name) + return + + if verbose: + tty.info('Removing package: "%s"' % spec.name) + for dirpath, dirnames, filenames in os.walk(spec.prefix): + if not filenames: + continue + targdir = transform_path(spec, dirpath, path) + for fname in filenames: + dst = os.path.join(targdir, fname) + if not os.path.exists(dst): + continue + os.unlink(dst) + + +def link_one(spec, path, link=os.symlink, verbose=False): + 'Link all files in `spec` into directory `path`.' + + dotspack = transform_path(spec, '.spack', path) + if os.path.exists(dotspack): + tty.warn('Skipping existing package: "%s"' % spec.name) + return + + if verbose: + tty.info('Linking package: "%s"' % spec.name) + for dirpath, dirnames, filenames in os.walk(spec.prefix): + if not filenames: + continue # avoid explicitly making empty dirs + + targdir = transform_path(spec, dirpath, path) + assuredir(targdir) + + for fname in filenames: + src = os.path.join(dirpath, fname) + dst = os.path.join(targdir, fname) + if os.path.exists(dst): + if '.spack' in dst.split(os.path.sep): + continue # silence these + tty.warn("Skipping existing file: %s" % dst) + continue + link(src, dst) + + +def visitor_symlink(specs, args): + 'Symlink all files found in specs' + path = args.path[0] + assuredir(path) + for spec in specs: + link_one(spec, path, verbose=args.verbose) +visitor_add = visitor_symlink +visitor_soft = visitor_symlink + + +def visitor_hardlink(specs, args): + 'Hardlink all files found in specs' + path = args.path[0] + assuredir(path) + for spec in specs: + link_one(spec, path, os.link, verbose=args.verbose) +visitor_hard = visitor_hardlink + + +def visitor_remove(specs, args): + 'Remove all files and directories found in specs from args.path' + path = args.path[0] + for spec in specs: + remove_one(spec, path, verbose=args.verbose) + purge_empty_directories(path) +visitor_rm = visitor_remove + + +def visitor_statlink(specs, args): + 'Give status of view in args.path relative to specs' + path = args.path[0] + for spec in specs: + check_one(spec, path, verbose=args.verbose) +visitor_status = visitor_statlink +visitor_check = visitor_statlink + + +def visitor_print(specs, args): + 'Print a string for each spec using args.format.' + fmt = args.format[0] + from string import Template + t = Template(fmt) + + for spec in specs: + kwds = spec2dict(spec) + try: + text = t.substitute(kwds) + except KeyError: + tty.error("Format error, use keywords: %s" % + (', '.join(kwds.keys()), )) + raise + # argparser escapes these + text = text.replace(r'\n', '\n').replace(r'\t', '\t') + sys.stdout.write(text) + + +def view(parser, args): + 'Produce a view of a set of packages.' + + # Process common args + seeds = [spack.cmd.disambiguate_spec(s) for s in args.specs] + specs = flatten(seeds, args.dependencies.lower() in ['yes', 'true']) + specs = filter_exclude(specs, args.exclude) + + # Execute the visitation. + try: + visitor = globals()['visitor_' + args.action] + except KeyError: + tty.error('Unknown action: "%s"' % args.action) + visitor(specs, args) |