summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/docs/basic_usage.rst83
-rw-r--r--lib/spack/docs/developer_guide.rst9
-rw-r--r--lib/spack/docs/mirrors.rst18
-rw-r--r--lib/spack/docs/packaging_guide.rst20
-rw-r--r--lib/spack/spack/cmd/load.py2
-rw-r--r--lib/spack/spack/cmd/module.py113
6 files changed, 201 insertions, 44 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst
index ec193e767d..d8d5a6a23a 100644
--- a/lib/spack/docs/basic_usage.rst
+++ b/lib/spack/docs/basic_usage.rst
@@ -1131,6 +1131,79 @@ of module files:
"""Set up the compile and runtime environments for a package."""
pass
+
+Recursive Modules
+``````````````````
+
+In some cases, it is desirable to load not just a module, but also all
+the modules it depends on. This is not required for most modules
+because Spack builds binaries with RPATH support. However, not all
+packages use RPATH to find their dependencies: this can be true in
+particular for Python extensions, which are currently *not* built with
+RPATH.
+
+Modules may be loaded recursively with the command:
+
+.. code-block:: sh
+
+ $ module load `spack module tcl --dependencies <spec>...
+
+More than one spec may be placed on the command line here.
+
+Module Comamnds for Shell Scripts
+``````````````````````````````````
+
+Although Spack is flexbile, the ``module`` command is much faster.
+This could become an issue when emitting a series of ``spack load``
+commands inside a shell script. By adding the ``--shell`` flag,
+``spack module find`` may also be used to generate code that can be
+cut-and-pasted into a shell script. For example:
+
+.. code-block:: sh
+
+ $ spack module find tcl --dependencies --shell py-numpy git
+ # bzip2@1.0.6%gcc@4.9.3=linux-x86_64
+ module load bzip2-1.0.6-gcc-4.9.3-ktnrhkrmbbtlvnagfatrarzjojmkvzsx
+ # ncurses@6.0%gcc@4.9.3=linux-x86_64
+ module load ncurses-6.0-gcc-4.9.3-kaazyneh3bjkfnalunchyqtygoe2mncv
+ # zlib@1.2.8%gcc@4.9.3=linux-x86_64
+ module load zlib-1.2.8-gcc-4.9.3-v3ufwaahjnviyvgjcelo36nywx2ufj7z
+ # sqlite@3.8.5%gcc@4.9.3=linux-x86_64
+ module load sqlite-3.8.5-gcc-4.9.3-a3eediswgd5f3rmto7g3szoew5nhehbr
+ # readline@6.3%gcc@4.9.3=linux-x86_64
+ module load readline-6.3-gcc-4.9.3-se6r3lsycrwxyhreg4lqirp6xixxejh3
+ # python@3.5.1%gcc@4.9.3=linux-x86_64
+ module load python-3.5.1-gcc-4.9.3-5q5rsrtjld4u6jiicuvtnx52m7tfhegi
+ # py-setuptools@20.5%gcc@4.9.3=linux-x86_64
+ module load py-setuptools-20.5-gcc-4.9.3-4qr2suj6p6glepnedmwhl4f62x64wxw2
+ # py-nose@1.3.7%gcc@4.9.3=linux-x86_64
+ module load py-nose-1.3.7-gcc-4.9.3-pwhtjw2dvdvfzjwuuztkzr7b4l6zepli
+ # openblas@0.2.17%gcc@4.9.3+shared=linux-x86_64
+ module load openblas-0.2.17-gcc-4.9.3-pw6rmlom7apfsnjtzfttyayzc7nx5e7y
+ # py-numpy@1.11.0%gcc@4.9.3+blas+lapack=linux-x86_64
+ module load py-numpy-1.11.0-gcc-4.9.3-mulodttw5pcyjufva4htsktwty4qd52r
+ # curl@7.47.1%gcc@4.9.3=linux-x86_64
+ module load curl-7.47.1-gcc-4.9.3-ohz3fwsepm3b462p5lnaquv7op7naqbi
+ # autoconf@2.69%gcc@4.9.3=linux-x86_64
+ module load autoconf-2.69-gcc-4.9.3-bkibjqhgqm5e3o423ogfv2y3o6h2uoq4
+ # cmake@3.5.0%gcc@4.9.3~doc+ncurses+openssl~qt=linux-x86_64
+ module load cmake-3.5.0-gcc-4.9.3-x7xnsklmgwla3ubfgzppamtbqk5rwn7t
+ # expat@2.1.0%gcc@4.9.3=linux-x86_64
+ module load expat-2.1.0-gcc-4.9.3-6pkz2ucnk2e62imwakejjvbv6egncppd
+ # git@2.8.0-rc2%gcc@4.9.3+curl+expat=linux-x86_64
+ module load git-2.8.0-rc2-gcc-4.9.3-3bib4hqtnv5xjjoq5ugt3inblt4xrgkd
+
+The script may be further edited by removing unnecessary modules.
+This script may be directly executed in bash via
+
+.. code-block :: sh
+
+ source <( spack module find tcl --dependencies --shell py-numpy git )
+
+
+Regenerating Module files
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
.. code-block:: python
def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
@@ -1395,23 +1468,23 @@ files in the ``cmake`` package while retaining its dependencies.
.. code-block:: sh
-
+
$ spack view -v symlink myview cmake@3.5.2
==> Linking package: "ncurses"
==> Linking package: "zlib"
==> Linking package: "openssl"
==> Linking package: "cmake"
-
+
$ ls myview/
bin doc etc include lib share
$ ls myview/bin/
captoinfo clear cpack ctest infotocap openssl tabs toe tset
ccmake cmake c_rehash infocmp ncurses6-config reset tic tput
-
+
$ spack view -v -d false rm myview cmake@3.5.2
==> Removing package: "cmake"
-
+
$ ls myview/bin/
captoinfo c_rehash infotocap openssl tabs toe tset
clear infocmp ncurses6-config reset tic tput
@@ -1421,7 +1494,7 @@ Limitations of Filesystem Views
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
This section describes some limitations that should be considered in
-using filesystems views.
+using filesystems views.
Filesystem views are merely organizational. The binary executable
programs, shared libraries and other build products found in a view
diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst
index 0b618aa683..d28fe4b2a5 100644
--- a/lib/spack/docs/developer_guide.rst
+++ b/lib/spack/docs/developer_guide.rst
@@ -80,10 +80,11 @@ with a high level view of Spack's directory structure::
var/
spack/ <- build & stage directories
- repos/ <- contains package repositories
- builtin/ <- pkg repository that comes with Spack
- repo.yaml <- descriptor for the builtin repository
- packages/ <- directories under here contain packages
+ repos/ <- contains package repositories
+ builtin/ <- pkg repository that comes with Spack
+ repo.yaml <- descriptor for the builtin repository
+ packages/ <- directories under here contain packages
+ cache/ <- saves resources downloaded during installs
opt/
spack/ <- packages are installed here
diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst
index dad04d053b..583575a565 100644
--- a/lib/spack/docs/mirrors.rst
+++ b/lib/spack/docs/mirrors.rst
@@ -214,3 +214,21 @@ Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``::
If you want to change the order in which mirrors are searched for
packages, you can edit this file and reorder the sections. Spack will
search the topmost mirror first and the bottom-most mirror last.
+
+.. _caching:
+
+Local Default Cache
+----------------------------
+
+Spack caches resources that are downloaded as part of installs. The cache is
+a valid spack mirror: it uses the same directory structure and naming scheme
+as other Spack mirrors (so it can be copied anywhere and referenced with a URL
+like other mirrors). The mirror is maintained locally (within the Spack
+installation directory) at :file:`var/spack/cache/`. It is always enabled (and
+is always searched first when attempting to retrieve files for an installation)
+but can be cleared with :ref:`purge <spack-purge>`; the cache directory can also
+be deleted manually without issue.
+
+Caching includes retrieved tarball archives and source control repositories, but
+only resources with an associated digest or commit ID (e.g. a revision number
+for SVN) will be cached.
diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst
index 433760f8c8..82f12cedda 100644
--- a/lib/spack/docs/packaging_guide.rst
+++ b/lib/spack/docs/packaging_guide.rst
@@ -703,6 +703,13 @@ Fetching a revision
Subversion branches are handled as part of the directory structure, so
you can check out a branch or tag by changing the ``url``.
+Automatic caching of files fetched during installation
+------------------------------------------------------
+
+Spack maintains a cache (described :ref:`here <caching>`) which saves files
+retrieved during package installations to avoid re-downloading in the case that
+a package is installed with a different specification (but the same version) or
+reinstalled on account of a change in the hashing scheme.
.. _license:
@@ -2618,11 +2625,16 @@ build process will start from scratch.
``spack purge``
~~~~~~~~~~~~~~~~~
-Cleans up all of Spack's temporary files. Use this to recover disk
-space if temporary files from interrupted or failed installs
-accumulate in the staging area. This is equivalent to running ``spack
-clean`` for every package you have fetched or staged.
+Cleans up all of Spack's temporary and cached files. This can be used to
+recover disk space if temporary files from interrupted or failed installs
+accumulate in the staging area.
+
+When called with ``--stage`` or ``--all`` (or without arguments, in which case
+the default is ``--all``) this removes all staged files; this is equivalent to
+running ``spack clean`` for every package you have fetched or staged.
+When called with ``--cache`` or ``--all`` this will clear all resources
+:ref:`cached <caching>` during installs.
Keeping the stage directory on success
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/lib/spack/spack/cmd/load.py b/lib/spack/spack/cmd/load.py
index 54cf01eb43..205abbb6b3 100644
--- a/lib/spack/spack/cmd/load.py
+++ b/lib/spack/spack/cmd/load.py
@@ -31,7 +31,7 @@ def setup_parser(subparser):
"""Parser is only constructed so that this prints a nice help
message with -h. """
subparser.add_argument(
- 'spec', nargs=argparse.REMAINDER, help='Spec of package to load with modules.')
+ 'spec', nargs=argparse.REMAINDER, help="Spec of package to load with modules. (If -, read specs from STDIN)")
def load(parser, args):
diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py
index 5292d42225..3cefb512c2 100644
--- a/lib/spack/spack/cmd/module.py
+++ b/lib/spack/spack/cmd/module.py
@@ -22,6 +22,7 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
+from __future__ import print_function
import os
import shutil
import sys
@@ -41,46 +42,98 @@ def setup_parser(subparser):
sp.add_parser('refresh', help='Regenerate all module files.')
find_parser = sp.add_parser('find', help='Find module files for packages.')
- find_parser.add_argument('module_type',
- help="Type of module to find file for. [" +
- '|'.join(module_types) + "]")
- find_parser.add_argument('spec',
- nargs='+',
- help='spec to find a module file for.')
+ find_parser.add_argument(
+ 'module_type',
+ help="Type of module to find file for. [" +
+ '|'.join(module_types) + "]")
-def module_find(mtype, spec_array):
+ find_parser.add_argument(
+ '-r', '--dependencies', action='store_true',
+ dest='recurse_dependencies',
+ help='Recursively traverse dependencies for modules to load.')
+
+ find_parser.add_argument(
+ '-s', '--shell', action='store_true', dest='shell',
+ help='Generate shell script (instead of input for module command)')
+
+ find_parser.add_argument(
+ '-p', '--prefix', dest='prefix',
+ help='Prepend to module names when issuing module load commands')
+
+ find_parser.add_argument(
+ 'spec', nargs='+',
+ help='spec to find a module file for.')
+
+
+def module_find(mtype, flags, spec_array):
"""Look at all installed packages and see if the spec provided
matches any. If it does, check whether there is a module file
of type <mtype> there, and print out the name that the user
should type to use that package's module.
+ prefix:
+ Prepend this to module names when issuing "module load" commands.
+ Some systems seem to need it.
"""
if mtype not in module_types:
tty.die("Invalid module type: '%s'. Options are %s" %
(mtype, comma_or(module_types)))
- specs = spack.cmd.parse_specs(spec_array)
- if len(specs) > 1:
- tty.die("You can only pass one spec.")
- spec = specs[0]
-
- specs = spack.installed_db.query(spec)
- if len(specs) == 0:
- tty.die("No installed packages match spec %s" % spec)
-
- if len(specs) > 1:
- tty.error("Multiple matches for spec %s. Choose one:" % spec)
- for s in specs:
- sys.stderr.write(s.tree(color=True))
- sys.exit(1)
-
- mt = module_types[mtype]
- mod = mt(specs[0])
- if not os.path.isfile(mod.file_name):
- tty.die("No %s module is installed for %s" % (mtype, spec))
-
- print(mod.use_name)
-
+ # --------------------------------------
+ def _find_modules(spec, modules_list):
+ """Finds all modules and sub-modules for a spec"""
+ if str(spec.version) == 'system':
+ # No Spack module for system-installed packages
+ return
+
+ if flags.recurse_dependencies:
+ for dep in spec.dependencies.values():
+ _find_modules(dep, modules_list)
+
+ mod = module_types[mtype](spec)
+ if not os.path.isfile(mod.file_name):
+ tty.die("No %s module is installed for %s" % (mtype, spec))
+ modules_list.append((spec, mod))
+
+
+ # --------------------------------------
+ raw_specs = spack.cmd.parse_specs(spec_array)
+ modules = set() # Modules we will load
+ seen = set()
+ for raw_spec in raw_specs:
+
+ # ----------- Make sure the spec only resolves to ONE thing
+ specs = spack.installed_db.query(raw_spec)
+ if len(specs) == 0:
+ tty.die("No installed packages match spec %s" % raw_spec)
+
+ if len(specs) > 1:
+ tty.error("Multiple matches for spec %s. Choose one:" % spec)
+ for s in specs:
+ sys.stderr.write(s.tree(color=True))
+ sys.exit(1)
+ spec = specs[0]
+
+ # ----------- Chase down modules for it and all its dependencies
+ modules_dups = list()
+ _find_modules(spec, modules_dups)
+
+ # Remove duplicates while keeping order
+ modules_unique = list()
+ for spec,mod in modules_dups:
+ if mod.use_name not in seen:
+ modules_unique.append((spec,mod))
+ seen.add(mod.use_name)
+
+ # Output...
+ if flags.shell:
+ module_cmd = {'tcl': 'module load', 'dotkit': 'dotkit use'}[mtype]
+ for spec,mod in modules_unique:
+ if flags.shell:
+ print('# %s' % spec.format())
+ print('%s %s%s' % (module_cmd, flags.prefix, mod.use_name))
+ else:
+ print(mod.use_name)
def module_refresh():
"""Regenerate all module files for installed packages known to
@@ -101,4 +154,4 @@ def module(parser, args):
module_refresh()
elif args.module_command == 'find':
- module_find(args.module_type, args.spec)
+ module_find(args.module_type, args, args.spec)
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-10-28 17:04:18 +0000