From c6d8208150f7827da5a32c4c9187714ccb6651a1 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 1 Apr 2016 22:51:14 -0400 Subject: Added recursive module loading --- lib/spack/spack/cmd/load.py | 2 +- lib/spack/spack/cmd/module.py | 87 ++++++++++++++++++++++++++++++++----------- 2 files changed, 67 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/load.py b/lib/spack/spack/cmd/load.py index 30d86c3b01..aa299a0bb3 100644 --- a/lib/spack/spack/cmd/load.py +++ b/lib/spack/spack/cmd/load.py @@ -31,7 +31,7 @@ def setup_parser(subparser): """Parser is only constructed so that this prints a nice help message with -h. """ subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, help='Spec of package to load with modules.') + 'spec', nargs=argparse.REMAINDER, help="Spec of package to load with modules. (If -, read specs from STDIN)") def load(parser, args): diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index 315d9fc926..b79737ab22 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -48,40 +48,85 @@ def setup_parser(subparser): find_parser = sp.add_parser('find', help='Find module files for packages.') find_parser.add_argument( 'module_type', help="Type of module to find file for. [" + '|'.join(module_types) + "]") + find_parser.add_argument( + '-r', '--dependencies', action='store_true', dest='recurse_dependencies', + help='Recursively traverse dependencies for modules to load.') + + find_parser.add_argument( + '-s', '--shell', action='store_true', dest='shell', + help='Generate shell script (instead of input for module command)') + find_parser.add_argument('spec', nargs='+', help='spec to find a module file for.') -def module_find(mtype, spec_array): + + +def module_find(mtype, flags, spec_array): """Look at all installed packages and see if the spec provided matches any. If it does, check whether there is a module file of type there, and print out the name that the user should type to use that package's module. """ - if mtype not in module_types: - tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types))) - specs = spack.cmd.parse_specs(spec_array) - if len(specs) > 1: - tty.die("You can only pass one spec.") - spec = specs[0] + # -------------------------------------- + def _find_modules(spec, modules_list): + """Finds all modules and sub-modules for a spec""" + if str(spec.version) == 'system': + # No Spack module for system-installed packages + return + + if flags.recurse_dependencies: + for dep in spec.dependencies.values(): + _find_modules(dep, modules_list) - specs = spack.installed_db.query(spec) - if len(specs) == 0: - tty.die("No installed packages match spec %s" % spec) + mod = module_types[mtype](spec) + if not os.path.isfile(mod.file_name): + tty.die("No %s module is installed for %s" % (mtype, spec)) + modules_list.append((spec, mod)) - if len(specs) > 1: - tty.error("Multiple matches for spec %s. Choose one:" % spec) - for s in specs: - sys.stderr.write(s.tree(color=True)) - sys.exit(1) - mt = module_types[mtype] - mod = mt(specs[0]) - if not os.path.isfile(mod.file_name): - tty.die("No %s module is installed for %s" % (mtype, spec)) + # -------------------------------------- - print(mod.use_name) + if mtype not in module_types: + tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types))) + raw_specs = spack.cmd.parse_specs(spec_array) + modules = set() # Modules we will load + seen = set() + for raw_spec in raw_specs: + + # ----------- Make sure the spec only resolves to ONE thing + specs = spack.installed_db.query(raw_spec) + if len(specs) == 0: + tty.die("No installed packages match spec %s" % raw_spec) + + if len(specs) > 1: + tty.error("Multiple matches for spec %s. Choose one:" % spec) + for s in specs: + sys.stderr.write(s.tree(color=True)) + sys.exit(1) + spec = specs[0] + + # ----------- Chase down modules for it and all its dependencies + modules_dups = list() + _find_modules(spec, modules_dups) + + # Remove duplicates while keeping order + modules_unique = list() + for spec,mod in modules_dups: + if mod.use_name not in seen: + modules_unique.append((spec,mod)) + seen.add(mod.use_name) + + # Output... + if flags.shell: + module_cmd = {'tcl' : 'module load', 'dotkit' : 'dotkit use'}[mtype] + for spec,mod in modules_unique: + if flags.shell: + print '# %s' % spec.format() + print '%s %s' % (module_cmd, mod.use_name) + else: + print mod.use_name def module_refresh(): """Regenerate all module files for installed packages known to @@ -104,4 +149,4 @@ def module(parser, args): module_refresh() elif args.module_command == 'find': - module_find(args.module_type, args.spec) + module_find(args.module_type, args, args.spec) -- cgit v1.2.3-70-g09d2 From 281835887a91e13bee17b5954a9ff8c7db8e299c Mon Sep 17 00:00:00 2001 From: citibeth Date: Fri, 1 Apr 2016 22:52:36 -0400 Subject: Added documentation for recursive modules --- lib/spack/docs/basic_usage.rst | 69 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) (limited to 'lib') diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index accf09cc2a..78fc73d738 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -943,6 +943,75 @@ used ``gcc``. You could therefore just type: To identify just the one built with the Intel compiler. +Recursive Modules +`````````````````` + +In some cases, it is desirable to load not just a module, but also all +the modules it depends on. This is not required for most modules +because Spack builds binaries with RPATH support. However, not all +packages use RPATH to find their dependencies: this can be true in +particular for Python extensions, which are currently *not* built with +RPATH. + +Modules may be loaded recursively with the command: + +.. code-block:: sh + + $ module load `spack module tcl --dependencies ... + +More than one spec may be placed on the command line here. + +Module Comamnds for Shell Scripts +`````````````````````````````````` + +Although Spack is flexbile, the ``module`` command is much faster. +This could become an issue when emitting a series of ``spack load`` +commands inside a shell script. By adding the ``--shell`` flag, +``spack module find`` may also be used to generate code that can be +cut-and-pasted into a shell script. For example: + +.. code-block:: sh + + $ spack module find tcl --dependencies --shell py-numpy git + # bzip2@1.0.6%gcc@4.9.3=linux-x86_64 + module load bzip2-1.0.6-gcc-4.9.3-ktnrhkrmbbtlvnagfatrarzjojmkvzsx + # ncurses@6.0%gcc@4.9.3=linux-x86_64 + module load ncurses-6.0-gcc-4.9.3-kaazyneh3bjkfnalunchyqtygoe2mncv + # zlib@1.2.8%gcc@4.9.3=linux-x86_64 + module load zlib-1.2.8-gcc-4.9.3-v3ufwaahjnviyvgjcelo36nywx2ufj7z + # sqlite@3.8.5%gcc@4.9.3=linux-x86_64 + module load sqlite-3.8.5-gcc-4.9.3-a3eediswgd5f3rmto7g3szoew5nhehbr + # readline@6.3%gcc@4.9.3=linux-x86_64 + module load readline-6.3-gcc-4.9.3-se6r3lsycrwxyhreg4lqirp6xixxejh3 + # python@3.5.1%gcc@4.9.3=linux-x86_64 + module load python-3.5.1-gcc-4.9.3-5q5rsrtjld4u6jiicuvtnx52m7tfhegi + # py-setuptools@20.5%gcc@4.9.3=linux-x86_64 + module load py-setuptools-20.5-gcc-4.9.3-4qr2suj6p6glepnedmwhl4f62x64wxw2 + # py-nose@1.3.7%gcc@4.9.3=linux-x86_64 + module load py-nose-1.3.7-gcc-4.9.3-pwhtjw2dvdvfzjwuuztkzr7b4l6zepli + # openblas@0.2.17%gcc@4.9.3+shared=linux-x86_64 + module load openblas-0.2.17-gcc-4.9.3-pw6rmlom7apfsnjtzfttyayzc7nx5e7y + # py-numpy@1.11.0%gcc@4.9.3+blas+lapack=linux-x86_64 + module load py-numpy-1.11.0-gcc-4.9.3-mulodttw5pcyjufva4htsktwty4qd52r + # curl@7.47.1%gcc@4.9.3=linux-x86_64 + module load curl-7.47.1-gcc-4.9.3-ohz3fwsepm3b462p5lnaquv7op7naqbi + # autoconf@2.69%gcc@4.9.3=linux-x86_64 + module load autoconf-2.69-gcc-4.9.3-bkibjqhgqm5e3o423ogfv2y3o6h2uoq4 + # cmake@3.5.0%gcc@4.9.3~doc+ncurses+openssl~qt=linux-x86_64 + module load cmake-3.5.0-gcc-4.9.3-x7xnsklmgwla3ubfgzppamtbqk5rwn7t + # expat@2.1.0%gcc@4.9.3=linux-x86_64 + module load expat-2.1.0-gcc-4.9.3-6pkz2ucnk2e62imwakejjvbv6egncppd + # git@2.8.0-rc2%gcc@4.9.3+curl+expat=linux-x86_64 + module load git-2.8.0-rc2-gcc-4.9.3-3bib4hqtnv5xjjoq5ugt3inblt4xrgkd + +The script may be further edited by removing unnecessary modules. +This script may be directly executed in bash via + +.. code-block :: sh + + source <( spack module find tcl --dependencies --shell py-numpy git ) + + Regenerating Module files ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- cgit v1.2.3-70-g09d2 From 8f846d507d7523d5ce6f0a90b33d9409bab2a72c Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Wed, 22 Jun 2016 15:36:04 -0400 Subject: Added --prefix to module command. --- lib/spack/spack/cmd/module.py | 48 ++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index b79737ab22..443e95a3c2 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -1,49 +1,45 @@ ############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # # For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. +# conditions of the GNU Lesser General Public License for more details. # -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys +from __future__ import print_function import os import shutil -import argparse +import sys import llnl.util.tty as tty -from llnl.util.lang import partition_list -from llnl.util.filesystem import mkdirp - import spack.cmd +from llnl.util.filesystem import mkdirp from spack.modules import module_types from spack.util.string import * -from spack.spec import Spec - -description ="Manipulate modules and dotkits." +description = "Manipulate modules and dotkits." def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='module_command') - refresh_parser = sp.add_parser('refresh', help='Regenerate all module files.') + sp.add_parser('refresh', help='Regenerate all module files.') find_parser = sp.add_parser('find', help='Find module files for packages.') find_parser.add_argument( @@ -56,6 +52,10 @@ def setup_parser(subparser): '-s', '--shell', action='store_true', dest='shell', help='Generate shell script (instead of input for module command)') + find_parser.add_argument( + '-p', '--prefix', dest='prefix', + help='Prepend to module names when issuing module load commands') + find_parser.add_argument('spec', nargs='+', help='spec to find a module file for.') @@ -66,6 +66,9 @@ def module_find(mtype, flags, spec_array): matches any. If it does, check whether there is a module file of type there, and print out the name that the user should type to use that package's module. + prefix: + Prepend this to module names when issuing "module load" commands. + Some systems seem to need it. """ # -------------------------------------- @@ -88,7 +91,8 @@ def module_find(mtype, flags, spec_array): # -------------------------------------- if mtype not in module_types: - tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types))) + tty.die("Invalid module type: '%s'. Options are %s" % + (mtype, comma_or(module_types))) raw_specs = spack.cmd.parse_specs(spec_array) modules = set() # Modules we will load @@ -123,10 +127,10 @@ def module_find(mtype, flags, spec_array): module_cmd = {'tcl' : 'module load', 'dotkit' : 'dotkit use'}[mtype] for spec,mod in modules_unique: if flags.shell: - print '# %s' % spec.format() - print '%s %s' % (module_cmd, mod.use_name) + print('# %s' % spec.format()) + print('%s %s%s' % (module_cmd, flags.prefix, mod.use_name)) else: - print mod.use_name + print(mod.use_name) def module_refresh(): """Regenerate all module files for installed packages known to @@ -139,11 +143,9 @@ def module_refresh(): shutil.rmtree(cls.path, ignore_errors=False) mkdirp(cls.path) for spec in specs: - tty.debug(" Writing file for %s" % spec) cls(spec).write() - def module(parser, args): if args.module_command == 'refresh': module_refresh() -- cgit v1.2.3-70-g09d2