diff options
| -rw-r--r-- | lib/spack/docs/basic_usage.rst | 27 | ||||
| -rw-r--r-- | lib/spack/docs/module_file_support.rst | 39 | ||||
| -rw-r--r-- | lib/spack/docs/workflows.rst | 33 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/__init__.py | 14 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/find.py | 10 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/load.py | 66 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/module.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/unload.py | 63 | ||||
| -rw-r--r-- | lib/spack/spack/environment.py | 68 | ||||
| -rw-r--r-- | lib/spack/spack/main.py | 11 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/find.py | 13 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/load.py | 125 | ||||
| -rw-r--r-- | lib/spack/spack/user_environment.py | 91 | ||||
| -rw-r--r-- | share/spack/csh/spack.csh | 54 | ||||
| -rwxr-xr-x | share/spack/qa/setup-env-test.sh | 17 | ||||
| -rwxr-xr-x | share/spack/setup-env.sh | 61 | ||||
| -rwxr-xr-x | share/spack/spack-completion.bash | 6 |
17 files changed, 516 insertions, 184 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 3ad83c57bd..8342ab4489 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -1317,10 +1317,9 @@ directly when you run ``python``: Using Extensions ^^^^^^^^^^^^^^^^ -There are three ways to get ``numpy`` working in Python. The first is -to use :ref:`shell-support`. You can simply ``load`` the -module for the extension, and it will be added to the ``PYTHONPATH`` -in your current shell: +There are four ways to get ``numpy`` working in Python. The first is +to use :ref:`shell-support`. You can simply ``load`` the extension, +and it will be added to the ``PYTHONPATH`` in your current shell: .. code-block:: console @@ -1330,11 +1329,29 @@ in your current shell: Now ``import numpy`` will succeed for as long as you keep your current session open. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Loading Extensions via Modules +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Instead of using Spack's environment modification capabilities through +the ``spack load`` command, you can load numpy through your +environment modules (using ``environment-modules`` or ``lmod``). This +will also add the extension to the ``PYTHONPATH`` in your current +shell. + +.. code-block:: console + + $ module load <name of numpy module> + +If you do not know the name of the specific numpy module you wish to +load, you can use the ``spack module tcl|lmod loads`` command to get +the name of the module from the Spack spec. + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Activating Extensions in a View ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The second way to use extensions is to create a view, which merges the +Another way to use extensions is to create a view, which merges the python installation along with the extensions into a single prefix. See :ref:`filesystem-views` for a more in-depth description of views and :ref:`cmd-spack-view` for usage of the ``spack view`` command. diff --git a/lib/spack/docs/module_file_support.rst b/lib/spack/docs/module_file_support.rst index ba964c2d61..aa7eb57653 100644 --- a/lib/spack/docs/module_file_support.rst +++ b/lib/spack/docs/module_file_support.rst @@ -119,7 +119,7 @@ For example this will add the ``mpich`` package built with ``gcc`` to your path: # ... wait for install ... - $ spack load mpich %gcc@4.4.7 # modules + $ spack load mpich %gcc@4.4.7 $ which mpicc ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/mpich@3.0.4/bin/mpicc @@ -129,27 +129,29 @@ want to use a package, you can type unload or unuse similarly: .. code-block:: console - $ spack unload mpich %gcc@4.4.7 # modules + $ spack unload mpich %gcc@4.4.7 .. note:: - The ``load`` and ``unload`` subcommands are - only available if you have enabled Spack's shell support *and* you - have environment-modules installed on your machine. + The ``load`` and ``unload`` subcommands are only available if you + have enabled Spack's shell support. These command DO NOT use the + underlying Spack-generated module files. -^^^^^^^^^^^^^^^^^^^^^^ -Ambiguous module names -^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^ +Ambiguous specs +^^^^^^^^^^^^^^^ -If a spec used with load/unload or use/unuse is ambiguous (i.e. more -than one installed package matches it), then Spack will warn you: +If a spec used with load/unload or is ambiguous (i.e. more than one +installed package matches it), then Spack will warn you: .. code-block:: console $ spack load libelf - ==> Error: Multiple matches for spec libelf. Choose one: - libelf@0.8.13%gcc@4.4.7 arch=linux-debian7-x86_64 - libelf@0.8.13%intel@15.0.0 arch=linux-debian7-x86_64 + ==> Error: libelf matches multiple packages. + Matching packages: + libelf@0.8.13%gcc@4.4.7 arch=linux-debian7-x86_64 + libelf@0.8.13%intel@15.0.0 arch=linux-debian7-x86_64 + Use a more specific spec You can either type the ``spack load`` command again with a fully qualified argument, or you can add just enough extra constraints to @@ -171,8 +173,15 @@ To identify just the one built with the Intel compiler. ``spack module tcl loads`` ^^^^^^^^^^^^^^^^^^^^^^^^^^ -In some cases, it is desirable to load not just a module, but also all -the modules it depends on. This is not required for most modules +In some cases, it is desirable to use a Spack-generated module, rather +than relying on Spack's built-in user-environment modification +capabilities. To translate a spec into a module name, use ``spack +module tcl loads`` or ``spack module lmod loads`` depending on the +module system desired. + + +To load not just a module, but also all the modules it depends on, use +the ``--dependencies`` option. This is not required for most modules because Spack builds binaries with RPATH support. However, not all packages use RPATH to find their dependencies: this can be true in particular for Python extensions, which are currently *not* built with diff --git a/lib/spack/docs/workflows.rst b/lib/spack/docs/workflows.rst index 73f4655d1e..914f84041b 100644 --- a/lib/spack/docs/workflows.rst +++ b/lib/spack/docs/workflows.rst @@ -253,14 +253,14 @@ However, other more powerful methods are generally preferred for user environments. -^^^^^^^^^^^^^^^^^^^^^^^ -Spack-Generated Modules -^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Using ``spack load`` to Manage the User Environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Suppose that Spack has been used to install a set of command-line programs, which users now wish to use. One can in principle put a number of ``spack load`` commands into ``.bashrc``, for example, to -load a set of Spack-generated modules: +load a set of Spack packages: .. code-block:: sh @@ -273,7 +273,7 @@ load a set of Spack-generated modules: Although simple load scripts like this are useful in many cases, they have some drawbacks: -1. The set of modules loaded by them will in general not be +1. The set of packages loaded by them will in general not be consistent. They are a decent way to load commands to be called from command shells. See below for better ways to assemble a consistent set of packages for building application programs. @@ -285,19 +285,24 @@ have some drawbacks: other hand, are not very smart: if the user-supplied spec matches more than one installed package, then ``spack module tcl loads`` will fail. This may change in the future. For now, the workaround is to - be more specific on any ``spack module tcl loads`` lines that fail. + be more specific on any ``spack load`` commands that fail. """""""""""""""""""""" Generated Load Scripts """""""""""""""""""""" -Another problem with using `spack load` is, it is slow; a typical user -environment could take several seconds to load, and would not be -appropriate to put into ``.bashrc`` directly. It is preferable to use -a series of ``spack module tcl loads`` commands to pre-compute which -modules to load. These can be put in a script that is run whenever -installed Spack packages change. For example: +Another problem with using `spack load` is, it can be slow; a typical +user environment could take several seconds to load, and would not be +appropriate to put into ``.bashrc`` directly. This is because it +requires the full start-up overhead of python/Spack for each command. +In some circumstances it is preferable to use a series of ``spack +module tcl loads`` (or ``spack module lmod loads``) commands to +pre-compute which modules to load. This will generate the modulenames +to load the packages using environment modules, rather than Spack's +built-in support for environment modifications. These can be put in a +script that is run whenever installed Spack packages change. For +example: .. code-block:: sh @@ -634,7 +639,7 @@ Global Activations Python (and similar systems) packages directly or creating a view. If extensions are globally activated, then ``spack load python`` will also load all the extensions activated for the given ``python``. -This reduces the need for users to load a large number of modules. +This reduces the need for users to load a large number of packages. However, Spack global activations have two potential drawbacks: @@ -1254,7 +1259,7 @@ In order to build and run the image, execute: RUN spack install tar \ && spack clean -a - # need the modules already during image build? + # need the executables from a package already during image build? #RUN /bin/bash -l -c ' \ # spack load tar \ # && which tar' diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index ef620c112b..2a75a87b54 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -190,6 +190,20 @@ def disambiguate_spec(spec, env, local=False, installed=True): database query. See ``spack.database.Database._query`` for details. """ hashes = env.all_hashes() if env else None + return disambiguate_spec_from_hashes(spec, hashes, local, installed) + + +def disambiguate_spec_from_hashes(spec, hashes, local=False, installed=True): + """Given a spec and a list of hashes, get concrete spec the spec refers to. + + Arguments: + spec (spack.spec.Spec): a spec to disambiguate + hashes (iterable): a set of hashes of specs among which to disambiguate + local (boolean, default False): do not search chained spack instances + installed (boolean or any, or spack.database.InstallStatus or iterable + of spack.database.InstallStatus): install status argument passed to + database query. See ``spack.database.Database._query`` for details. + """ if local: matching_specs = spack.store.db.query_local(spec, hashes=hashes, installed=installed) diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index b337cd0738..fa36e1cd26 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -4,7 +4,9 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) from __future__ import print_function + import copy +import os import llnl.util.tty as tty import llnl.util.tty.color as color @@ -14,6 +16,7 @@ import spack.environment as ev import spack.repo import spack.cmd as cmd import spack.cmd.common.arguments as arguments +import spack.user_environment as uenv from spack.util.string import plural from spack.database import InstallStatuses @@ -81,6 +84,9 @@ def setup_parser(subparser): action='store_true', dest='variants', help='show variants in output (can be long)') + subparser.add_argument( + '--loaded', action='store_true', + help='show only packages loaded in the user environment') subparser.add_argument('-M', '--only-missing', action='store_true', dest='only_missing', @@ -220,6 +226,10 @@ def find(parser, args): packages_with_tags = spack.repo.path.packages_with_tags(*args.tags) results = [x for x in results if x.name in packages_with_tags] + if args.loaded: + hashes = os.environ.get(uenv.spack_loaded_hashes_var, '').split(':') + results = [x for x in results if x.dag_hash() in hashes] + # Display the result if args.json: cmd.display_specs_as_json(results, deps=args.deps) diff --git a/lib/spack/spack/cmd/load.py b/lib/spack/spack/cmd/load.py index 9c48fe802a..80c7263a7a 100644 --- a/lib/spack/spack/cmd/load.py +++ b/lib/spack/spack/cmd/load.py @@ -3,10 +3,18 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack.cmd.common import print_module_placeholder_help, arguments +import sys -description = "add package to environment using `module load`" -section = "modules" +import llnl.util.tty as tty + +import spack.cmd +import spack.cmd.common.arguments as arguments +import spack.environment as ev +import spack.util.environment +import spack.user_environment as uenv + +description = "add package to the user environment" +section = "user environment" level = "short" @@ -14,8 +22,56 @@ def setup_parser(subparser): """Parser is only constructed so that this prints a nice help message with -h. """ arguments.add_common_arguments( - subparser, ['recurse_dependencies', 'installed_spec']) + subparser, ['recurse_dependencies', 'installed_specs']) + + shells = subparser.add_mutually_exclusive_group() + shells.add_argument( + '--sh', action='store_const', dest='shell', const='sh', + help="print sh commands to load the package") + shells.add_argument( + '--csh', action='store_const', dest='shell', const='csh', + help="print csh commands to load the package") + + subparser.add_argument( + '--only', + default='package,dependencies', + dest='things_to_load', + choices=['package', 'dependencies'], + help="""select whether to load the package and its dependencies +the default is to load the package and all dependencies +alternatively one can decide to load only the package or only +the dependencies""" + ) def load(parser, args): - print_module_placeholder_help() + env = ev.get_env(args, 'load') + specs = [spack.cmd.disambiguate_spec(spec, env) + for spec in spack.cmd.parse_specs(args.specs)] + + if not args.shell: + msg = [ + "This command works best with Spack's shell support", + "" + ] + spack.cmd.common.shell_init_instructions + [ + 'Or, if you want to use `spack load` without initializing', + 'shell support, you can run one of these:', + '', + ' eval `spack load --sh %s` # for bash/sh' % args.specs, + ' eval `spack load --csh %s` # for csh/tcsh' % args.specs, + ] + tty.msg(*msg) + return 1 + + if 'dependencies' in args.things_to_load: + include_roots = 'package' in args.things_to_load + specs = [dep for spec in specs + for dep in spec.traverse(root=include_roots, order='post')] + + env_mod = spack.util.environment.EnvironmentModifications() + for spec in specs: + env_mod.extend(uenv.environment_modifications_for_spec(spec)) + env_mod.prepend_path(uenv.spack_loaded_hashes_var, spec.dag_hash()) + cmds = env_mod.shell_modifications(args.shell) + + sys.stdout.write(cmds) diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index 19a2eacce1..f86f3e5f25 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -11,7 +11,7 @@ import spack.cmd.modules.lmod import spack.cmd.modules.tcl description = "manipulate module files" -section = "modules" +section = "user environment" level = "short" diff --git a/lib/spack/spack/cmd/unload.py b/lib/spack/spack/cmd/unload.py index 92a25478b6..d19a33102f 100644 --- a/lib/spack/spack/cmd/unload.py +++ b/lib/spack/spack/cmd/unload.py @@ -3,18 +3,71 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack.cmd.common import print_module_placeholder_help, arguments +import sys +import os -description = "remove package from environment using `module unload`" -section = "modules" +import llnl.util.tty as tty + +import spack.cmd +import spack.cmd.common.arguments as arguments +import spack.util.environment +import spack.user_environment as uenv +import spack.error + +description = "remove package from the user environment" +section = "user environment" level = "short" def setup_parser(subparser): """Parser is only constructed so that this prints a nice help message with -h. """ - arguments.add_common_arguments(subparser, ['installed_spec']) + arguments.add_common_arguments(subparser, ['installed_specs']) + + shells = subparser.add_mutually_exclusive_group() + shells.add_argument( + '--sh', action='store_const', dest='shell', const='sh', + help="print sh commands to activate the environment") + shells.add_argument( + '--csh', action='store_const', dest='shell', const='csh', + help="print csh commands to activate the environment") + + subparser.add_argument('-a', '--all', action='store_true', + help='unload all loaded Spack packages.') def unload(parser, args): - print_module_placeholder_help() + """Unload spack packages from the user environment.""" + if args.specs and args.all: + raise spack.error.SpackError("Cannot specify specs on command line" + " when unloading all specs with '--all'") + + hashes = os.environ.get(uenv.spack_loaded_hashes_var, '').split(':') + if args.specs: + specs = [spack.cmd.disambiguate_spec_from_hashes(spec, hashes) + for spec in spack.cmd.parse_specs(args.specs)] + else: + specs = spack.store.db.query(hashes=hashes) + + if not args.shell: + msg = [ + "This command works best with Spack's shell support", + "" + ] + spack.cmd.common.shell_init_instructions + [ + 'Or, if you want to use `spack unload` without initializing', + 'shell support, you can run one of these:', + '', + ' eval `spack unload --sh %s` # for bash/sh' % args.specs, + ' eval `spack unload --csh %s` # for csh/tcsh' % args.specs, + ] + tty.msg(*msg) + return 1 + + env_mod = spack.util.environment.EnvironmentModifications() + for spec in specs: + env_mod.extend( + uenv.environment_modifications_for_spec(spec).reversed()) + env_mod.remove_path(uenv.spack_loaded_hashes_var, spec.dag_hash()) + cmds = env_mod.shell_modifications(args.shell) + + sys.stdout.write(cmds) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 341d62fecb..351120b127 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -29,9 +29,7 @@ import spack.store import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml import spack.config -import spack.build_environment as build_env - -from spack.util.prefix import Prefix +import spack.user_environment as uenv from spack.filesystem_view import YamlFilesystemView import spack.util.environment import spack.architecture as architecture @@ -1070,62 +1068,6 @@ class Environment(object): for view in self.views.values(): view.regenerate(specs, self.roots()) - prefix_inspections = { - 'bin': ['PATH'], - 'lib': ['LD_LIBRARY_PATH', 'LIBRARY_PATH', 'DYLD_LIBRARY_PATH'], - 'lib64': ['LD_LIBRARY_PATH', 'LIBRARY_PATH', 'DYLD_LIBRARY_PATH'], - 'man': ['MANPATH'], - 'share/man': ['MANPATH'], - 'share/aclocal': ['ACLOCAL_PATH'], - 'include': ['CPATH'], - 'lib/pkgconfig': ['PKG_CONFIG_PATH'], - 'lib64/pkgconfig': ['PKG_CONFIG_PATH'], - '': ['CMAKE_PREFIX_PATH'] - } - - def unconditional_environment_modifications(self, view): - """List of environment (shell) modifications to be processed for view. - - This list does not depend on the specs in this environment""" - env = spack.util.environment.EnvironmentModifications() - - for subdir, vars in self.prefix_inspections.items(): - full_subdir = os.path.join(view.root, subdir) - for var in vars: - env.prepend_path(var, full_subdir) - - return env - - def environment_modifications_for_spec(self, spec, view=None): - """List of environment (shell) modifications to be processed for spec. - - This list is specific to the location of the spec or its projection in - the view.""" - spec = spec.copy() - if view: - spec.prefix = Prefix(view.view().get_projection_for_spec(spec)) - - # generic environment modifications determined by inspecting the spec - # prefix - env = spack.util.environment.inspect_path( - spec.prefix, - self.prefix_inspections, - exclude=spack.util.environment.is_system_path - ) - - # Let the extendee/dependency modify their extensions/dependents - # before asking for package-specific modifications - env.extend( - build_env.modifications_from_dependencies( - spec, context='run' - ) - ) - # Package specific modifications - build_env.set_module_variables_for_package(spec.package) - spec.package.setup_run_environment(env) - - return env - def add_default_view_to_shell(self, shell): env_mod = spack.util.environment.EnvironmentModifications() @@ -1133,12 +1075,12 @@ class Environment(object): # No default view to add to shell return env_mod.shell_modifications(shell) - env_mod.extend(self.unconditional_environment_modifications( + env_mod.extend(uenv.unconditional_environment_modifications( self.default_view)) for _, spec in self.concretized_specs(): if spec in self.default_view and spec.package.installed: - env_mod.extend(self.environment_modifications_for_spec( + env_mod.extend(uenv.environment_modifications_for_spec( spec, self.default_view)) # deduplicate paths from specs mapped to the same location @@ -1154,13 +1096,13 @@ class Environment(object): # No default view to add to shell return env_mod.shell_modifications(shell) - env_mod.extend(self.unconditional_environment_modifications( + env_mod.extend(uenv.unconditional_environment_modifications( self.default_view).reversed()) for _, spec in self.concretized_specs(): if spec in self.default_view and spec.package.installed: env_mod.extend( - self.environment_modifications_for_spec( + uenv.environment_modifications_for_spec( spec, self.default_view).reversed()) return env_mod.shell_modifications(shell) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 2a6b774536..4ce4ae331e 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -645,6 +645,17 @@ def main(argv=None): parser.add_argument('command', nargs=argparse.REMAINDER) args, unknown = parser.parse_known_args(argv) + # Recover stored LD_LIBRARY_PATH variables from spack shell function + # This is necessary because MacOS System Integrity Protection clears + # (DY?)LD_LIBRARY_PATH variables on process start. + # Spack clears these variables before building and installing packages, + # but needs to know the prior state for commands like `spack load` and + # `spack env activate that modify the user environment. + for var in ('LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH'): + stored_var_name = 'SPACK_%s' % var + if stored_var_name in os.environ: + os.environ[var] = os.environ[stored_var_name] + # activate an environment if one was specified on the command line if not args.no_env: env = ev.find_environment(args) diff --git a/lib/spack/spack/test/cmd/find.py b/lib/spack/spack/test/cmd/find.py index 560d8169f5..8516569592 100644 --- a/lib/spack/spack/test/cmd/find.py +++ b/lib/spack/spack/test/cmd/find.py @@ -5,10 +5,12 @@ import argparse import json +import os import pytest import spack.cmd as cmd import spack.cmd.find +import spack.user_environment as uenv from spack.main import SpackCommand from spack.spec import Spec from spack.util.pattern import Bunch @@ -318,3 +320,14 @@ def test_find_prefix_in_env(mutable_mock_env_path, install_mockery, mock_fetch, find('-l') find('-L') # Would throw error on regression + + +def test_find_loaded(database, working_env): + output = find('--loaded', '--group') + assert output == '' # 0 packages installed printed separately + + os.environ[uenv.spack_loaded_hashes_var] = ':'.join( + [x.dag_hash() for x in spack.store.db.query()]) + output = find('--loaded') + expected = find() + assert output == expected diff --git a/lib/spack/spack/test/cmd/load.py b/lib/spack/spack/test/cmd/load.py new file mode 100644 index 0000000000..a10b99d45b --- /dev/null +++ b/lib/spack/spack/test/cmd/load.py @@ -0,0 +1,125 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os +from spack.main import SpackCommand +import spack.spec +import spack.user_environment as uenv + +load = SpackCommand('load') +unload = SpackCommand('unload') +install = SpackCommand('install') +location = SpackCommand('location') + + +def test_load(install_mockery, mock_fetch, mock_archive, mock_packages): + """Test that the commands generated by load add the specified prefix + inspections. Also test that Spack records loaded specs by hash in the + user environment. + + CMAKE_PREFIX_PATH is the only prefix inspection guaranteed for fake + packages, since it keys on the prefix instead of a subdir.""" + install('mpileaks') + mpileaks_spec = spack.spec.Spec('mpileaks').concretized() + + sh_out = load('--sh', '--only', 'package', 'mpileaks') + csh_out = load('--csh', '--only', 'package', 'mpileaks') + + # Test prefix inspections + sh_out_test = 'export CMAKE_PREFIX_PATH=%s' % mpileaks_spec.prefix + csh_out_test = 'setenv CMAKE_PREFIX_PATH %s' % mpileaks_spec.prefix + assert sh_out_test in sh_out + assert csh_out_test in csh_out + + # Test hashes recorded properly + hash_test_replacements = (uenv.spack_loaded_hashes_var, + mpileaks_spec.dag_hash()) + sh_hash_test = 'export %s=%s' % hash_test_replacements + csh_hash_test = 'setenv %s %s' % hash_test_replacements + assert sh_hash_test in sh_out + assert csh_hash_test in csh_out + + +def test_load_recursive(install_mockery, mock_fetch, mock_archive, + mock_packages): + """Test that the '-r' option to the load command prepends dependency prefix + inspections in post-order""" + install('mpileaks') + mpileaks_spec = spack.spec.Spec('mpileaks').concretized() + + sh_out = load('--sh', 'mpileaks') + csh_out = load('--csh', 'mpileaks') + + # Test prefix inspections + prefix_test_replacement = ':'.join(reversed( + [s.prefix for s in mpileaks_spec.traverse(order='post')])) + + sh_prefix_test = 'export CMAKE_PREFIX_PATH=%s' % prefix_test_replacement + csh_prefix_test = 'setenv CMAKE_PREFIX_PATH %s' % prefix_test_replacement + assert sh_prefix_test in sh_out + assert csh_prefix_test in csh_out + + # Test spack records loaded hashes properly + hash_test_replacement = (uenv.spack_loaded_hashes_var, ':'.join(reversed( + [s.dag_hash() for s in mpileaks_spec.traverse(order='post')]))) + sh_hash_test = 'export %s=%s' % hash_test_replacement + csh_hash_test = 'setenv %s %s' % hash_test_replacement + assert sh_hash_test in sh_out + assert csh_hash_test in csh_out + + +def test_load_includes_run_env(install_mockery, mock_fetch, mock_archive, + mock_packages): + """Tests that environment changes from the package's + `setup_run_environment` method are added to the user environment in + addition to the prefix inspections""" + install('mpileaks') + + sh_out = load('--sh', 'mpileaks') + csh_out = load('--csh', 'mpileaks') + + assert 'export FOOBAR=mpileaks' in sh_out + assert 'setenv FOOBAR mpileaks' in csh_out + + +def test_load_fails_no_shell(install_mockery, mock_fetch, mock_archive, + mock_packages): + """Test that spack load prints an error message without a shell.""" + install('mpileaks') + + out = load('mpileaks', fail_on_error=False) + assert "To initialize spack's shell commands" in out + + +def test_unload(install_mockery, mock_fetch, mock_archive, mock_packages, + working_env): + """Tests that any variables set in the user environment are undone by the + unload command""" + install('mpileaks') + mpileaks_spec = spack.spec.Spec('mpileaks').concretized() + + # Set so unload has something to do + os.environ['FOOBAR'] = 'mpileaks' + os.environ[uenv.spack_loaded_hashes_var] = '%s:%s' % ( + mpileaks_spec.dag_hash(), 'garbage') + + sh_out = unload('--sh', 'mpileaks') + csh_out = unload('--csh', 'mpileaks') + + assert 'unset FOOBAR' in sh_out + assert 'unsetenv FOOBAR' in csh_out + + assert 'export %s=garbage' % uenv.spack_loaded_hashes_var in sh_out + assert 'setenv %s garbage' % uenv.spack_loaded_hashes_var in csh_out + + +def test_unload_fails_no_shell(install_mockery, mock_fetch, mock_archive, + mock_packages, working_env): + """Test that spack unload prints an error message without a shell.""" + install('mpileaks') + mpileaks_spec = spack.spec.Spec('mpileaks').concretized() + os.environ[uenv.spack_loaded_hashes_var] = mpileaks_spec.dag_hash() + + out = unload('mpileaks', fail_on_error=False) + assert "To initialize spack's shell commands" in out diff --git a/lib/spack/spack/user_environment.py b/lib/spack/spack/user_environment.py new file mode 100644 index 0000000000..4c9fdc3d67 --- /dev/null +++ b/lib/spack/spack/user_environment.py @@ -0,0 +1,91 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import sys +import os + +import spack.util.prefix as prefix +import spack.util.environment as environment +import spack.build_environment as build_env + +#: Environment variable name Spack uses to track individually loaded packages +spack_loaded_hashes_var = 'SPACK_LOADED_HASHES' + + +def prefix_inspections(platform): + """Get list of prefix inspections for platform + + Arguments: + platform (string): the name of the platform to consider. The platform + determines what environment variables Spack will use for some + inspections. + + Returns: + A dictionary mapping subdirectory names to lists of environment + variables to modify with that directory if it exists. + """ + inspections = { + 'bin': ['PATH'], + 'lib': ['LD_LIBRARY_PATH', 'LIBRARY_PATH'], + 'lib64': ['LD_LIBRARY_PATH', 'LIBRARY_PATH'], + 'man': ['MANPATH'], + 'share/man': ['MANPATH'], + 'share/aclocal': ['ACLOCAL_PATH'], + 'include': ['CPATH'], + 'lib/pkgconfig': ['PKG_CONFIG_PATH'], + 'lib64/pkgconfig': ['PKG_CONFIG_PATH'], + '': ['CMAKE_PREFIX_PATH'] + } + + if platform == 'darwin': + for subdir in ('lib', 'lib64'): + inspections[subdir].append('DYLD_LIBRARY_PATH') + + return inspections + + +def unconditional_environment_modifications(view): + """List of environment (shell) modifications to be processed for view. + + This list does not depend on the specs in this environment""" + env = environment.EnvironmentModifications() + + for subdir, vars in prefix_inspections(sys.platform).items(): + full_subdir = os.path.join(view.root, subdir) + for var in vars: + env.prepend_path(var, full_subdir) + + return env + + +def environment_modifications_for_spec(spec, view=None): + """List of environment (shell) modifications to be processed for spec. + + This list is specific to the location of the spec or its projection in + the view.""" + spec = spec.copy() + if view: + spec.prefix = prefix.Prefix(view.view().get_projection_for_spec(spec)) + + # generic environment modifications determined by inspecting the spec + # prefix + env = environment.inspect_path( + spec.prefix, + prefix_inspections(spec.platform), + exclude=environment.is_system_path + ) + + # Let the extendee/dependency modify their extensions/dependents + # before asking for package-specific modifications + env.extend( + build_env.modifications_from_dependencies( + spec, context='run' + ) + ) + + # Package specific modifications + build_env.set_module_variables_for_package(spec.package) + spec.package.setup_run_environment(env) + + return env diff --git a/share/spack/csh/spack.csh b/share/spack/csh/spack.csh index fe2c5a1f08..b4d963ae10 100644 --- a/share/spack/csh/spack.csh +++ b/share/spack/csh/spack.csh @@ -27,6 +27,16 @@ # avoids the need to come up with a user-friendly naming scheme for # spack module files. ######################################################################## +# Store LD_LIBRARY_PATH variables from spack shell function +# This is necessary because MacOS System Integrity Protection clears +# (DY?)LD_LIBRARY_PATH variables on process start. +if ( ${?LD_LIBRARY_PATH} ) then + setenv SPACK_LD_LIBRARY_PATH $LD_LIBRARY_PATH +endif +if ( ${?DYLD_LIBRARY_PATH} ) then + setenv SPACK_DYLD_LIBRARY_PATH $DYLD_LIBRARY_PATH +endif + # accumulate initial flags for main spack command set _sp_flags = "" while ( $#_sp_args > 0 ) @@ -47,8 +57,7 @@ set _sp_spec="" [ $#_sp_args -gt 0 ] && set _sp_subcommand = ($_sp_args[1]) [ $#_sp_args -gt 1 ] && set _sp_spec = ($_sp_args[2-]) -# Figure out what type of module we're running here. -set _sp_modtype = "" +# Run subcommand switch ($_sp_subcommand) case cd: shift _sp_args # get rid of 'cd' @@ -106,35 +115,17 @@ case env: endif case load: case unload: - set _sp_module_args="""" - if ( "$_sp_spec" =~ "-*" ) then - set _sp_module_args = $_sp_spec[1] - shift _sp_spec - set _sp_spec = ($_sp_spec) + # Space in `-h` portion is important for differentiating -h option + # from variants that begin with "h" or packages with "-h" in name + if ( "$_sp_spec" =~ "*--sh*" || "$_sp_spec" =~ "*--csh*" || \ + " $_sp_spec" =~ "* -h*" || "$_sp_spec" =~ "*--help*") then + # IF a shell is given, print shell output + \spack $_sp_flags $_sp_subcommand $_sp_spec + else + # otherwise eval with csh + eval `\spack $_sp_flags $_sp_subcommand --csh $_sp_spec || \ + echo "exit 1"` endif - - # Here the user has run load or unload with a spec. Find a matching - # spec using 'spack module find', then use the appropriate module - # tool's commands to add/remove the result from the environment. - switch ($_sp_subcommand) - case "load": - # _sp_module_args may be "-r" for recursive spec retrieval - set _sp_full_spec = ( "`\spack $_sp_flags module tcl find $_sp_module_args $_sp_spec`" ) - if ( "$_sp_module_args" == "-r" ) then - # module load can handle the list of modules to load and "-r" is not a valid option - set _sp_module_args = "" - endif - if ( $? == 0 ) then - module load $_sp_module_args $_sp_full_spec - endif - breaksw - case "unload": - set _sp_full_spec = ( "`\spack $_sp_flags module tcl find $_sp_spec`" ) - if ( $? == 0 ) then - module unload $_sp_module_args $_sp_full_spec - endif - breaksw - endsw breaksw default: @@ -143,6 +134,5 @@ default: endsw _sp_end: -unset _sp_args _sp_full_spec _sp_modtype _sp_module_args -unset _sp_sh_cmd _sp_spec _sp_subcommand _sp_flags +unset _sp_args _sp_full_spec _sp_sh_cmd _sp_spec _sp_subcommand _sp_flags unset _sp_arg _sp_env_arg diff --git a/share/spack/qa/setup-env-test.sh b/share/spack/qa/setup-env-test.sh index 66284d1a96..da4fb9657d 100755 --- a/share/spack/qa/setup-env-test.sh +++ b/share/spack/qa/setup-env-test.sh @@ -104,20 +104,25 @@ contains "usage: spack module " spack -m module --help contains "usage: spack module " spack -m module title 'Testing `spack load`' -contains "module load $b_module" spack -m load b +contains "export LD_LIBRARY_PATH=$(spack -m location -i b)/lib" spack -m load --only package --sh b +succeeds spack -m load b fails spack -m load -l -contains "module load -l --arg $b_module" spack -m load -l --arg b -contains "module load $b_module $a_module" spack -m load -r a -contains "module load $b_module $a_module" spack -m load --dependencies a +# test a variable MacOS clears and one it doesn't for recursive loads +contains "export LD_LIBRARY_PATH=$(spack -m location -i a)/lib:$(spack -m location -i b)/lib" spack -m load --sh a +contains "export LIBRARY_PATH=$(spack -m location -i a)/lib:$(spack -m location -i b)/lib" spack -m load --sh a +succeeds spack -m load --only dependencies a +succeeds spack -m load --only package a fails spack -m load d contains "usage: spack load " spack -m load -h contains "usage: spack load " spack -m load -h d contains "usage: spack load " spack -m load --help title 'Testing `spack unload`' -contains "module unload $b_module" spack -m unload b +spack -m load b a # setup +succeeds spack -m unload b +succeeds spack -m unload --all +spack -m unload --all # cleanup fails spack -m unload -l -contains "module unload -l --arg $b_module" spack -m unload -l --arg b fails spack -m unload d contains "usage: spack unload " spack -m unload -h contains "usage: spack unload " spack -m unload -h d diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 23a29ce6a2..207e9c4a80 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -40,6 +40,16 @@ ######################################################################## spack() { + # Store LD_LIBRARY_PATH variables from spack shell function + # This is necessary because MacOS System Integrity Protection clears + # (DY?)LD_LIBRARY_PATH variables on process start. + if [ -n "${LD_LIBRARY_PATH-}" ]; then + export SPACK_LD_LIBRARY_PATH=$LD_LIBRARY_PATH + fi + if [ -n "${DYLD_LIBRARY_PATH-}" ]; then + export SPACK_DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH + fi + # Zsh does not do word splitting by default, this enables it for this # function only if [ -n "${ZSH_VERSION:-}" ]; then @@ -141,41 +151,22 @@ spack() { return ;; "load"|"unload") - # Shift any other args for use off before parsing spec. - _sp_subcommand_args="" - _sp_module_args="" - while [ "${1#-}" != "${1}" ]; do - if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then - command spack $_sp_flags $_sp_subcommand $_sp_subcommand_args "$@" - return - elif [ "$1" = "-r" ] || [ "$1" = "--dependencies" ]; then - _sp_subcommand_args="$_sp_subcommand_args $1" - else - _sp_module_args="$_sp_module_args $1" - fi - shift - done - - # Here the user has run use or unuse with a spec. Find a matching - # spec using 'spack module find', then use the appropriate module - # tool's commands to add/remove the result from the environment. - # If spack module command comes back with an error, do nothing. - case $_sp_subcommand in - "load") - if _sp_full_spec=$(command spack $_sp_flags module tcl find $_sp_subcommand_args "$@"); then - module load $_sp_module_args $_sp_full_spec - else - $(exit 1) - fi - ;; - "unload") - if _sp_full_spec=$(command spack $_sp_flags module tcl find $_sp_subcommand_args "$@"); then - module unload $_sp_module_args $_sp_full_spec - else - $(exit 1) - fi - ;; - esac + # get --sh, --csh, --help, or -h arguments + # space is important for -h case to differentiate between `-h` + # argument and specs with "-h" in package name or variant settings + _a=" $@" + if [ "${_a#* --sh}" != "$_a" ] || \ + [ "${_a#* --csh}" != "$_a" ] || \ + [ "${_a#* -h}" != "$_a" ] || \ + [ "${_a#* --help}" != "$_a" ]; + then + # just execute the command if --sh or --csh are provided + # or if the -h or --help arguments are provided + command spack $_sp_flags $_sp_subcommand "$@" + else + eval $(command spack $_sp_flags $_sp_subcommand --sh "$@" || \ + echo "return 1") # return 1 if spack command fails + fi ;; *) command spack $_sp_flags $_sp_subcommand "$@" diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 0284e81113..79dcf8e559 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -817,7 +817,7 @@ _spack_fetch() { _spack_find() { if $list_options then - SPACK_COMPREPLY="-h --help --format --json -d --deps -p --paths --groups --no-groups -l --long -L --very-long -t --tags -c --show-concretized -f --show-flags --show-full-compiler -x --explicit -X --implicit -u --unknown -m --missing -v --variants -M --only-missing --deprecated --only-deprecated -N --namespace --start-date --end-date" + SPACK_COMPREPLY="-h --help --format --json -d --deps -p --paths --groups --no-groups -l --long -L --very-long -t --tags -c --show-concretized -f --show-flags --show-full-compiler -x --explicit -X --implicit -u --unknown -m --missing -v --variants --loaded -M --only-missing --deprecated --only-deprecated -N --namespace --start-date --end-date" else _installed_packages fi @@ -972,7 +972,7 @@ _spack_list() { _spack_load() { if $list_options then - SPACK_COMPREPLY="-h --help -r --dependencies" + SPACK_COMPREPLY="-h --help -r --dependencies --sh --csh --only" else _installed_packages fi @@ -1420,7 +1420,7 @@ _spack_uninstall() { _spack_unload() { if $list_options then - SPACK_COMPREPLY="-h --help" + SPACK_COMPREPLY="-h --help --sh --csh -a --all" else _installed_packages fi |