summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/docs/basic_usage.rst25
-rw-r--r--lib/spack/docs/packaging_guide.rst13
-rw-r--r--lib/spack/llnl/util/filesystem.py76
-rw-r--r--lib/spack/spack/build_environment.py86
-rw-r--r--lib/spack/spack/cmd/compiler.py12
-rw-r--r--lib/spack/spack/cmd/list.py50
-rw-r--r--lib/spack/spack/compilers/__init__.py18
-rw-r--r--lib/spack/spack/compilers/gcc.py4
-rw-r--r--lib/spack/spack/environment.py135
-rw-r--r--lib/spack/spack/hooks/sbang.py8
-rw-r--r--lib/spack/spack/test/data/sourceme_first.sh3
-rw-r--r--lib/spack/spack/test/data/sourceme_second.sh3
-rw-r--r--lib/spack/spack/test/environment.py62
13 files changed, 394 insertions, 101 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst
index 50c48b802b..ec193e767d 100644
--- a/lib/spack/docs/basic_usage.rst
+++ b/lib/spack/docs/basic_usage.rst
@@ -24,12 +24,29 @@ Spack can install:
.. command-output:: spack list
-The packages are listed by name in alphabetical order. You can also
-do wildcats searches using ``*``:
+The packages are listed by name in alphabetical order. If you specify a
+pattern to match, it will follow this set of rules. A pattern with no
+wildcards, ``*`` or ``?``, will be treated as though it started and ended with
+``*``, so ``util`` is equivalent to ``*util*``. A pattern with no capital
+letters will be treated as case-insensitive. You can also add the ``-i`` flag
+to specify a case insensitive search, or ``-d`` to search the description of
+the package in addition to the name. Some examples:
-.. command-output:: spack list m*
+All packages whose names contain "sql" case insensitive:
-.. command-output:: spack list *util*
+.. command-output:: spack list sql
+
+All packages whose names start with a capital M:
+
+.. command-output:: spack list 'M*'
+
+All packages whose names or descriptions contain Documentation:
+
+.. command-output:: spack list -d Documentation
+
+All packages whose names contain documentation case insensitive:
+
+.. command-output:: spack list -d documentation
.. _spack-info:
diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst
index 1f83f611b0..54b886310a 100644
--- a/lib/spack/docs/packaging_guide.rst
+++ b/lib/spack/docs/packaging_guide.rst
@@ -1950,6 +1950,19 @@ instead of hard-coding ``join_path(self.spec['mpi'].prefix.bin, 'mpicc')`` for
the reasons outlined above.
+Blas and Lapack libraries
+~~~~~~~~~~~~~~~~~~~~~~~~~
+Different packages provide implementation of ``Blas`` and ``Lapack`` routines.
+The names of the resulting static and/or shared libraries differ from package
+to package. In order to make ``install()`` method indifferent to the
+choice of ``Blas`` implementation, each package which provides it
+sets up ``self.spec.blas_shared_lib`` and ``self.spec.blas_static_lib `` to
+point to the shared and static ``Blas`` libraries, respectively. The same
+applies to packages which provide ``Lapack``. Package developers are advised to
+use these variables, for example ``spec['blas'].blas_shared_lib`` instead of
+hard-coding ``join_path(spec['blas'].prefix.lib, 'libopenblas.so')``.
+
+
Forking ``install()``
~~~~~~~~~~~~~~~~~~~~~
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index 6661a80f27..d72e8bae92 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -22,28 +22,28 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
-__all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree',
- 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp',
- 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file',
- 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink',
- 'set_executable', 'copy_mode', 'unset_executable_mode',
- 'remove_dead_links', 'remove_linked_tree', 'find_library_path',
- 'fix_darwin_install_name']
-
import os
import glob
-import sys
import re
import shutil
import stat
import errno
import getpass
from contextlib import contextmanager, closing
-from tempfile import NamedTemporaryFile
import subprocess
import llnl.util.tty as tty
-from spack.util.compression import ALLOWED_ARCHIVE_TYPES
+
+__all__ = ['set_install_permissions', 'install', 'install_tree',
+ 'traverse_tree',
+ 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp',
+ 'force_remove', 'join_path', 'ancestor', 'can_access',
+ 'filter_file',
+ 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink',
+ 'set_executable', 'copy_mode', 'unset_executable_mode',
+ 'remove_dead_links', 'remove_linked_tree', 'find_library_path',
+ 'fix_darwin_install_name', 'to_link_flags']
+
def filter_file(regex, repl, *filenames, **kwargs):
"""Like sed, but uses python regular expressions.
@@ -69,6 +69,7 @@ def filter_file(regex, repl, *filenames, **kwargs):
# Allow strings to use \1, \2, etc. for replacement, like sed
if not callable(repl):
unescaped = repl.replace(r'\\', '\\')
+
def replace_groups_with_groupid(m):
def groupid_to_group(x):
return m.group(int(x.group(1)))
@@ -157,9 +158,12 @@ def set_install_permissions(path):
def copy_mode(src, dest):
src_mode = os.stat(src).st_mode
dest_mode = os.stat(dest).st_mode
- if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR
- if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP
- if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH
+ if src_mode & stat.S_IXUSR:
+ dest_mode |= stat.S_IXUSR
+ if src_mode & stat.S_IXGRP:
+ dest_mode |= stat.S_IXGRP
+ if src_mode & stat.S_IXOTH:
+ dest_mode |= stat.S_IXOTH
os.chmod(dest, dest_mode)
@@ -224,9 +228,10 @@ def force_remove(*paths):
for path in paths:
try:
os.remove(path)
- except OSError, e:
+ except OSError:
pass
+
@contextmanager
def working_dir(dirname, **kwargs):
if kwargs.get('create', False):
@@ -240,7 +245,7 @@ def working_dir(dirname, **kwargs):
def touch(path):
"""Creates an empty file at the specified path."""
- with open(path, 'a') as file:
+ with open(path, 'a'):
os.utime(path, None)
@@ -253,7 +258,7 @@ def touchp(path):
def force_symlink(src, dest):
try:
os.symlink(src, dest)
- except OSError as e:
+ except OSError:
os.remove(dest)
os.symlink(src, dest)
@@ -275,7 +280,7 @@ def ancestor(dir, n=1):
def can_access(file_name):
"""True if we have read/write access to the file."""
- return os.access(file_name, os.R_OK|os.W_OK)
+ return os.access(file_name, os.R_OK | os.W_OK)
def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
@@ -343,13 +348,14 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
# Treat as a directory
if os.path.isdir(source_child) and (
- follow_links or not os.path.islink(source_child)):
+ follow_links or not os.path.islink(source_child)):
# When follow_nonexisting isn't set, don't descend into dirs
# in source that do not exist in dest
if follow_nonexisting or os.path.exists(dest_child):
- tuples = traverse_tree(source_root, dest_root, rel_child, **kwargs)
- for t in tuples: yield t
+ tuples = traverse_tree(source_root, dest_root, rel_child, **kwargs) # NOQA: ignore=E501
+ for t in tuples:
+ yield t
# Treat as a file.
elif not ignore(os.path.join(rel_path, f)):
@@ -379,6 +385,7 @@ def remove_dead_links(root):
if not os.path.exists(real_path):
os.unlink(path)
+
def remove_linked_tree(path):
"""
Removes a directory and its contents. If the directory is a
@@ -402,28 +409,41 @@ def fix_darwin_install_name(path):
Fix install name of dynamic libraries on Darwin to have full path.
There are two parts of this task:
(i) use install_name('-id',...) to change install name of a single lib;
- (ii) use install_name('-change',...) to change the cross linking between libs.
- The function assumes that all libraries are in one folder and currently won't
- follow subfolders.
+ (ii) use install_name('-change',...) to change the cross linking between
+ libs. The function assumes that all libraries are in one folder and
+ currently won't follow subfolders.
Args:
path: directory in which .dylib files are alocated
"""
- libs = glob.glob(join_path(path,"*.dylib"))
+ libs = glob.glob(join_path(path, "*.dylib"))
for lib in libs:
# fix install name first:
- subprocess.Popen(["install_name_tool", "-id",lib,lib], stdout=subprocess.PIPE).communicate()[0]
- long_deps = subprocess.Popen(["otool", "-L",lib], stdout=subprocess.PIPE).communicate()[0].split('\n')
+ subprocess.Popen(["install_name_tool", "-id", lib, lib], stdout=subprocess.PIPE).communicate()[0] # NOQA: ignore=E501
+ long_deps = subprocess.Popen(["otool", "-L", lib], stdout=subprocess.PIPE).communicate()[0].split('\n') # NOQA: ignore=E501
deps = [dep.partition(' ')[0][1::] for dep in long_deps[2:-1]]
# fix all dependencies:
for dep in deps:
for loc in libs:
if dep == os.path.basename(loc):
- subprocess.Popen(["install_name_tool", "-change",dep,loc,lib], stdout=subprocess.PIPE).communicate()[0]
+ subprocess.Popen(["install_name_tool", "-change", dep, loc, lib], stdout=subprocess.PIPE).communicate()[0] # NOQA: ignore=E501
break
+def to_link_flags(library):
+ """Transforms a path to a <library> into linking flags -L<dir> -l<name>.
+
+ Return:
+ A string of linking flags.
+ """
+ dir = os.path.dirname(library)
+ # Asume libXYZ.suffix
+ name = os.path.basename(library)[3:].split(".")[0]
+ res = '-L%s -l%s' % (dir, name)
+ return res
+
+
def find_library_path(libname, *paths):
"""Searches for a file called <libname> in each path.
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index c5efa97c7b..83468be8e6 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -75,13 +75,13 @@ SPACK_NO_PARALLEL_MAKE = 'SPACK_NO_PARALLEL_MAKE'
# set_build_environment_variables and used to pass parameters to
# Spack's compiler wrappers.
#
-SPACK_ENV_PATH = 'SPACK_ENV_PATH'
-SPACK_DEPENDENCIES = 'SPACK_DEPENDENCIES'
-SPACK_PREFIX = 'SPACK_PREFIX'
-SPACK_INSTALL = 'SPACK_INSTALL'
-SPACK_DEBUG = 'SPACK_DEBUG'
-SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC'
-SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR'
+SPACK_ENV_PATH = 'SPACK_ENV_PATH'
+SPACK_DEPENDENCIES = 'SPACK_DEPENDENCIES'
+SPACK_PREFIX = 'SPACK_PREFIX'
+SPACK_INSTALL = 'SPACK_INSTALL'
+SPACK_DEBUG = 'SPACK_DEBUG'
+SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC'
+SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR'
# Platform-specific library suffix.
@@ -98,6 +98,7 @@ class MakeExecutable(Executable):
Note that if the SPACK_NO_PARALLEL_MAKE env var is set it overrides
everything.
"""
+
def __init__(self, name, jobs):
super(MakeExecutable, self).__init__(name)
self.jobs = jobs
@@ -129,8 +130,13 @@ def load_module(mod):
text = modulecmd('show', mod, output=str, error=str).split()
for i, word in enumerate(text):
if word == 'conflict':
+<<<<<<< HEAD
+ exec(compile(modulecmd('unload', text[
+ i + 1], output=str, error=str), '<string>', 'exec'))
+=======
exec(compile(modulecmd('unload', text[i + 1], output=str,
error=str), '<string>', 'exec'))
+>>>>>>> aa86488fd9809ced16704e4bd4d607c89d6dda75
# Load the module now that there are no conflicts
load = modulecmd('load', mod, output=str, error=str)
exec(compile(load, '<string>', 'exec'))
@@ -188,20 +194,20 @@ def set_compiler_environment_variables(pkg, env):
# TODO : add additional kwargs for better diagnostics, like requestor,
# ttyout, ttyerr, etc.
link_dir = spack.build_env_path
- env.set('CC', join_path(link_dir, compiler.link_paths['cc']))
- env.set('CXX', join_path(link_dir, compiler.link_paths['cxx']))
- env.set('F77', join_path(link_dir, compiler.link_paths['f77']))
- env.set('FC', join_path(link_dir, compiler.link_paths['fc']))
# Set SPACK compiler variables so that our wrapper knows what to call
if compiler.cc:
env.set('SPACK_CC', compiler.cc)
+ env.set('CC', join_path(link_dir, compiler.link_paths['cc']))
if compiler.cxx:
env.set('SPACK_CXX', compiler.cxx)
+ env.set('CXX', join_path(link_dir, compiler.link_paths['cxx']))
if compiler.f77:
env.set('SPACK_F77', compiler.f77)
+ env.set('F77', join_path(link_dir, compiler.link_paths['f77']))
if compiler.fc:
env.set('SPACK_FC', compiler.fc)
+ env.set('FC', join_path(link_dir, compiler.link_paths['fc']))
# Set SPACK compiler rpath flags so that our wrapper knows what to use
env.set('SPACK_CC_RPATH_ARG', compiler.cc_rpath_arg)
@@ -238,8 +244,13 @@ def set_build_environment_variables(pkg, env):
# handled by putting one in the <build_env_path>/case-insensitive
# directory. Add that to the path too.
env_paths = []
+<<<<<<< HEAD
+ compiler_specific = join_path(spack.build_env_path, pkg.compiler.name)
+ for item in [spack.build_env_path, compiler_specific]:
+=======
for item in [spack.build_env_path, join_path(spack.build_env_path,
pkg.compiler.name)]:
+>>>>>>> aa86488fd9809ced16704e4bd4d607c89d6dda75
env_paths.append(item)
ci = join_path(item, 'case-insensitive')
if os.path.isdir(ci):
@@ -269,8 +280,13 @@ def set_build_environment_variables(pkg, env):
env.unset('DYLD_LIBRARY_PATH')
# Add bin directories from dependencies to the PATH for the build.
+<<<<<<< HEAD
+ bin_dirs = reversed(
+ filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes]))
+=======
bin_dirs = reversed(filter(os.path.isdir,
['%s/bin' % prefix for prefix in dep_prefixes]))
+>>>>>>> aa86488fd9809ced16704e4bd4d607c89d6dda75
for item in bin_dirs:
env.prepend_path('PATH', item)
@@ -285,6 +301,10 @@ def set_build_environment_variables(pkg, env):
for directory in ('lib', 'lib64', 'share'):
pcdir = join_path(pre, directory, 'pkgconfig')
if os.path.isdir(pcdir):
+<<<<<<< HEAD
+ # pkg_config_dirs.append(pcdir)
+=======
+>>>>>>> aa86488fd9809ced16704e4bd4d607c89d6dda75
env.prepend_path('PKG_CONFIG_PATH', pcdir)
if pkg.spec.architecture.target.module_name:
@@ -308,7 +328,7 @@ def set_module_variables_for_package(pkg, module):
m.make_jobs = jobs
# TODO: make these build deps that can be installed if not found.
- m.make = MakeExecutable('make', jobs)
+ m.make = MakeExecutable('make', jobs)
m.gmake = MakeExecutable('gmake', jobs)
m.scons = MakeExecutable('scons', jobs)
@@ -339,29 +359,29 @@ def set_module_variables_for_package(pkg, module):
# Put spack compiler paths in module scope.
link_dir = spack.build_env_path
- m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc'])
+ m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc'])
m.spack_cxx = join_path(link_dir, pkg.compiler.link_paths['cxx'])
m.spack_f77 = join_path(link_dir, pkg.compiler.link_paths['f77'])
- m.spack_fc = join_path(link_dir, pkg.compiler.link_paths['fc'])
+ m.spack_fc = join_path(link_dir, pkg.compiler.link_paths['fc'])
# Emulate some shell commands for convenience
- m.pwd = os.getcwd
- m.cd = os.chdir
- m.mkdir = os.mkdir
- m.makedirs = os.makedirs
- m.remove = os.remove
- m.removedirs = os.removedirs
- m.symlink = os.symlink
-
- m.mkdirp = mkdirp
- m.install = install
+ m.pwd = os.getcwd
+ m.cd = os.chdir
+ m.mkdir = os.mkdir
+ m.makedirs = os.makedirs
+ m.remove = os.remove
+ m.removedirs = os.removedirs
+ m.symlink = os.symlink
+
+ m.mkdirp = mkdirp
+ m.install = install
m.install_tree = install_tree
- m.rmtree = shutil.rmtree
- m.move = shutil.move
+ m.rmtree = shutil.rmtree
+ m.move = shutil.move
# Useful directories within the prefix are encapsulated in
# a Prefix object.
- m.prefix = pkg.prefix
+ m.prefix = pkg.prefix
# Platform-specific library suffix.
m.dso_suffix = dso_suffix
@@ -382,7 +402,13 @@ def get_rpaths(pkg):
def parent_class_modules(cls):
+<<<<<<< HEAD
+ """
+ Get list of super class modules that are all descend from spack.Package
+ """
+=======
"""Get list of super class modules that all descend from spack.Package"""
+>>>>>>> aa86488fd9809ced16704e4bd4d607c89d6dda75
if not issubclass(cls, spack.Package) or issubclass(spack.Package, cls):
return []
result = []
@@ -405,7 +431,7 @@ def load_external_modules(pkg):
def setup_package(pkg):
"""Execute all environment setup routines."""
spack_env = EnvironmentModifications()
- run_env = EnvironmentModifications()
+ run_env = EnvironmentModifications()
# Before proceeding, ensure that specs and packages are consistent
#
@@ -510,7 +536,9 @@ def fork(pkg, function):
# message. Just make the parent exit with an error code.
pid, returncode = os.waitpid(pid, 0)
if returncode != 0:
- raise InstallError("Installation process had nonzero exit code.")
+ message = "Installation process had nonzero exit code : {code}"
+ strcode = str(returncode)
+ raise InstallError(message.format(code=strcode))
class InstallError(spack.error.SpackError):
diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py
index c95045ef85..c325628ebf 100644
--- a/lib/spack/spack/cmd/compiler.py
+++ b/lib/spack/spack/cmd/compiler.py
@@ -76,10 +76,16 @@ def compiler_find(args):
if not paths:
paths = get_path('PATH')
- compilers = [c for c in spack.compilers.find_compilers(*args.add_paths)
- if c.spec not in spack.compilers.all_compilers(scope=args.scope)]
+ # Don't initialize compilers config via compilers.get_compiler_config.
+ # Just let compiler_find do the
+ # entire process and return an empty config from all_compilers
+ # Default for any other process is init_config=True
+ compilers = [c for c in spack.compilers.find_compilers(*paths)
+ if c.spec not in spack.compilers.all_compilers(
+ scope=args.scope, init_config=False)]
if compilers:
- spack.compilers.add_compilers_to_config(compilers, scope=args.scope)
+ spack.compilers.add_compilers_to_config(compilers, scope=args.scope,
+ init_config=False)
n = len(compilers)
s = 's' if n > 1 else ''
filename = spack.config.get_config_filename(args.scope, 'compilers')
diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py
index 1e3699cee0..c921efd1bd 100644
--- a/lib/spack/spack/cmd/list.py
+++ b/lib/spack/spack/cmd/list.py
@@ -29,36 +29,62 @@ from llnl.util.tty.colify import colify
import spack
import fnmatch
+import re
+
+description = "List available spack packages"
-description ="List available spack packages"
def setup_parser(subparser):
subparser.add_argument(
'filter', nargs=argparse.REMAINDER,
help='Optional glob patterns to filter results.')
subparser.add_argument(
- '-i', '--insensitive', action='store_true', default=False,
- help='Filtering will be case insensitive.')
+ '-s', '--sensitive', action='store_true', default=False,
+ help='Use case-sensitive filtering. Default is case sensitive, '
+ 'unless the query contains a capital letter.')
+ subparser.add_argument(
+ '-d', '--search-description', action='store_true', default=False,
+ help='Filtering will also search the description for a match.')
def list(parser, args):
# Start with all package names.
- pkgs = spack.repo.all_package_names()
+ pkgs = set(spack.repo.all_package_names())
# filter if a filter arg was provided
if args.filter:
- def match(p, f):
- if args.insensitive:
- p = p.lower()
- f = f.lower()
- return fnmatch.fnmatchcase(p, f)
- pkgs = [p for p in pkgs if any(match(p, f) for f in args.filter)]
+ res = []
+ for f in args.filter:
+ if '*' not in f and '?' not in f:
+ r = fnmatch.translate('*' + f + '*')
+ else:
+ r = fnmatch.translate(f)
+
+ re_flags = re.I
+ if any(l.isupper for l in f) or args.sensitive:
+ re_flags = 0
+ rc = re.compile(r, flags=re_flags)
+ res.append(rc)
+
+ if args.search_description:
+ def match(p, f):
+ if f.match(p):
+ return True
+
+ pkg = spack.repo.get(p)
+ if pkg.__doc__:
+ return f.match(pkg.__doc__)
+ return False
+ else:
+ def match(p, f):
+ return f.match(p)
+ pkgs = [p for p in pkgs if any(match(p, f) for f in res)]
# sort before displaying.
- sorted_packages = sorted(pkgs, key=lambda s:s.lower())
+ sorted_packages = sorted(pkgs, key=lambda s: s.lower())
# Print all the package names in columns
- indent=0
+ indent = 0
if sys.stdout.isatty():
tty.msg("%d packages." % len(sorted_packages))
colify(sorted_packages, indent=indent)
diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py
index ae72b743b2..a70d42982f 100644
--- a/lib/spack/spack/compilers/__init__.py
+++ b/lib/spack/spack/compilers/__init__.py
@@ -83,7 +83,7 @@ def _to_dict(compiler):
return {'compiler': d}
-def get_compiler_config(scope=None):
+def get_compiler_config(scope=None, init_config=True):
"""Return the compiler configuration for the specified architecture.
"""
def init_compiler_config():
@@ -93,12 +93,12 @@ def get_compiler_config(scope=None):
for compiler in compilers:
compilers_dict.append(_to_dict(compiler))
spack.config.update_config('compilers', compilers_dict, scope=scope)
-
+
config = spack.config.get_config('compilers', scope=scope)
# Update the configuration if there are currently no compilers
# configured. Avoid updating automatically if there ARE site
# compilers configured but no user ones.
- if not config:
+ if not config and init_config:
if scope is None:
# We know no compilers were configured in any scope.
init_compiler_config()
@@ -117,14 +117,14 @@ def get_compiler_config(scope=None):
return [] # Return empty list which we will later append to.
-def add_compilers_to_config(compilers, scope=None):
+def add_compilers_to_config(compilers, scope=None, init_config=True):
"""Add compilers to the config for the specified architecture.
Arguments:
- compilers: a list of Compiler objects.
- scope: configuration scope to modify.
"""
- compiler_config = get_compiler_config(scope)
+ compiler_config = get_compiler_config(scope, init_config)
for compiler in compilers:
compiler_config.append(_to_dict(compiler))
global _cache_config_file
@@ -153,23 +153,23 @@ def remove_compiler_from_config(compiler_spec, scope=None):
spack.config.update_config('compilers', filtered_compiler_config, scope)
-def all_compilers_config(scope=None):
+def all_compilers_config(scope=None, init_config=True):
"""Return a set of specs for all the compiler versions currently
available to build with. These are instances of CompilerSpec.
"""
# Get compilers for this architecture.
global _cache_config_file #Create a cache of the config file so we don't load all the time.
if not _cache_config_file:
- _cache_config_file = get_compiler_config(scope)
+ _cache_config_file = get_compiler_config(scope, init_config)
return _cache_config_file
else:
return _cache_config_file
-def all_compilers(scope=None):
+def all_compilers(scope=None, init_config=True):
# Return compiler specs from the merged config.
return [spack.spec.CompilerSpec(s['compiler']['spec'])
- for s in all_compilers_config(scope)]
+ for s in all_compilers_config(scope, init_config)]
def default_compiler():
diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py
index 3f552eaece..2fae6688db 100644
--- a/lib/spack/spack/compilers/gcc.py
+++ b/lib/spack/spack/compilers/gcc.py
@@ -77,9 +77,9 @@ class Gcc(Compiler):
return get_compiler_version(
fc, '-dumpversion',
# older gfortran versions don't have simple dumpversion output.
- r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)', module)
+ r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)')
@classmethod
def f77_version(cls, f77):
- return cls.fc_version(f77, module)
+ return cls.fc_version(f77)
diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py
index af642dcc9b..30c6228ca4 100644
--- a/lib/spack/spack/environment.py
+++ b/lib/spack/spack/environment.py
@@ -1,4 +1,4 @@
-##############################################################################
+#
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
@@ -21,14 +21,17 @@
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-##############################################################################
+#
import collections
import inspect
+import json
import os
import os.path
+import subprocess
class NameModifier(object):
+
def __init__(self, name, **kwargs):
self.name = name
self.args = {'name': name}
@@ -36,6 +39,7 @@ class NameModifier(object):
class NameValueModifier(object):
+
def __init__(self, name, value, **kwargs):
self.name = name
self.value = value
@@ -45,23 +49,27 @@ class NameValueModifier(object):
class SetEnv(NameValueModifier):
+
def execute(self):
os.environ[self.name] = str(self.value)
class UnsetEnv(NameModifier):
+
def execute(self):
# Avoid throwing if the variable was not set
os.environ.pop(self.name, None)
class SetPath(NameValueModifier):
+
def execute(self):
string_path = concatenate_paths(self.value, separator=self.separator)
os.environ[self.name] = string_path
class AppendPath(NameValueModifier):
+
def execute(self):
environment_value = os.environ.get(self.name, '')
directories = environment_value.split(
@@ -71,6 +79,7 @@ class AppendPath(NameValueModifier):
class PrependPath(NameValueModifier):
+
def execute(self):
environment_value = os.environ.get(self.name, '')
directories = environment_value.split(
@@ -80,6 +89,7 @@ class PrependPath(NameValueModifier):
class RemovePath(NameValueModifier):
+
def execute(self):
environment_value = os.environ.get(self.name, '')
directories = environment_value.split(
@@ -90,6 +100,7 @@ class RemovePath(NameValueModifier):
class EnvironmentModifications(object):
+
"""
Keeps track of requests to modify the current environment.
@@ -240,6 +251,126 @@ class EnvironmentModifications(object):
for x in actions:
x.execute()
+ @staticmethod
+ def from_sourcing_files(*args, **kwargs):
+ """
+ Creates an instance of EnvironmentModifications that, if executed,
+ has the same effect on the environment as sourcing the files passed as
+ parameters
+
+ Args:
+ *args: list of files to be sourced
+
+ Returns:
+ instance of EnvironmentModifications
+ """
+ env = EnvironmentModifications()
+ # Check if the files are actually there
+ if not all(os.path.isfile(file) for file in args):
+ raise RuntimeError('trying to source non-existing files')
+ # Relevant kwd parameters and formats
+ info = dict(kwargs)
+ info.setdefault('shell', '/bin/bash')
+ info.setdefault('shell_options', '-c')
+ info.setdefault('source_command', 'source')
+ info.setdefault('suppress_output', '&> /dev/null')
+ info.setdefault('concatenate_on_success', '&&')
+
+ shell = '{shell}'.format(**info)
+ shell_options = '{shell_options}'.format(**info)
+ source_file = '{source_command} {file} {concatenate_on_success}'
+ dump_environment = 'python -c "import os, json; print json.dumps(dict(os.environ))"' # NOQA: ignore=E501
+ # Construct the command that will be executed
+ command = [source_file.format(file=file, **info) for file in args]
+ command.append(dump_environment)
+ command = ' '.join(command)
+ command = [
+ shell,
+ shell_options,
+ command
+ ]
+
+ # Try to source all the files,
+ proc = subprocess.Popen(
+ command, stdout=subprocess.PIPE, env=os.environ)
+ proc.wait()
+ if proc.returncode != 0:
+ raise RuntimeError('sourcing files returned a non-zero exit code')
+ output = ''.join([line for line in proc.stdout])
+ # Construct a dictionary with all the variables in the new environment
+ after_source_env = dict(json.loads(output))
+ this_environment = dict(os.environ)
+
+ # Filter variables that are not related to sourcing a file
+ to_be_filtered = 'SHLVL', '_', 'PWD', 'OLDPWD'
+ for d in after_source_env, this_environment:
+ for name in to_be_filtered:
+ d.pop(name, None)
+
+ # Fill the EnvironmentModifications instance
+
+ # New variables
+ new_variables = set(after_source_env) - set(this_environment)
+ for x in new_variables:
+ env.set(x, after_source_env[x])
+ # Variables that have been unset
+ unset_variables = set(this_environment) - set(after_source_env)
+ for x in unset_variables:
+ env.unset(x)
+ # Variables that have been modified
+ common_variables = set(this_environment).intersection(set(after_source_env)) # NOQA: ignore=E501
+ modified_variables = [x for x in common_variables if this_environment[x] != after_source_env[x]] # NOQA: ignore=E501
+
+ def return_separator_if_any(first_value, second_value):
+ separators = ':', ';'
+ for separator in separators:
+ if separator in first_value and separator in second_value:
+ return separator
+ return None
+
+ for x in modified_variables:
+ current = this_environment[x]
+ modified = after_source_env[x]
+ sep = return_separator_if_any(current, modified)
+ if sep is None:
+ # We just need to set the variable to the new value
+ env.set(x, after_source_env[x])
+ else:
+ current_list = current.split(sep)
+ modified_list = modified.split(sep)
+ # Paths that have been removed
+ remove_list = [
+ ii for ii in current_list if ii not in modified_list]
+ # Check that nothing has been added in the middle of vurrent
+ # list
+ remaining_list = [
+ ii for ii in current_list if ii in modified_list]
+ start = modified_list.index(remaining_list[0])
+ end = modified_list.index(remaining_list[-1])
+ search = sep.join(modified_list[start:end + 1])
+ if search not in current:
+ # We just need to set the variable to the new value
+ env.set(x, after_source_env[x])
+ break
+ else:
+ try:
+ prepend_list = modified_list[:start]
+ except KeyError:
+ prepend_list = []
+ try:
+ append_list = modified_list[end + 1:]
+ except KeyError:
+ append_list = []
+
+ for item in remove_list:
+ env.remove_path(x, item)
+ for item in append_list:
+ env.append_path(x, item)
+ for item in prepend_list:
+ env.prepend_path(x, item)
+
+ return env
+
def concatenate_paths(paths, separator=':'):
"""
diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py
index 83d67ea225..cb0ad42b14 100644
--- a/lib/spack/spack/hooks/sbang.py
+++ b/lib/spack/spack/hooks/sbang.py
@@ -24,7 +24,6 @@
##############################################################################
import os
-from llnl.util.filesystem import *
import llnl.util.tty as tty
import spack
@@ -34,6 +33,7 @@ import spack.modules
# here, as it is the shortest I could find on a modern OS.
shebang_limit = 127
+
def shebang_too_long(path):
"""Detects whether a file has a shebang line that is too long."""
with open(path, 'r') as script:
@@ -57,16 +57,10 @@ def filter_shebang(path):
if original.startswith(new_sbang_line):
return
- backup = path + ".shebang.bak"
- os.rename(path, backup)
-
with open(path, 'w') as new_file:
new_file.write(new_sbang_line)
new_file.write(original)
- copy_mode(backup, path)
- unset_executable_mode(backup)
-
tty.warn("Patched overly long shebang in %s" % path)
diff --git a/lib/spack/spack/test/data/sourceme_first.sh b/lib/spack/spack/test/data/sourceme_first.sh
new file mode 100644
index 0000000000..800f639ac8
--- /dev/null
+++ b/lib/spack/spack/test/data/sourceme_first.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+export NEW_VAR='new'
+export UNSET_ME='overridden'
diff --git a/lib/spack/spack/test/data/sourceme_second.sh b/lib/spack/spack/test/data/sourceme_second.sh
new file mode 100644
index 0000000000..9955a0e6d6
--- /dev/null
+++ b/lib/spack/spack/test/data/sourceme_second.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+export PATH_LIST='/path/first:/path/second:/path/fourth'
+unset EMPTY_PATH_LIST \ No newline at end of file
diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py
index a0d959db2f..f3644cb0b7 100644
--- a/lib/spack/spack/test/environment.py
+++ b/lib/spack/spack/test/environment.py
@@ -24,16 +24,21 @@
##############################################################################
import unittest
import os
-import copy
+
+from spack import spack_root
+from llnl.util.filesystem import join_path
from spack.environment import EnvironmentModifications
+from spack.environment import SetEnv, UnsetEnv
+from spack.environment import RemovePath, PrependPath, AppendPath
class EnvironmentTest(unittest.TestCase):
+
def setUp(self):
os.environ['UNSET_ME'] = 'foo'
os.environ['EMPTY_PATH_LIST'] = ''
os.environ['PATH_LIST'] = '/path/second:/path/third'
- os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g'
+ os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' # NOQA: ignore=E501
def tearDown(self):
pass
@@ -77,9 +82,18 @@ class EnvironmentTest(unittest.TestCase):
env.remove_path('REMOVE_PATH_LIST', '/duplicate/')
env.apply_modifications()
- self.assertEqual('/path/first:/path/second:/path/third:/path/last', os.environ['PATH_LIST'])
- self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST'])
- self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST'])
+ self.assertEqual(
+ '/path/first:/path/second:/path/third:/path/last',
+ os.environ['PATH_LIST']
+ )
+ self.assertEqual(
+ '/path/first:/path/middle:/path/last',
+ os.environ['EMPTY_PATH_LIST']
+ )
+ self.assertEqual(
+ '/path/first:/path/middle:/path/last',
+ os.environ['NEWLY_CREATED_PATH_LIST']
+ )
self.assertEqual('/a/b:/a/c:/a/d:/f/g', os.environ['REMOVE_PATH_LIST'])
def test_extra_arguments(self):
@@ -98,3 +112,41 @@ class EnvironmentTest(unittest.TestCase):
self.assertEqual(len(copy_construct), 2)
for x, y in zip(env, copy_construct):
assert x is y
+
+ def test_source_files(self):
+ datadir = join_path(spack_root, 'lib', 'spack',
+ 'spack', 'test', 'data')
+ files = [
+ join_path(datadir, 'sourceme_first.sh'),
+ join_path(datadir, 'sourceme_second.sh')
+ ]
+ env = EnvironmentModifications.from_sourcing_files(*files)
+ modifications = env.group_by_name()
+
+ self.assertEqual(len(modifications), 4)
+ # Set new variables
+ self.assertEqual(len(modifications['NEW_VAR']), 1)
+ self.assertTrue(isinstance(modifications['NEW_VAR'][0], SetEnv))
+ self.assertEqual(modifications['NEW_VAR'][0].value, 'new')
+ # Unset variables
+ self.assertEqual(len(modifications['EMPTY_PATH_LIST']), 1)
+ self.assertTrue(isinstance(
+ modifications['EMPTY_PATH_LIST'][0], UnsetEnv))
+ # Modified variables
+ self.assertEqual(len(modifications['UNSET_ME']), 1)
+ self.assertTrue(isinstance(modifications['UNSET_ME'][0], SetEnv))
+ self.assertEqual(modifications['UNSET_ME'][0].value, 'overridden')
+
+ self.assertEqual(len(modifications['PATH_LIST']), 3)
+ self.assertTrue(
+ isinstance(modifications['PATH_LIST'][0], RemovePath)
+ )
+ self.assertEqual(modifications['PATH_LIST'][0].value, '/path/third')
+ self.assertTrue(
+ isinstance(modifications['PATH_LIST'][1], AppendPath)
+ )
+ self.assertEqual(modifications['PATH_LIST'][1].value, '/path/fourth')
+ self.assertTrue(
+ isinstance(modifications['PATH_LIST'][2], PrependPath)
+ )
+ self.assertEqual(modifications['PATH_LIST'][2].value, '/path/first')
generated by cgit v1.2.3-60-g2f50 (git 2.42.0) at 2024-07-16 20:14:21 +0000