summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/docs/configuration.rst7
-rw-r--r--lib/spack/llnl/util/filesystem.py26
-rw-r--r--lib/spack/spack/cmd/repo.py2
-rw-r--r--lib/spack/spack/config.py2
-rw-r--r--lib/spack/spack/database.py6
-rw-r--r--lib/spack/spack/modules.py21
-rw-r--r--lib/spack/spack/repository.py2
-rw-r--r--lib/spack/spack/test/modules.py172
8 files changed, 207 insertions, 31 deletions
diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst
index c613071c65..a6f876b2aa 100644
--- a/lib/spack/docs/configuration.rst
+++ b/lib/spack/docs/configuration.rst
@@ -140,7 +140,6 @@ Here's an example packages.yaml file that sets preferred packages:
packages:
dyninst:
compiler: [gcc@4.9]
- variants: +debug
gperftools:
version: [2.2, 2.4, 2.3]
all:
@@ -150,8 +149,8 @@ Here's an example packages.yaml file that sets preferred packages:
At a high level, this example is specifying how packages should be
-concretized. The dyninst package should prefer using gcc 4.9 and
-be built with debug options. The gperftools package should prefer version
+concretized. The dyninst package should prefer using gcc 4.9.
+The gperftools package should prefer version
2.2 over 2.4. Every package on the system should prefer mvapich for
its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9).
These options are used to fill in implicit defaults. Any of them can be overwritten
@@ -160,7 +159,7 @@ on the command line if explicitly requested.
Each packages.yaml file begins with the string ``packages:`` and
package names are specified on the next level. The special string ``all``
applies settings to each package. Underneath each package name is
-one or more components: ``compiler``, ``variants``, ``version``,
+one or more components: ``compiler``, ``version``,
or ``providers``. Each component has an ordered list of spec
``constraints``, with earlier entries in the list being preferred over
later entries.
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index d72e8bae92..e800c6717a 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -48,7 +48,7 @@ __all__ = ['set_install_permissions', 'install', 'install_tree',
def filter_file(regex, repl, *filenames, **kwargs):
"""Like sed, but uses python regular expressions.
- Filters every line of file through regex and replaces the file
+ Filters every line of each file through regex and replaces the file
with a filtered version. Preserves mode of filtered files.
As with re.sub, ``repl`` can be either a string or a callable.
@@ -59,7 +59,7 @@ def filter_file(regex, repl, *filenames, **kwargs):
Keyword Options:
string[=False] If True, treat regex as a plain string.
- backup[=True] Make a backup files suffixed with ~
+ backup[=True] Make backup file(s) suffixed with ~
ignore_absent[=False] Ignore any files that don't exist.
"""
string = kwargs.get('string', False)
@@ -80,26 +80,26 @@ def filter_file(regex, repl, *filenames, **kwargs):
regex = re.escape(regex)
for filename in filenames:
- backup = filename + "~"
+ backup_filename = filename + "~"
if ignore_absent and not os.path.exists(filename):
continue
- shutil.copy(filename, backup)
+ shutil.copy(filename, backup_filename)
try:
- with closing(open(backup)) as infile:
+ with closing(open(backup_filename)) as infile:
with closing(open(filename, 'w')) as outfile:
for line in infile:
foo = re.sub(regex, repl, line)
outfile.write(foo)
except:
# clean up the original file on failure.
- shutil.move(backup, filename)
+ shutil.move(backup_filename, filename)
raise
finally:
if not backup:
- shutil.rmtree(backup, ignore_errors=True)
+ os.remove(backup_filename)
class FileFilter(object):
@@ -114,7 +114,7 @@ class FileFilter(object):
def change_sed_delimiter(old_delim, new_delim, *filenames):
"""Find all sed search/replace commands and change the delimiter.
e.g., if the file contains seds that look like 's///', you can
- call change_sed_delimeter('/', '@', file) to change the
+ call change_sed_delimiter('/', '@', file) to change the
delimiter to '@'.
NOTE that this routine will fail if the delimiter is ' or ".
@@ -179,7 +179,7 @@ def install(src, dest):
"""Manually install a file to a particular location."""
tty.debug("Installing %s to %s" % (src, dest))
- # Expand dsst to its eventual full path if it is a directory.
+ # Expand dest to its eventual full path if it is a directory.
if os.path.isdir(dest):
dest = join_path(dest, os.path.basename(src))
@@ -219,7 +219,7 @@ def mkdirp(*paths):
if not os.path.exists(path):
os.makedirs(path)
elif not os.path.isdir(path):
- raise OSError(errno.EEXIST, "File alredy exists", path)
+ raise OSError(errno.EEXIST, "File already exists", path)
def force_remove(*paths):
@@ -309,7 +309,7 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
Optional args:
- order=[pre|post] -- Whether to do pre- or post-order traveral.
+ order=[pre|post] -- Whether to do pre- or post-order traversal.
ignore=<predicate> -- Predicate indicating which files to ignore.
@@ -414,7 +414,7 @@ def fix_darwin_install_name(path):
currently won't follow subfolders.
Args:
- path: directory in which .dylib files are alocated
+ path: directory in which .dylib files are located
"""
libs = glob.glob(join_path(path, "*.dylib"))
@@ -438,7 +438,7 @@ def to_link_flags(library):
A string of linking flags.
"""
dir = os.path.dirname(library)
- # Asume libXYZ.suffix
+ # Assume libXYZ.suffix
name = os.path.basename(library)[3:].split(".")[0]
res = '-L%s -l%s' % (dir, name)
return res
diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py
index 399237b169..cbd8f4784e 100644
--- a/lib/spack/spack/cmd/repo.py
+++ b/lib/spack/spack/cmd/repo.py
@@ -26,7 +26,7 @@ import os
import re
import shutil
-from external import argparse
+import argparse
import llnl.util.tty as tty
from llnl.util.filesystem import join_path, mkdirp
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index db0787edc6..84179e1469 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -307,6 +307,8 @@ section_schemas = {
'autoload': {'$ref': '#/definitions/dependency_selection'},
'prerequisites': {'$ref': '#/definitions/dependency_selection'},
'conflict': {'$ref': '#/definitions/array_of_strings'},
+ 'load': {'$ref': '#/definitions/array_of_strings'},
+ 'suffixes': {'$ref': '#/definitions/dictionary_of_strings'},
'environment': {
'type': 'object',
'default': {},
diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py
index f941346bb1..a4bbff3d5a 100644
--- a/lib/spack/spack/database.py
+++ b/lib/spack/spack/database.py
@@ -631,11 +631,13 @@ class WriteTransaction(_Transaction):
class CorruptDatabaseError(SpackError):
def __init__(self, path, msg=''):
super(CorruptDatabaseError, self).__init__(
- "Spack database is corrupt: %s. %s" % (path, msg))
+ "Spack database is corrupt: %s. %s." + \
+ "Try running `spack reindex` to fix." % (path, msg))
class InvalidDatabaseVersionError(SpackError):
def __init__(self, expected, found):
super(InvalidDatabaseVersionError, self).__init__(
- "Expected database version %s but found version %s" %
+ "Expected database version %s but found version %s." + \
+ "Try running `spack reindex` to fix." %
(expected, found))
diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py
index d2b819e80a..fb58be6ce0 100644
--- a/lib/spack/spack/modules.py
+++ b/lib/spack/spack/modules.py
@@ -285,11 +285,18 @@ class EnvModule(object):
naming_tokens = self.tokens
naming_scheme = self.naming_scheme
name = naming_scheme.format(**naming_tokens)
- name += '-' + self.spec.dag_hash(
- ) # Always append the hash to make the module file unique
# Not everybody is working on linux...
parts = name.split('/')
name = join_path(*parts)
+ # Add optional suffixes based on constraints
+ configuration, _ = parse_config_options(self)
+ suffixes = [name]
+ for constraint, suffix in configuration.get('suffixes', {}).items():
+ if constraint in self.spec:
+ suffixes.append(suffix)
+ # Always append the hash to make the module file unique
+ suffixes.append(self.spec.dag_hash())
+ name = '-'.join(suffixes)
return name
@property
@@ -381,6 +388,8 @@ class EnvModule(object):
for x in filter_blacklisted(
module_configuration.pop('autoload', []), self.name):
module_file_content += self.autoload(x)
+ for x in module_configuration.pop('load', []):
+ module_file_content += self.autoload(x)
for x in filter_blacklisted(
module_configuration.pop('prerequisites', []), self.name):
module_file_content += self.prerequisite(x)
@@ -402,8 +411,12 @@ class EnvModule(object):
return tuple()
def autoload(self, spec):
- m = type(self)(spec)
- return self.autoload_format.format(module_file=m.use_name)
+ if not isinstance(spec, str):
+ m = type(self)(spec)
+ module_file = m.use_name
+ else:
+ module_file = spec
+ return self.autoload_format.format(module_file=module_file)
def prerequisite(self, spec):
m = type(self)(spec)
diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py
index 70134964ad..2c160a5f45 100644
--- a/lib/spack/spack/repository.py
+++ b/lib/spack/spack/repository.py
@@ -30,7 +30,7 @@ import imp
import re
import traceback
from bisect import bisect_left
-from external import yaml
+import yaml
import llnl.util.tty as tty
from llnl.util.filesystem import *
diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py
index 582e067860..443e4d13a6 100644
--- a/lib/spack/spack/test/modules.py
+++ b/lib/spack/spack/test/modules.py
@@ -27,6 +27,7 @@ from contextlib import contextmanager
import StringIO
import spack.modules
+import unittest
from spack.test.mock_packages_test import MockPackagesTest
FILE_REGISTRY = collections.defaultdict(StringIO.StringIO)
@@ -67,6 +68,24 @@ configuration_autoload_all = {
}
}
+configuration_prerequisites_direct = {
+ 'enable': ['tcl'],
+ 'tcl': {
+ 'all': {
+ 'prerequisites': 'direct'
+ }
+ }
+}
+
+configuration_prerequisites_all = {
+ 'enable': ['tcl'],
+ 'tcl': {
+ 'all': {
+ 'prerequisites': 'all'
+ }
+ }
+}
+
configuration_alter_environment = {
'enable': ['tcl'],
'tcl': {
@@ -74,8 +93,13 @@ configuration_alter_environment = {
'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']}
},
'platform=test target=x86_64': {
- 'environment': {'set': {'FOO': 'foo'},
- 'unset': ['BAR']}
+ 'environment': {
+ 'set': {'FOO': 'foo'},
+ 'unset': ['BAR']
+ }
+ },
+ 'platform=test target=x86_32': {
+ 'load': ['foo/bar']
}
}
}
@@ -83,7 +107,8 @@ configuration_alter_environment = {
configuration_blacklist = {
'enable': ['tcl'],
'tcl': {
- 'blacklist': ['callpath'],
+ 'whitelist': ['zmpi'],
+ 'blacklist': ['callpath', 'mpi'],
'all': {
'autoload': 'direct'
}
@@ -100,8 +125,68 @@ configuration_conflicts = {
}
}
+configuration_wrong_conflicts = {
+ 'enable': ['tcl'],
+ 'tcl': {
+ 'naming_scheme': '{name}/{version}-{compiler.name}',
+ 'all': {
+ 'conflict': ['{name}/{compiler.name}']
+ }
+ }
+}
+
+configuration_suffix = {
+ 'enable': ['tcl'],
+ 'tcl': {
+ 'mpileaks': {
+ 'suffixes': {
+ '+debug': 'foo',
+ '~debug': 'bar'
+ }
+ }
+ }
+}
+
+
+class HelperFunctionsTests(unittest.TestCase):
+
+ def test_update_dictionary_extending_list(self):
+ target = {
+ 'foo': {
+ 'a': 1,
+ 'b': 2,
+ 'd': 4
+ },
+ 'bar': [1, 2, 4],
+ 'baz': 'foobar'
+ }
+ update = {
+ 'foo': {
+ 'c': 3,
+ },
+ 'bar': [3],
+ 'baz': 'foobaz',
+ 'newkey': {
+ 'd': 4
+ }
+ }
+ spack.modules.update_dictionary_extending_lists(target, update)
+ self.assertTrue(len(target) == 4)
+ self.assertTrue(len(target['foo']) == 4)
+ self.assertTrue(len(target['bar']) == 4)
+ self.assertEqual(target['baz'], 'foobaz')
+
+ def test_inspect_path(self):
+ env = spack.modules.inspect_path('/usr')
+ names = [item.name for item in env]
+ self.assertTrue('PATH' in names)
+ self.assertTrue('LIBRARY_PATH' in names)
+ self.assertTrue('LD_LIBRARY_PATH' in names)
+ self.assertTrue('CPATH' in names)
+
class TclTests(MockPackagesTest):
+
def setUp(self):
super(TclTests, self).setUp()
self.configuration_obj = spack.modules.CONFIGURATION
@@ -116,7 +201,6 @@ class TclTests(MockPackagesTest):
def get_modulefile_content(self, spec):
spec.concretize()
- print spec, '&&&&&'
generator = spack.modules.TclModule(spec)
generator.write()
content = FILE_REGISTRY[generator.file_name].split('\n')
@@ -127,6 +211,8 @@ class TclTests(MockPackagesTest):
spec = spack.spec.Spec('mpich@3.0.4')
content = self.get_modulefile_content(spec)
self.assertTrue('module-whatis "mpich @3.0.4"' in content)
+ self.assertRaises(TypeError, spack.modules.dependencies,
+ spec, 'non-existing-tag')
def test_autoload(self):
spack.modules.CONFIGURATION = configuration_autoload_direct
@@ -141,11 +227,21 @@ class TclTests(MockPackagesTest):
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5)
self.assertEqual(len([x for x in content if 'module load ' in x]), 5)
+ def test_prerequisites(self):
+ spack.modules.CONFIGURATION = configuration_prerequisites_direct
+ spec = spack.spec.Spec('mpileaks arch=x86-linux')
+ content = self.get_modulefile_content(spec)
+ self.assertEqual(len([x for x in content if 'prereq' in x]), 2)
+
+ spack.modules.CONFIGURATION = configuration_prerequisites_all
+ spec = spack.spec.Spec('mpileaks arch=x86-linux')
+ content = self.get_modulefile_content(spec)
+ self.assertEqual(len([x for x in content if 'prereq' in x]), 5)
+
def test_alter_environment(self):
spack.modules.CONFIGURATION = configuration_alter_environment
spec = spack.spec.Spec('mpileaks platform=test target=x86_64')
content = self.get_modulefile_content(spec)
- print content
self.assertEqual(
len([x
for x in content
@@ -156,7 +252,6 @@ class TclTests(MockPackagesTest):
spec = spack.spec.Spec('libdwarf %clang platform=test target=x86_32')
content = self.get_modulefile_content(spec)
- print content
self.assertEqual(
len([x
for x in content
@@ -164,6 +259,10 @@ class TclTests(MockPackagesTest):
self.assertEqual(
len([x for x in content if 'setenv FOO "foo"' in x]), 0)
self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 0)
+ self.assertEqual(
+ len([x for x in content if 'is-loaded foo/bar' in x]), 1)
+ self.assertEqual(
+ len([x for x in content if 'module load foo/bar' in x]), 1)
def test_blacklist(self):
spack.modules.CONFIGURATION = configuration_blacklist
@@ -171,6 +270,13 @@ class TclTests(MockPackagesTest):
content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1)
self.assertEqual(len([x for x in content if 'module load ' in x]), 1)
+ spec = spack.spec.Spec('callpath arch=x86-linux')
+ # Returns a StringIO instead of a string as no module file was written
+ self.assertRaises(AttributeError, self.get_modulefile_content, spec)
+ spec = spack.spec.Spec('zmpi arch=x86-linux')
+ content = self.get_modulefile_content(spec)
+ self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1)
+ self.assertEqual(len([x for x in content if 'module load ' in x]), 1)
def test_conflicts(self):
spack.modules.CONFIGURATION = configuration_conflicts
@@ -182,3 +288,57 @@ class TclTests(MockPackagesTest):
len([x for x in content if x == 'conflict mpileaks']), 1)
self.assertEqual(
len([x for x in content if x == 'conflict intel/14.0.1']), 1)
+
+ spack.modules.CONFIGURATION = configuration_wrong_conflicts
+ self.assertRaises(SystemExit, self.get_modulefile_content, spec)
+
+ def test_suffixes(self):
+ spack.modules.CONFIGURATION = configuration_suffix
+ spec = spack.spec.Spec('mpileaks+debug arch=x86-linux')
+ spec.concretize()
+ generator = spack.modules.TclModule(spec)
+ self.assertTrue('foo' in generator.use_name)
+
+ spec = spack.spec.Spec('mpileaks~debug arch=x86-linux')
+ spec.concretize()
+ generator = spack.modules.TclModule(spec)
+ self.assertTrue('bar' in generator.use_name)
+
+
+configuration_dotkit = {
+ 'enable': ['dotkit'],
+ 'dotkit': {
+ 'all': {
+ 'prerequisites': 'direct'
+ }
+ }
+}
+
+
+class DotkitTests(MockPackagesTest):
+
+ def setUp(self):
+ super(DotkitTests, self).setUp()
+ self.configuration_obj = spack.modules.CONFIGURATION
+ spack.modules.open = mock_open
+ # Make sure that a non-mocked configuration will trigger an error
+ spack.modules.CONFIGURATION = None
+
+ def tearDown(self):
+ del spack.modules.open
+ spack.modules.CONFIGURATION = self.configuration_obj
+ super(DotkitTests, self).tearDown()
+
+ def get_modulefile_content(self, spec):
+ spec.concretize()
+ generator = spack.modules.Dotkit(spec)
+ generator.write()
+ content = FILE_REGISTRY[generator.file_name].split('\n')
+ return content
+
+ def test_dotkit(self):
+ spack.modules.CONFIGURATION = configuration_dotkit
+ spec = spack.spec.Spec('mpileaks arch=x86-linux')
+ content = self.get_modulefile_content(spec)
+ self.assertTrue('#c spack' in content)
+ self.assertTrue('#d mpileaks @2.3' in content)
generated by cgit v1.2.3-60-g2f50 (git 2.42.0) at 2024-07-21 21:17:42 +0000