summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--lib/spack/spack/cmd/module.py16
-rw-r--r--lib/spack/spack/config.py204
-rw-r--r--lib/spack/spack/environment.py51
-rw-r--r--lib/spack/spack/modules.py202
-rw-r--r--lib/spack/spack/test/__init__.py69
-rw-r--r--lib/spack/spack/test/modules.py42
6 files changed, 330 insertions, 254 deletions
diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py
index f996f4eb84..cfe59c8d98 100644
--- a/lib/spack/spack/cmd/module.py
+++ b/lib/spack/spack/cmd/module.py
@@ -32,18 +32,21 @@ from llnl.util.filesystem import mkdirp
from spack.modules import module_types
from spack.util.string import *
-description ="Manipulate modules and dotkits."
+description = "Manipulate modules and dotkits."
def setup_parser(subparser):
sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='module_command')
- refresh_parser = sp.add_parser('refresh', help='Regenerate all module files.')
+ sp.add_parser('refresh', help='Regenerate all module files.')
find_parser = sp.add_parser('find', help='Find module files for packages.')
- find_parser.add_argument(
- 'module_type', help="Type of module to find file for. [" + '|'.join(module_types) + "]")
- find_parser.add_argument('spec', nargs='+', help='spec to find a module file for.')
+ find_parser.add_argument('module_type',
+ help="Type of module to find file for. [" +
+ '|'.join(module_types) + "]")
+ find_parser.add_argument('spec',
+ nargs='+',
+ help='spec to find a module file for.')
def module_find(mtype, spec_array):
@@ -53,7 +56,8 @@ def module_find(mtype, spec_array):
should type to use that package's module.
"""
if mtype not in module_types:
- tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types)))
+ tty.die("Invalid module type: '%s'. Options are %s" %
+ (mtype, comma_or(module_types)))
specs = spack.cmd.parse_specs(spec_array)
if len(specs) > 1:
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index 6ddf07776b..684a420b3b 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -134,8 +134,6 @@ from yaml.error import MarkedYAMLError
# Hacked yaml for configuration files preserves line numbers.
import spack.util.spack_yaml as syaml
-
-
"""Dict from section names -> schema for that section."""
section_schemas = {
'compilers': {
@@ -149,25 +147,31 @@ section_schemas = {
'default': {},
'additionalProperties': False,
'patternProperties': {
- r'\w[\w-]*': { # architecture
+ r'\w[\w-]*': { # architecture
'type': 'object',
'additionalProperties': False,
'patternProperties': {
- r'\w[\w-]*@\w[\w-]*': { # compiler spec
+ r'\w[\w-]*@\w[\w-]*': { # compiler spec
'type': 'object',
'additionalProperties': False,
'required': ['cc', 'cxx', 'f77', 'fc'],
'properties': {
- 'cc': { 'anyOf': [ {'type' : 'string' },
- {'type' : 'null' }]},
- 'cxx': { 'anyOf': [ {'type' : 'string' },
- {'type' : 'null' }]},
- 'f77': { 'anyOf': [ {'type' : 'string' },
- {'type' : 'null' }]},
- 'fc': { 'anyOf': [ {'type' : 'string' },
- {'type' : 'null' }]},
- },},},},},},},},
-
+ 'cc': {'anyOf': [{'type': 'string'},
+ {'type': 'null'}]},
+ 'cxx': {'anyOf': [{'type': 'string'},
+ {'type': 'null'}]},
+ 'f77': {'anyOf': [{'type': 'string'},
+ {'type': 'null'}]},
+ 'fc': {'anyOf': [{'type': 'string'},
+ {'type': 'null'}]},
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
'mirrors': {
'$schema': 'http://json-schema.org/schema#',
'title': 'Spack mirror configuration file schema',
@@ -180,8 +184,12 @@ section_schemas = {
'additionalProperties': False,
'patternProperties': {
r'\w[\w-]*': {
- 'type': 'string'},},},},},
-
+ 'type': 'string'
+ },
+ },
+ },
+ },
+ },
'repos': {
'$schema': 'http://json-schema.org/schema#',
'title': 'Spack repository configuration file schema',
@@ -192,8 +200,11 @@ section_schemas = {
'type': 'array',
'default': [],
'items': {
- 'type': 'string'},},},},
-
+ 'type': 'string'
+ },
+ },
+ },
+ },
'packages': {
'$schema': 'http://json-schema.org/schema#',
'title': 'Spack package configuration file schema',
@@ -205,39 +216,48 @@ section_schemas = {
'default': {},
'additionalProperties': False,
'patternProperties': {
- r'\w[\w-]*': { # package name
+ r'\w[\w-]*': { # package name
'type': 'object',
'default': {},
'additionalProperties': False,
'properties': {
'version': {
- 'type' : 'array',
- 'default' : [],
- 'items' : { 'anyOf' : [ { 'type' : 'string' },
- { 'type' : 'number'}]}}, #version strings
+ 'type': 'array',
+ 'default': [],
+ 'items': {'anyOf': [{'type': 'string'},
+ {'type': 'number'}]}
+ }, # version strings
'compiler': {
- 'type' : 'array',
- 'default' : [],
- 'items' : { 'type' : 'string' } }, #compiler specs
+ 'type': 'array',
+ 'default': [],
+ 'items': {'type': 'string'}
+ }, # compiler specs
'buildable': {
- 'type': 'boolean',
+ 'type': 'boolean',
'default': True,
- },
+ },
'providers': {
- 'type': 'object',
+ 'type': 'object',
'default': {},
'additionalProperties': False,
'patternProperties': {
r'\w[\w-]*': {
- 'type' : 'array',
- 'default' : [],
- 'items' : { 'type' : 'string' },},},},
+ 'type': 'array',
+ 'default': [],
+ 'items': {'type': 'string'},
+ },
+ },
+ },
'paths': {
- 'type' : 'object',
- 'default' : {},
+ 'type': 'object',
+ 'default': {},
}
- },},},},},},
-
+ },
+ },
+ },
+ },
+ },
+ },
'modules': {
'$schema': 'http://json-schema.org/schema#',
'title': 'Spack module file configuration file schema',
@@ -283,17 +303,22 @@ section_schemas = {
}
},
'autoload': {'$ref': '#/definitions/dependency_selection'},
- 'prerequisites': {'$ref': '#/definitions/dependency_selection'},
+ 'prerequisites':
+ {'$ref': '#/definitions/dependency_selection'},
'conflict': {'$ref': '#/definitions/array_of_strings'},
'environment': {
'type': 'object',
'default': {},
'additionalProperties': False,
'properties': {
- 'set': {'$ref': '#/definitions/dictionary_of_strings'},
- 'unset': {'$ref': '#/definitions/array_of_strings'},
- 'prepend_path': {'$ref': '#/definitions/dictionary_of_strings'},
- 'append_path': {'$ref': '#/definitions/dictionary_of_strings'}
+ 'set':
+ {'$ref': '#/definitions/dictionary_of_strings'},
+ 'unset':
+ {'$ref': '#/definitions/array_of_strings'},
+ 'prepend_path':
+ {'$ref': '#/definitions/dictionary_of_strings'},
+ 'append_path':
+ {'$ref': '#/definitions/dictionary_of_strings'}
}
}
}
@@ -304,15 +329,20 @@ section_schemas = {
'anyOf': [
{
'properties': {
- 'whitelist': {'$ref': '#/definitions/array_of_strings'},
- 'blacklist': {'$ref': '#/definitions/array_of_strings'},
+ 'whitelist':
+ {'$ref': '#/definitions/array_of_strings'},
+ 'blacklist':
+ {'$ref': '#/definitions/array_of_strings'},
'naming_scheme': {
- 'type': 'string' # Can we be more specific here?
+ 'type':
+ 'string' # Can we be more specific here?
}
}
},
{
- 'patternProperties': {r'\w[\w-]*': {'$ref': '#/definitions/module_file_configuration'}}
+ 'patternProperties':
+ {r'\w[\w-]*':
+ {'$ref': '#/definitions/module_file_configuration'}}
}
]
}
@@ -326,7 +356,8 @@ section_schemas = {
'prefix_inspections': {
'type': 'object',
'patternProperties': {
- r'\w[\w-]*': { # path to be inspected for existence (relative to prefix)
+ r'\w[\w-]*':
+ { # path to be inspected (relative to prefix)
'$ref': '#/definitions/array_of_strings'
}
}
@@ -341,13 +372,15 @@ section_schemas = {
},
'tcl': {
'allOf': [
- {'$ref': '#/definitions/module_type_configuration'}, # Base configuration
+ {'$ref': '#/definitions/module_type_configuration'
+ }, # Base configuration
{} # Specific tcl extensions
]
},
'dotkit': {
'allOf': [
- {'$ref': '#/definitions/module_type_configuration'}, # Base configuration
+ {'$ref': '#/definitions/module_type_configuration'
+ }, # Base configuration
{} # Specific dotkit extensions
]
},
@@ -356,7 +389,6 @@ section_schemas = {
},
},
}
-
"""OrderedDict of config scopes keyed by name.
Later scopes will override earlier scopes.
"""
@@ -366,12 +398,13 @@ config_scopes = OrderedDict()
def validate_section_name(section):
"""Raise a ValueError if the section is not a valid section."""
if section not in section_schemas:
- raise ValueError("Invalid config section: '%s'. Options are %s"
- % (section, section_schemas))
+ raise ValueError("Invalid config section: '%s'. Options are %s" %
+ (section, section_schemas))
def extend_with_default(validator_class):
- """Add support for the 'default' attribute for properties and patternProperties.
+ """Add support for the 'default' attribute for
+ properties and patternProperties
jsonschema does not handle this out of the box -- it only
validates. This allows us to set default values for configs
@@ -380,13 +413,15 @@ def extend_with_default(validator_class):
"""
validate_properties = validator_class.VALIDATORS["properties"]
- validate_pattern_properties = validator_class.VALIDATORS["patternProperties"]
+ validate_pattern_properties = validator_class.VALIDATORS[
+ "patternProperties"]
def set_defaults(validator, properties, instance, schema):
for property, subschema in properties.iteritems():
if "default" in subschema:
instance.setdefault(property, subschema["default"])
- for err in validate_properties(validator, properties, instance, schema):
+ for err in validate_properties(validator, properties, instance,
+ schema):
yield err
def set_pp_defaults(validator, properties, instance, schema):
@@ -397,17 +432,19 @@ def extend_with_default(validator_class):
if re.match(property, key) and val is None:
instance[key] = subschema["default"]
- for err in validate_pattern_properties(validator, properties, instance, schema):
+ for err in validate_pattern_properties(validator, properties, instance,
+ schema):
yield err
return validators.extend(validator_class, {
- "properties" : set_defaults,
- "patternProperties" : set_pp_defaults
+ "properties": set_defaults,
+ "patternProperties": set_pp_defaults
})
DefaultSettingValidator = extend_with_default(Draft4Validator)
+
def validate_section(data, schema):
"""Validate data read in from a Spack YAML file.
@@ -429,9 +466,9 @@ class ConfigScope(object):
"""
def __init__(self, name, path):
- self.name = name # scope name.
- self.path = path # path to directory containing configs.
- self.sections = {} # sections read from config files.
+ self.name = name # scope name.
+ self.path = path # path to directory containing configs.
+ self.sections = {} # sections read from config files.
# Register in a dict of all ConfigScopes
# TODO: make this cleaner. Mocking up for testing is brittle.
@@ -442,16 +479,14 @@ class ConfigScope(object):
validate_section_name(section)
return os.path.join(self.path, "%s.yaml" % section)
-
def get_section(self, section):
- if not section in self.sections:
- path = self.get_section_filename(section)
+ if section not in self.sections:
+ path = self.get_section_filename(section)
schema = section_schemas[section]
- data = _read_config_file(path, schema)
+ data = _read_config_file(path, schema)
self.sections[section] = data
return self.sections[section]
-
def write_section(self, section):
filename = self.get_section_filename(section)
data = self.get_section(section)
@@ -463,8 +498,8 @@ class ConfigScope(object):
except jsonschema.ValidationError as e:
raise ConfigSanityError(e, data)
except (yaml.YAMLError, IOError) as e:
- raise ConfigFileError("Error writing to config file: '%s'" % str(e))
-
+ raise ConfigFileError("Error writing to config file: '%s'" %
+ str(e))
def clear(self):
"""Empty cached config information."""
@@ -496,8 +531,8 @@ def validate_scope(scope):
return config_scopes[scope]
else:
- raise ValueError("Invalid config scope: '%s'. Must be one of %s"
- % (scope, config_scopes.keys()))
+ raise ValueError("Invalid config scope: '%s'. Must be one of %s" %
+ (scope, config_scopes.keys()))
def _read_config_file(filename, schema):
@@ -523,12 +558,12 @@ def _read_config_file(filename, schema):
return data
except MarkedYAMLError as e:
- raise ConfigFileError(
- "Error parsing yaml%s: %s" % (str(e.context_mark), e.problem))
+ raise ConfigFileError("Error parsing yaml%s: %s" %
+ (str(e.context_mark), e.problem))
except IOError as e:
- raise ConfigFileError(
- "Error reading configuration file %s: %s" % (filename, str(e)))
+ raise ConfigFileError("Error reading configuration file %s: %s" %
+ (filename, str(e)))
def clear_config_caches():
@@ -551,6 +586,7 @@ def _merge_yaml(dest, source):
parent instead of merging.
"""
+
def they_are(t):
return isinstance(dest, t) and isinstance(source, t)
@@ -571,7 +607,7 @@ def _merge_yaml(dest, source):
# Source dict is merged into dest.
elif they_are(dict):
for sk, sv in source.iteritems():
- if not sk in dest:
+ if sk not in dest:
dest[sk] = copy.copy(sv)
else:
dest[sk] = _merge_yaml(dest[sk], source[sk])
@@ -653,7 +689,7 @@ def print_section(section):
data = syaml.syaml_dict()
data[section] = get_config(section)
syaml.dump(data, stream=sys.stdout, default_flow_style=False)
- except (yaml.YAMLError, IOError) as e:
+ except (yaml.YAMLError, IOError):
raise ConfigError("Error reading configuration: %s" % section)
@@ -683,15 +719,22 @@ def is_spec_buildable(spec):
"""Return true if the spec pkgspec is configured as buildable"""
allpkgs = get_config('packages')
name = spec.name
- if not spec.name in allpkgs:
+ if name not in allpkgs:
return True
- if not 'buildable' in allpkgs[spec.name]:
+ if 'buildable' not in allpkgs[name]:
return True
return allpkgs[spec.name]['buildable']
-class ConfigError(SpackError): pass
-class ConfigFileError(ConfigError): pass
+class ConfigError(SpackError):
+
+ pass
+
+
+class ConfigFileError(ConfigError):
+
+ pass
+
def get_path(path, data):
if path:
@@ -699,8 +742,10 @@ def get_path(path, data):
else:
return data
+
class ConfigFormatError(ConfigError):
"""Raised when a configuration format does not match its schema."""
+
def __init__(self, validation_error, data):
# Try to get line number from erroneous instance and its parent
instance_mark = getattr(validation_error.instance, '_start_mark', None)
@@ -733,5 +778,6 @@ class ConfigFormatError(ConfigError):
message = '%s: %s' % (location, validation_error.message)
super(ConfigError, self).__init__(message)
+
class ConfigSanityError(ConfigFormatError):
"""Same as ConfigFormatError, raised when config is written by Spack."""
diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py
index 92ab4e6bea..9748b4033a 100644
--- a/lib/spack/spack/environment.py
+++ b/lib/spack/spack/environment.py
@@ -26,7 +26,8 @@ class SetEnv(NameValueModifier):
class UnsetEnv(NameModifier):
def execute(self):
- os.environ.pop(self.name, None) # Avoid throwing if the variable was not set
+ # Avoid throwing if the variable was not set
+ os.environ.pop(self.name, None)
class SetPath(NameValueModifier):
@@ -55,7 +56,9 @@ class RemovePath(NameValueModifier):
def execute(self):
environment_value = os.environ.get(self.name, '')
directories = environment_value.split(':') if environment_value else []
- directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.value)]
+ directories = [os.path.normpath(x)
+ for x in directories
+ if x != os.path.normpath(self.value)]
os.environ[self.name] = ':'.join(directories)
@@ -63,7 +66,8 @@ class EnvironmentModifications(object):
"""
Keeps track of requests to modify the current environment.
- Each call to a method to modify the environment stores the extra information on the caller in the request:
+ Each call to a method to modify the environment stores the extra
+ information on the caller in the request:
- 'filename' : filename of the module where the caller is defined
- 'lineno': line number where the request occurred
- 'context' : line of code that issued the request that failed
@@ -71,10 +75,10 @@ class EnvironmentModifications(object):
def __init__(self, other=None):
"""
- Initializes a new instance, copying commands from other if it is not None
+ Initializes a new instance, copying commands from other if not None
Args:
- other: another instance of EnvironmentModifications from which (optional)
+ other: another instance of EnvironmentModifications
"""
self.env_modifications = []
if other is not None:
@@ -93,7 +97,7 @@ class EnvironmentModifications(object):
@staticmethod
def _check_other(other):
if not isinstance(other, EnvironmentModifications):
- raise TypeError('other must be an instance of EnvironmentModifications')
+ raise TypeError('not an instance of EnvironmentModifications')
def _get_outside_caller_attributes(self):
stack = inspect.stack()
@@ -101,12 +105,10 @@ class EnvironmentModifications(object):
_, filename, lineno, _, context, index = stack[2]
context = context[index].strip()
except Exception:
- filename, lineno, context = 'unknown file', 'unknown line', 'unknown context'
- args = {
- 'filename': filename,
- 'lineno': lineno,
- 'context': context
- }
+ filename = 'unknown file'
+ lineno = 'unknown line'
+ context = 'unknown context'
+ args = {'filename': filename, 'lineno': lineno, 'context': context}
return args
def set(self, name, value, **kwargs):
@@ -170,7 +172,7 @@ class EnvironmentModifications(object):
def remove_path(self, name, path, **kwargs):
"""
- Stores in the current object a request to remove a path from a path list
+ Stores in the current object a request to remove a path from a list
Args:
name: name of the path list in the environment
@@ -185,7 +187,8 @@ class EnvironmentModifications(object):
Returns a dict of the modifications grouped by variable name
Returns:
- dict mapping the environment variable name to the modifications to be done on it
+ dict mapping the environment variable name to the modifications
+ to be done on it
"""
modifications = collections.defaultdict(list)
for item in self:
@@ -203,7 +206,8 @@ class EnvironmentModifications(object):
Applies the modifications and clears the list
"""
modifications = self.group_by_name()
- # Apply the modifications to the environment variables one variable at a time
+ # Apply the modifications to the environment variables one variable
+ # at a time
for name, actions in sorted(modifications.items()):
for x in actions:
x.execute()
@@ -224,13 +228,17 @@ def concatenate_paths(paths):
def set_or_unset_not_first(variable, changes, errstream):
"""
- Check if we are going to set or unset something after other modifications have already been requested
+ Check if we are going to set or unset something after other modifications
+ have already been requested
"""
- indexes = [ii for ii, item in enumerate(changes) if ii != 0 and type(item) in [SetEnv, UnsetEnv]]
+ indexes = [ii
+ for ii, item in enumerate(changes)
+ if ii != 0 and type(item) in [SetEnv, UnsetEnv]]
if indexes:
good = '\t \t{context} at {filename}:{lineno}'
nogood = '\t--->\t{context} at {filename}:{lineno}'
- errstream('Suspicious requests to set or unset the variable \'{var}\' found'.format(var=variable))
+ message = 'Suspicious requests to set or unset the variable \'{var}\' found' # NOQA: ignore=E501
+ errstream(message.format(var=variable))
for ii, item in enumerate(changes):
print_format = nogood if ii in indexes else good
errstream(print_format.format(**item.args))
@@ -238,8 +246,8 @@ def set_or_unset_not_first(variable, changes, errstream):
def validate(env, errstream):
"""
- Validates the environment modifications to check for the presence of suspicious patterns. Prompts a warning for
- everything that was found
+ Validates the environment modifications to check for the presence of
+ suspicious patterns. Prompts a warning for everything that was found
Current checks:
- set or unset variables after other changes on the same variable
@@ -254,7 +262,8 @@ def validate(env, errstream):
def filter_environment_blacklist(env, variables):
"""
- Generator that filters out any change to environment variables present in the input list
+ Generator that filters out any change to environment variables present in
+ the input list
Args:
env: list of environment modifications
diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py
index ffed469b20..0dc6f06f55 100644
--- a/lib/spack/spack/modules.py
+++ b/lib/spack/spack/modules.py
@@ -23,36 +23,34 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""
-This module contains code for creating environment modules, which can include dotkits, tcl modules, lmod, and others.
+This module contains code for creating environment modules, which can include
+dotkits, tcl modules, lmod, and others.
-The various types of modules are installed by post-install hooks and removed after an uninstall by post-uninstall hooks.
-This class consolidates the logic for creating an abstract description of the information that module systems need.
-Currently that includes a number of directories to be appended to paths in the user's environment:
+The various types of modules are installed by post-install hooks and removed
+after an uninstall by post-uninstall hooks. This class consolidates the logic
+for creating an abstract description of the information that module systems
+need.
- * /bin directories to be appended to PATH
- * /lib* directories for LD_LIBRARY_PATH
- * /include directories for CPATH
- * /man* and /share/man* directories for MANPATH
- * the package prefix for CMAKE_PREFIX_PATH
+This module also includes logic for coming up with unique names for the module
+files so that they can be found by the various shell-support files in
+$SPACK/share/spack/setup-env.*.
-This module also includes logic for coming up with unique names for the module files so that they can be found by the
-various shell-support files in $SPACK/share/spack/setup-env.*.
-
-Each hook in hooks/ implements the logic for writing its specific type of module file.
+Each hook implements the logic for writing its specific type of module.
"""
import copy
import datetime
import os
import os.path
import re
-import textwrap
import string
+import textwrap
import llnl.util.tty as tty
import spack
import spack.config
from llnl.util.filesystem import join_path, mkdirp
-from spack.build_environment import parent_class_modules, set_module_variables_for_package
+from spack.build_environment import parent_class_modules
+from spack.build_environment import set_module_variables_for_package
from spack.environment import *
__all__ = ['EnvModule', 'Dotkit', 'TclModule']
@@ -67,30 +65,26 @@ def print_help():
"""
For use by commands to tell user how to activate shell support.
"""
- tty.msg("This command requires spack's shell integration.",
- "",
+ tty.msg("This command requires spack's shell integration.", "",
"To initialize spack's shell commands, you must run one of",
"the commands below. Choose the right command for your shell.",
- "",
- "For bash and zsh:",
- " . %s/setup-env.sh" % spack.share_path,
- "",
- "For csh and tcsh:",
- " setenv SPACK_ROOT %s" % spack.prefix,
- " source %s/setup-env.csh" % spack.share_path,
- "")
+ "", "For bash and zsh:",
+ " . %s/setup-env.sh" % spack.share_path, "",
+ "For csh and tcsh:", " setenv SPACK_ROOT %s" % spack.prefix,
+ " source %s/setup-env.csh" % spack.share_path, "")
def inspect_path(prefix):
"""
- Inspects the prefix of an installation to search for common layouts. Issues a request to modify the environment
- accordingly when an item is found.
+ Inspects the prefix of an installation to search for common layouts.
+ Issues a request to modify the environment when an item is found.
Args:
prefix: prefix of the installation
Returns:
- instance of EnvironmentModifications containing the requested modifications
+ instance of EnvironmentModifications containing the requested
+ modifications
"""
env = EnvironmentModifications()
# Inspect the prefix to check for the existence of common directories
@@ -105,18 +99,22 @@ def inspect_path(prefix):
def dependencies(spec, request='all'):
"""
- Returns the list of dependent specs for a given spec, according to the given request
+ Returns the list of dependent specs for a given spec, according to the
+ given request
Args:
spec: target spec
request: either 'none', 'direct' or 'all'
Returns:
- empty list if 'none', direct dependency list if 'direct', all dependencies if 'all'
+ empty list if 'none', direct dependency list if 'direct', all
+ dependencies if 'all'
"""
if request not in ('none', 'direct', 'all'):
- raise tty.error("Wrong value for argument 'request' : should be one of ('none', 'direct', 'all') "
- " [current value is '%s']" % request)
+ message = "Wrong value for argument 'request' : "
+ message += "should be one of ('none', 'direct', 'all')"
+ message += " [current value is '{0}']"
+ raise tty.error(message.format(request))
if request == 'none':
return []
@@ -124,12 +122,19 @@ def dependencies(spec, request='all'):
if request == 'direct':
return [xx for _, xx in spec.dependencies.items()]
- # FIXME : during module file creation nodes seem to be visited multiple times even if cover='nodes'
- # FIXME : is given. This work around permits to get a unique list of spec anyhow.
- # FIXME : Possibly we miss a merge step among nodes that refer to the same package.
+ # FIXME : during module file creation nodes seem to be visited
+ # FIXME : multiple times even if cover='nodes' is given. This work around
+ # FIXME : permits to get a unique list of spec anyhow. Maybe we miss a
+ # FIXME : merge step among nodes that refer to the same package?
seen = set()
seen_add = seen.add
- l = [xx for xx in sorted(spec.traverse(order='post', depth=True, cover='nodes', root=False), reverse=True)]
+ l = [xx
+ for xx in sorted(
+ spec.traverse(order='post',
+ depth=True,
+ cover='nodes',
+ root=False),
+ reverse=True)]
return [xx for ii, xx in l if not (xx in seen or seen_add(xx))]
@@ -146,7 +151,8 @@ def update_dictionary_extending_lists(target, update):
def parse_config_options(module_generator):
"""
- Parse the configuration file and returns a bunch of items that will be needed during module file generation
+ Parse the configuration file and returns a bunch of items that will be
+ needed during module file generation
Args:
module_generator: module generator for a given spec
@@ -154,11 +160,14 @@ def parse_config_options(module_generator):
Returns:
autoloads: list of specs to be autoloaded
prerequisites: list of specs to be marked as prerequisite
- filters: list of environment variables whose modification is blacklisted in module files
- env: list of custom environment modifications to be applied in the module file
+ filters: list of environment variables whose modification is
+ blacklisted in module files
+ env: list of custom environment modifications to be applied in the
+ module file
"""
# Get the configuration for this kind of generator
- module_configuration = copy.deepcopy(CONFIGURATION.get(module_generator.name, {}))
+ module_configuration = copy.deepcopy(CONFIGURATION.get(
+ module_generator.name, {}))
#####
# Merge all the rules
@@ -179,9 +188,12 @@ def parse_config_options(module_generator):
#####
# Automatic loading loads
- module_file_actions['autoload'] = dependencies(module_generator.spec, module_file_actions.get('autoload', 'none'))
+ module_file_actions['autoload'] = dependencies(
+ module_generator.spec, module_file_actions.get('autoload', 'none'))
# Prerequisites
- module_file_actions['prerequisites'] = dependencies(module_generator.spec, module_file_actions.get('prerequisites', 'none'))
+ module_file_actions['prerequisites'] = dependencies(
+ module_generator.spec, module_file_actions.get('prerequisites',
+ 'none'))
# Environment modifications
environment_actions = module_file_actions.pop('environment', {})
env = EnvironmentModifications()
@@ -189,7 +201,7 @@ def parse_config_options(module_generator):
def process_arglist(arglist):
if method == 'unset':
for x in arglist:
- yield (x,)
+ yield (x, )
else:
for x in arglist.iteritems():
yield x
@@ -198,19 +210,20 @@ def parse_config_options(module_generator):
for args in process_arglist(arglist):
getattr(env, method)(*args)
- # for item in arglist:
- # if method == 'unset':
- # args = [item]
- # else:
- # args = item.split(',')
- # getattr(env, method)(*args)
+ # for item in arglist:
+ # if method == 'unset':
+ # args = [item]
+ # else:
+ # args = item.split(',')
+ # getattr(env, method)(*args)
return module_file_actions, env
def filter_blacklisted(specs, module_name):
"""
- Given a sequence of specs, filters the ones that are blacklisted in the module configuration file.
+ Given a sequence of specs, filters the ones that are blacklisted in the
+ module configuration file.
Args:
specs: sequence of spec instances
@@ -233,7 +246,8 @@ class EnvModule(object):
class __metaclass__(type):
def __init__(cls, name, bases, dict):
type.__init__(cls, name, bases, dict)
- if cls.name != 'env_module' and cls.name in CONFIGURATION['enable']:
+ if cls.name != 'env_module' and cls.name in CONFIGURATION[
+ 'enable']:
module_types[cls.name] = cls
def __init__(self, spec=None):
@@ -249,7 +263,8 @@ class EnvModule(object):
# long description is the docstring with reduced whitespace.
self.long_description = None
if self.spec.package.__doc__:
- self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__)
+ self.long_description = re.sub(r'\s+', ' ',
+ self.spec.package.__doc__)
@property
def naming_scheme(self):
@@ -271,12 +286,14 @@ class EnvModule(object):
@property
def use_name(self):
"""
- Subclasses should implement this to return the name the module command uses to refer to the package.
+ Subclasses should implement this to return the name the module command
+ uses to refer to the package.
"""
naming_tokens = self.tokens
naming_scheme = self.naming_scheme
name = naming_scheme.format(**naming_tokens)
- name += '-' + self.spec.dag_hash() # Always append the hash to make the module file unique
+ name += '-' + self.spec.dag_hash(
+ ) # Always append the hash to make the module file unique
# Not everybody is working on linux...
parts = name.split('/')
name = join_path(*parts)
@@ -296,8 +313,12 @@ class EnvModule(object):
@property
def blacklisted(self):
configuration = CONFIGURATION.get(self.name, {})
- whitelist_matches = [x for x in configuration.get('whitelist', []) if self.spec.satisfies(x)]
- blacklist_matches = [x for x in configuration.get('blacklist', []) if self.spec.satisfies(x)]
+ whitelist_matches = [x
+ for x in configuration.get('whitelist', [])
+ if self.spec.satisfies(x)]
+ blacklist_matches = [x
+ for x in configuration.get('blacklist', [])
+ if self.spec.satisfies(x)]
if whitelist_matches:
message = '\tWHITELIST : %s [matches : ' % self.spec.cshort_spec
for rule in whitelist_matches:
@@ -327,7 +348,8 @@ class EnvModule(object):
"""
if self.blacklisted:
return
- tty.debug("\tWRITE : %s [%s]" % (self.spec.cshort_spec, self.file_name))
+ tty.debug("\tWRITE : %s [%s]" %
+ (self.spec.cshort_spec, self.file_name))
module_dir = os.path.dirname(self.file_name)
if not os.path.exists(module_dir):
@@ -337,11 +359,12 @@ class EnvModule(object):
# installation prefix
env = inspect_path(self.spec.prefix)
- # Let the extendee/dependency modify their extensions/dependencies before asking for
- # package-specific modifications
+ # Let the extendee/dependency modify their extensions/dependencies
+ # before asking for package-specific modifications
spack_env = EnvironmentModifications()
- # TODO : the code down below is quite similar to build_environment.setup_package and needs to be
- # TODO : factored out to a single place
+ # TODO : the code down below is quite similar to
+ # TODO : build_environment.setup_package and needs to be factored out
+ # TODO : to a single place
for item in dependencies(self.spec, 'all'):
package = self.spec[item.name].package
modules = parent_class_modules(package.__class__)
@@ -358,14 +381,18 @@ class EnvModule(object):
# Parse configuration file
module_configuration, conf_env = parse_config_options(self)
env.extend(conf_env)
- filters = module_configuration.get('filter', {}).get('environment_blacklist',{})
+ filters = module_configuration.get('filter', {}).get(
+ 'environment_blacklist', {})
# Build up the module file content
module_file_content = self.header
- for x in filter_blacklisted(module_configuration.pop('autoload', []), self.name):
+ for x in filter_blacklisted(
+ module_configuration.pop('autoload', []), self.name):
module_file_content += self.autoload(x)
- for x in filter_blacklisted(module_configuration.pop('prerequisites', []), self.name):
+ for x in filter_blacklisted(
+ module_configuration.pop('prerequisites', []), self.name):
module_file_content += self.prerequisite(x)
- for line in self.process_environment_command(filter_environment_blacklist(env, filters)):
+ for line in self.process_environment_command(
+ filter_environment_blacklist(env, filters)):
module_file_content += line
for line in self.module_specific_content(module_configuration):
module_file_content += line
@@ -392,10 +419,13 @@ class EnvModule(object):
def process_environment_command(self, env):
for command in env:
try:
- yield self.environment_modifications_formats[type(command)].format(**command.args)
+ yield self.environment_modifications_formats[type(
+ command)].format(**command.args)
except KeyError:
- tty.warn('Cannot handle command of type {command} : skipping request'.format(command=type(command)))
- tty.warn('{context} at {filename}:{lineno}'.format(**command.args))
+ message = 'Cannot handle command of type {command} : skipping request' # NOQA: ignore=E501
+ tty.warn(message.format(command=type(command)))
+ context = '{context} at {filename}:{lineno}'
+ tty.warn(context.format(**command.args))
@property
def file_name(self):
@@ -408,9 +438,12 @@ class EnvModule(object):
if os.path.exists(mod_file):
try:
os.remove(mod_file) # Remove the module file
- os.removedirs(os.path.dirname(mod_file)) # Remove all the empty directories from the leaf up
+ os.removedirs(
+ os.path.dirname(mod_file)
+ ) # Remove all the empty directories from the leaf up
except OSError:
- pass # removedirs throws OSError on first non-empty directory found
+ # removedirs throws OSError on first non-empty directory found
+ pass
class Dotkit(EnvModule):
@@ -424,13 +457,12 @@ class Dotkit(EnvModule):
autoload_format = 'dk_op {module_file}\n'
- prerequisite_format = None # TODO : does something like prerequisite exist for dotkit?
-
- default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}'
+ default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' # NOQA: ignore=E501
@property
def file_name(self):
- return join_path(Dotkit.path, self.spec.architecture, '%s.dk' % self.use_name)
+ return join_path(Dotkit.path, self.spec.architecture,
+ '%s.dk' % self.use_name)
@property
def header(self):
@@ -474,7 +506,7 @@ class TclModule(EnvModule):
prerequisite_format = 'prereq {module_file}\n'
- default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}'
+ default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' # NOQA: ignore=E501
@property
def file_name(self):
@@ -482,9 +514,10 @@ class TclModule(EnvModule):
@property
def header(self):
+ timestamp = datetime.datetime.now()
# TCL Modulefile header
header = '#%Module1.0\n'
- header += '## Module file created by spack (https://github.com/LLNL/spack) on %s\n' % datetime.datetime.now()
+ header += '## Module file created by spack (https://github.com/LLNL/spack) on %s\n' % timestamp # NOQA: ignore=E501
header += '##\n'
header += '## %s\n' % self.spec.short_spec
header += '##\n'
@@ -509,16 +542,19 @@ class TclModule(EnvModule):
f = string.Formatter()
for item in conflict_format:
line = 'conflict ' + item + '\n'
- if len([x for x in f.parse(line)]) > 1: # We do have placeholder to substitute
- for naming_dir, conflict_dir in zip(self.naming_scheme.split('/'), item.split('/')):
+ if len([x for x in f.parse(line)
+ ]) > 1: # We do have placeholder to substitute
+ for naming_dir, conflict_dir in zip(
+ self.naming_scheme.split('/'), item.split('/')):
if naming_dir != conflict_dir:
- message = 'conflict scheme does not match naming scheme [{spec}]\n\n'
+ message = 'conflict scheme does not match naming'
+ message += ' [{spec}]\n\n'
message += 'naming scheme : "{nformat}"\n'
message += 'conflict scheme : "{cformat}"\n\n'
- message += '** You may want to check your `modules.yaml` configuration file **\n'
- tty.error(
- message.format(spec=self.spec, nformat=self.naming_scheme, cformat=item)
- )
+ message += '** You may want to check your `modules.yaml` configuration file **\n' # NOQA: ignore=E501
+ tty.error(message.format(spec=self.spec,
+ nformat=self.naming_scheme,
+ cformat=item))
raise SystemExit('Module generation aborted.')
line = line.format(**naming_tokens)
yield line
diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py
index 05f58ab7b1..395ca0c87a 100644
--- a/lib/spack/spack/test/__init__.py
+++ b/lib/spack/spack/test/__init__.py
@@ -23,53 +23,23 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import sys
-import unittest
-import nose
-from spack.test.tally_plugin import Tally
-from llnl.util.filesystem import join_path
import llnl.util.tty as tty
-from llnl.util.tty.colify import colify
-
+import nose
import spack
-
+from llnl.util.filesystem import join_path
+from llnl.util.tty.colify import colify
+from spack.test.tally_plugin import Tally
"""Names of tests to be included in Spack's test suite"""
-test_names = ['versions',
- 'url_parse',
- 'url_substitution',
- 'packages',
- 'stage',
- 'spec_syntax',
- 'spec_semantics',
- 'spec_dag',
- 'concretize',
- 'multimethod',
- 'install',
- 'package_sanity',
- 'config',
- 'directory_layout',
- 'pattern',
- 'python_version',
- 'git_fetch',
- 'svn_fetch',
- 'hg_fetch',
- 'mirror',
- 'modules',
- 'url_extrapolate',
- 'cc',
- 'link_tree',
- 'spec_yaml',
- 'optional_deps',
- 'make_executable',
- 'configure_guess',
- 'lock',
- 'database',
- 'namespace_trie',
- 'yaml',
- 'sbang',
- 'environment',
- 'cmd.uninstall',
- 'cmd.test_install']
+test_names = ['versions', 'url_parse', 'url_substitution', 'packages', 'stage',
+ 'spec_syntax', 'spec_semantics', 'spec_dag', 'concretize',
+ 'multimethod', 'install', 'package_sanity', 'config',
+ 'directory_layout', 'pattern', 'python_version', 'git_fetch',
+ 'svn_fetch', 'hg_fetch', 'mirror', 'modules', 'url_extrapolate',
+ 'cc', 'link_tree', 'spec_yaml', 'optional_deps',
+ 'make_executable', 'configure_guess', 'lock', 'database',
+ 'namespace_trie', 'yaml', 'sbang', 'environment',
+ 'cmd.uninstall', 'cmd.test_install']
def list_tests():
@@ -80,7 +50,6 @@ def list_tests():
def run(names, outputDir, verbose=False):
"""Run tests with the supplied names. Names should be a list. If
it's empty, run ALL of Spack's tests."""
- verbosity = 1 if not verbose else 2
if not names:
names = test_names
@@ -95,7 +64,7 @@ def run(names, outputDir, verbose=False):
tally = Tally()
for test in names:
module = 'spack.test.' + test
- print module
+ print(module)
tty.msg("Running test: %s" % test)
@@ -105,15 +74,13 @@ def run(names, outputDir, verbose=False):
xmlOutputFname = "unittests-{0}.xml".format(test)
xmlOutputPath = join_path(outputDir, xmlOutputFname)
runOpts += ["--with-xunit",
- "--xunit-file={0}".format(xmlOutputPath)]
+ "--xunit-file={0}".format(xmlOutputPath)]
argv = [""] + runOpts + [module]
- result = nose.run(argv=argv, addplugins=[tally])
+ nose.run(argv=argv, addplugins=[tally])
succeeded = not tally.failCount and not tally.errorCount
- tty.msg("Tests Complete.",
- "%5d tests run" % tally.numberOfTestsRun,
- "%5d failures" % tally.failCount,
- "%5d errors" % tally.errorCount)
+ tty.msg("Tests Complete.", "%5d tests run" % tally.numberOfTestsRun,
+ "%5d failures" % tally.failCount, "%5d errors" % tally.errorCount)
if succeeded:
tty.info("OK", format='g')
diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py
index b8b0d6fc6a..f0ff778a10 100644
--- a/lib/spack/spack/test/modules.py
+++ b/lib/spack/spack/test/modules.py
@@ -2,14 +2,18 @@ import collections
from contextlib import contextmanager
import StringIO
+import spack.modules
+from spack.test.mock_packages_test import MockPackagesTest
FILE_REGISTRY = collections.defaultdict(StringIO.StringIO)
+
# Monkey-patch open to write module files to a StringIO instance
@contextmanager
def mock_open(filename, mode):
if not mode == 'w':
- raise RuntimeError('test.modules : unexpected opening mode for monkey-patched open')
+ message = 'test.modules : unexpected opening mode [mock_open]'
+ raise RuntimeError(message)
FILE_REGISTRY[filename] = StringIO.StringIO()
@@ -20,7 +24,6 @@ def mock_open(filename, mode):
FILE_REGISTRY[filename] = handle.getvalue()
handle.close()
-import spack.modules
configuration_autoload_direct = {
'enable': ['tcl'],
@@ -47,7 +50,8 @@ configuration_alter_environment = {
'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']}
},
'=x86-linux': {
- 'environment': {'set': {'FOO': 'foo'}, 'unset': ['BAR']}
+ 'environment': {'set': {'FOO': 'foo'},
+ 'unset': ['BAR']}
}
}
}
@@ -72,15 +76,14 @@ configuration_conflicts = {
}
}
-from spack.test.mock_packages_test import MockPackagesTest
-
class TclTests(MockPackagesTest):
def setUp(self):
super(TclTests, self).setUp()
self.configuration_obj = spack.modules.CONFIGURATION
spack.modules.open = mock_open
- spack.modules.CONFIGURATION = None # Make sure that a non-mocked configuration will trigger an error
+ # Make sure that a non-mocked configuration will trigger an error
+ spack.modules.CONFIGURATION = None
def tearDown(self):
del spack.modules.open
@@ -98,7 +101,7 @@ class TclTests(MockPackagesTest):
spack.modules.CONFIGURATION = configuration_autoload_direct
spec = spack.spec.Spec('mpich@3.0.4=x86-linux')
content = self.get_modulefile_content(spec)
- self.assertTrue('module-whatis "mpich @3.0.4"' in content )
+ self.assertTrue('module-whatis "mpich @3.0.4"' in content)
def test_autoload(self):
spack.modules.CONFIGURATION = configuration_autoload_direct
@@ -117,14 +120,22 @@ class TclTests(MockPackagesTest):
spack.modules.CONFIGURATION = configuration_alter_environment
spec = spack.spec.Spec('mpileaks=x86-linux')
content = self.get_modulefile_content(spec)
- self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0)
- self.assertEqual(len([x for x in content if 'setenv FOO "foo"' in x]), 1)
+ self.assertEqual(
+ len([x
+ for x in content
+ if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0)
+ self.assertEqual(
+ len([x for x in content if 'setenv FOO "foo"' in x]), 1)
self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 1)
spec = spack.spec.Spec('libdwarf=x64-linux')
content = self.get_modulefile_content(spec)
- self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0)
- self.assertEqual(len([x for x in content if 'setenv FOO "foo"' in x]), 0)
+ self.assertEqual(
+ len([x
+ for x in content
+ if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0)
+ self.assertEqual(
+ len([x for x in content if 'setenv FOO "foo"' in x]), 0)
self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 0)
def test_blacklist(self):
@@ -138,6 +149,9 @@ class TclTests(MockPackagesTest):
spack.modules.CONFIGURATION = configuration_conflicts
spec = spack.spec.Spec('mpileaks=x86-linux')
content = self.get_modulefile_content(spec)
- self.assertEqual(len([x for x in content if x.startswith('conflict')]), 2)
- self.assertEqual(len([x for x in content if x == 'conflict mpileaks']), 1)
- self.assertEqual(len([x for x in content if x == 'conflict intel/14.0.1']), 1)
+ self.assertEqual(
+ len([x for x in content if x.startswith('conflict')]), 2)
+ self.assertEqual(
+ len([x for x in content if x == 'conflict mpileaks']), 1)
+ self.assertEqual(
+ len([x for x in content if x == 'conflict intel/14.0.1']), 1)