diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/spack/cmd/module.py | 16 | ||||
| -rw-r--r-- | lib/spack/spack/environment.py | 58 | ||||
| -rw-r--r-- | lib/spack/spack/modules.py | 196 | ||||
| -rw-r--r-- | lib/spack/spack/package.py | 353 | ||||
| -rw-r--r-- | lib/spack/spack/test/__init__.py | 70 | ||||
| -rw-r--r-- | lib/spack/spack/test/modules.py | 42 |
6 files changed, 391 insertions, 344 deletions
diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index f996f4eb84..cfe59c8d98 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -32,18 +32,21 @@ from llnl.util.filesystem import mkdirp from spack.modules import module_types from spack.util.string import * -description ="Manipulate modules and dotkits." +description = "Manipulate modules and dotkits." def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='module_command') - refresh_parser = sp.add_parser('refresh', help='Regenerate all module files.') + sp.add_parser('refresh', help='Regenerate all module files.') find_parser = sp.add_parser('find', help='Find module files for packages.') - find_parser.add_argument( - 'module_type', help="Type of module to find file for. [" + '|'.join(module_types) + "]") - find_parser.add_argument('spec', nargs='+', help='spec to find a module file for.') + find_parser.add_argument('module_type', + help="Type of module to find file for. [" + + '|'.join(module_types) + "]") + find_parser.add_argument('spec', + nargs='+', + help='spec to find a module file for.') def module_find(mtype, spec_array): @@ -53,7 +56,8 @@ def module_find(mtype, spec_array): should type to use that package's module. """ if mtype not in module_types: - tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types))) + tty.die("Invalid module type: '%s'. Options are %s" % + (mtype, comma_or(module_types))) specs = spack.cmd.parse_specs(spec_array) if len(specs) > 1: diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 92ab4e6bea..3fbe2531c1 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -1,7 +1,7 @@ -import os -import os.path import collections import inspect +import os +import os.path class NameModifier(object): @@ -26,7 +26,8 @@ class SetEnv(NameValueModifier): class UnsetEnv(NameModifier): def execute(self): - os.environ.pop(self.name, None) # Avoid throwing if the variable was not set + # Avoid throwing if the variable was not set + os.environ.pop(self.name, None) class SetPath(NameValueModifier): @@ -55,7 +56,9 @@ class RemovePath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split(':') if environment_value else [] - directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.value)] + directories = [os.path.normpath(x) + for x in directories + if x != os.path.normpath(self.value)] os.environ[self.name] = ':'.join(directories) @@ -63,7 +66,8 @@ class EnvironmentModifications(object): """ Keeps track of requests to modify the current environment. - Each call to a method to modify the environment stores the extra information on the caller in the request: + Each call to a method to modify the environment stores the extra + information on the caller in the request: - 'filename' : filename of the module where the caller is defined - 'lineno': line number where the request occurred - 'context' : line of code that issued the request that failed @@ -71,10 +75,10 @@ class EnvironmentModifications(object): def __init__(self, other=None): """ - Initializes a new instance, copying commands from other if it is not None + Initializes a new instance, copying commands from other if not None Args: - other: another instance of EnvironmentModifications from which (optional) + other: another instance of EnvironmentModifications (optional) """ self.env_modifications = [] if other is not None: @@ -93,7 +97,8 @@ class EnvironmentModifications(object): @staticmethod def _check_other(other): if not isinstance(other, EnvironmentModifications): - raise TypeError('other must be an instance of EnvironmentModifications') + raise TypeError( + 'other must be an instance of EnvironmentModifications') def _get_outside_caller_attributes(self): stack = inspect.stack() @@ -101,12 +106,10 @@ class EnvironmentModifications(object): _, filename, lineno, _, context, index = stack[2] context = context[index].strip() except Exception: - filename, lineno, context = 'unknown file', 'unknown line', 'unknown context' - args = { - 'filename': filename, - 'lineno': lineno, - 'context': context - } + filename = 'unknown file' + lineno = 'unknown line' + context = 'unknown context' + args = {'filename': filename, 'lineno': lineno, 'context': context} return args def set(self, name, value, **kwargs): @@ -170,7 +173,8 @@ class EnvironmentModifications(object): def remove_path(self, name, path, **kwargs): """ - Stores in the current object a request to remove a path from a path list + Stores in the current object a request to remove a path from a path + list Args: name: name of the path list in the environment @@ -185,7 +189,8 @@ class EnvironmentModifications(object): Returns a dict of the modifications grouped by variable name Returns: - dict mapping the environment variable name to the modifications to be done on it + dict mapping the environment variable name to the modifications to + be done on it """ modifications = collections.defaultdict(list) for item in self: @@ -203,7 +208,7 @@ class EnvironmentModifications(object): Applies the modifications and clears the list """ modifications = self.group_by_name() - # Apply the modifications to the environment variables one variable at a time + # Apply modifications one variable at a time for name, actions in sorted(modifications.items()): for x in actions: x.execute() @@ -224,13 +229,19 @@ def concatenate_paths(paths): def set_or_unset_not_first(variable, changes, errstream): """ - Check if we are going to set or unset something after other modifications have already been requested + Check if we are going to set or unset something after other modifications + have already been requested """ - indexes = [ii for ii, item in enumerate(changes) if ii != 0 and type(item) in [SetEnv, UnsetEnv]] + indexes = [ii + for ii, item in enumerate(changes) + if ii != 0 and type(item) in [SetEnv, UnsetEnv]] if indexes: good = '\t \t{context} at {filename}:{lineno}' nogood = '\t--->\t{context} at {filename}:{lineno}' - errstream('Suspicious requests to set or unset the variable \'{var}\' found'.format(var=variable)) + message = 'Suspicious requests to set or unset the variable \'{var}\' found' # NOQA: ignore=E501 + errstream( + message.format( + var=variable)) for ii, item in enumerate(changes): print_format = nogood if ii in indexes else good errstream(print_format.format(**item.args)) @@ -238,8 +249,8 @@ def set_or_unset_not_first(variable, changes, errstream): def validate(env, errstream): """ - Validates the environment modifications to check for the presence of suspicious patterns. Prompts a warning for - everything that was found + Validates the environment modifications to check for the presence of + suspicious patterns. Prompts a warning for everything that was found Current checks: - set or unset variables after other changes on the same variable @@ -254,7 +265,8 @@ def validate(env, errstream): def filter_environment_blacklist(env, variables): """ - Generator that filters out any change to environment variables present in the input list + Generator that filters out any change to environment variables present in + the input list Args: env: list of environment modifications diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index ffed469b20..53f054094a 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -23,36 +23,35 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## """ -This module contains code for creating environment modules, which can include dotkits, tcl modules, lmod, and others. +This module contains code for creating environment modules, which can include +dotkits, tcl modules, lmod, and others. -The various types of modules are installed by post-install hooks and removed after an uninstall by post-uninstall hooks. -This class consolidates the logic for creating an abstract description of the information that module systems need. -Currently that includes a number of directories to be appended to paths in the user's environment: +The various types of modules are installed by post-install hooks and removed +after an uninstall by post-uninstall hooks. This class consolidates the logic +for creating an abstract description of the information that module systems +need. - * /bin directories to be appended to PATH - * /lib* directories for LD_LIBRARY_PATH - * /include directories for CPATH - * /man* and /share/man* directories for MANPATH - * the package prefix for CMAKE_PREFIX_PATH +This module also includes logic for coming up with unique names for the module +files so that they can be found by the various shell-support files in +$SPACK/share/spack/setup-env.*. -This module also includes logic for coming up with unique names for the module files so that they can be found by the -various shell-support files in $SPACK/share/spack/setup-env.*. - -Each hook in hooks/ implements the logic for writing its specific type of module file. +Each hook in hooks/ implements the logic for writing its specific type of +module file. """ import copy import datetime import os import os.path import re -import textwrap import string +import textwrap import llnl.util.tty as tty import spack import spack.config from llnl.util.filesystem import join_path, mkdirp -from spack.build_environment import parent_class_modules, set_module_variables_for_package +from spack.build_environment import parent_class_modules +from spack.build_environment import set_module_variables_for_package from spack.environment import * __all__ = ['EnvModule', 'Dotkit', 'TclModule'] @@ -67,30 +66,26 @@ def print_help(): """ For use by commands to tell user how to activate shell support. """ - tty.msg("This command requires spack's shell integration.", - "", + tty.msg("This command requires spack's shell integration.", "", "To initialize spack's shell commands, you must run one of", "the commands below. Choose the right command for your shell.", - "", - "For bash and zsh:", - " . %s/setup-env.sh" % spack.share_path, - "", - "For csh and tcsh:", - " setenv SPACK_ROOT %s" % spack.prefix, - " source %s/setup-env.csh" % spack.share_path, - "") + "", "For bash and zsh:", + " . %s/setup-env.sh" % spack.share_path, "", + "For csh and tcsh:", " setenv SPACK_ROOT %s" % spack.prefix, + " source %s/setup-env.csh" % spack.share_path, "") def inspect_path(prefix): """ - Inspects the prefix of an installation to search for common layouts. Issues a request to modify the environment - accordingly when an item is found. + Inspects the prefix of an installation to search for common layouts. Issues + a request to modify the environment accordingly when an item is found. Args: prefix: prefix of the installation Returns: - instance of EnvironmentModifications containing the requested modifications + instance of EnvironmentModifications containing the requested + modifications """ env = EnvironmentModifications() # Inspect the prefix to check for the existence of common directories @@ -105,18 +100,21 @@ def inspect_path(prefix): def dependencies(spec, request='all'): """ - Returns the list of dependent specs for a given spec, according to the given request + Returns the list of dependent specs for a given spec, according to the + given request Args: spec: target spec request: either 'none', 'direct' or 'all' Returns: - empty list if 'none', direct dependency list if 'direct', all dependencies if 'all' + empty list if 'none', direct dependency list if 'direct', all + dependencies if 'all' """ if request not in ('none', 'direct', 'all'): - raise tty.error("Wrong value for argument 'request' : should be one of ('none', 'direct', 'all') " - " [current value is '%s']" % request) + message = "Wrong value for argument 'request' : " + message += "should be one of ('none', 'direct', 'all')" + raise tty.error(message + " [current value is '%s']" % request) if request == 'none': return [] @@ -124,12 +122,19 @@ def dependencies(spec, request='all'): if request == 'direct': return [xx for _, xx in spec.dependencies.items()] - # FIXME : during module file creation nodes seem to be visited multiple times even if cover='nodes' - # FIXME : is given. This work around permits to get a unique list of spec anyhow. - # FIXME : Possibly we miss a merge step among nodes that refer to the same package. + # FIXME : during module file creation nodes seem to be visited multiple + # FIXME : times even if cover='nodes' is given. This work around permits + # FIXME : to get a unique list of spec anyhow. Do we miss a merge + # FIXME : step among nodes that refer to the same package? seen = set() seen_add = seen.add - l = [xx for xx in sorted(spec.traverse(order='post', depth=True, cover='nodes', root=False), reverse=True)] + l = [xx + for xx in sorted( + spec.traverse(order='post', + depth=True, + cover='nodes', + root=False), + reverse=True)] return [xx for ii, xx in l if not (xx in seen or seen_add(xx))] @@ -146,7 +151,8 @@ def update_dictionary_extending_lists(target, update): def parse_config_options(module_generator): """ - Parse the configuration file and returns a bunch of items that will be needed during module file generation + Parse the configuration file and returns a bunch of items that will be + needed during module file generation Args: module_generator: module generator for a given spec @@ -154,11 +160,14 @@ def parse_config_options(module_generator): Returns: autoloads: list of specs to be autoloaded prerequisites: list of specs to be marked as prerequisite - filters: list of environment variables whose modification is blacklisted in module files - env: list of custom environment modifications to be applied in the module file + filters: list of environment variables whose modification is + blacklisted in module files + env: list of custom environment modifications to be applied in the + module file """ # Get the configuration for this kind of generator - module_configuration = copy.deepcopy(CONFIGURATION.get(module_generator.name, {})) + module_configuration = copy.deepcopy(CONFIGURATION.get( + module_generator.name, {})) ##### # Merge all the rules @@ -179,9 +188,12 @@ def parse_config_options(module_generator): ##### # Automatic loading loads - module_file_actions['autoload'] = dependencies(module_generator.spec, module_file_actions.get('autoload', 'none')) + module_file_actions['autoload'] = dependencies( + module_generator.spec, module_file_actions.get('autoload', 'none')) # Prerequisites - module_file_actions['prerequisites'] = dependencies(module_generator.spec, module_file_actions.get('prerequisites', 'none')) + module_file_actions['prerequisites'] = dependencies( + module_generator.spec, module_file_actions.get('prerequisites', + 'none')) # Environment modifications environment_actions = module_file_actions.pop('environment', {}) env = EnvironmentModifications() @@ -189,7 +201,7 @@ def parse_config_options(module_generator): def process_arglist(arglist): if method == 'unset': for x in arglist: - yield (x,) + yield (x, ) else: for x in arglist.iteritems(): yield x @@ -198,19 +210,13 @@ def parse_config_options(module_generator): for args in process_arglist(arglist): getattr(env, method)(*args) - # for item in arglist: - # if method == 'unset': - # args = [item] - # else: - # args = item.split(',') - # getattr(env, method)(*args) - return module_file_actions, env def filter_blacklisted(specs, module_name): """ - Given a sequence of specs, filters the ones that are blacklisted in the module configuration file. + Given a sequence of specs, filters the ones that are blacklisted in the + module configuration file. Args: specs: sequence of spec instances @@ -233,7 +239,8 @@ class EnvModule(object): class __metaclass__(type): def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) - if cls.name != 'env_module' and cls.name in CONFIGURATION['enable']: + if cls.name != 'env_module' and cls.name in CONFIGURATION[ + 'enable']: module_types[cls.name] = cls def __init__(self, spec=None): @@ -249,7 +256,8 @@ class EnvModule(object): # long description is the docstring with reduced whitespace. self.long_description = None if self.spec.package.__doc__: - self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) + self.long_description = re.sub(r'\s+', ' ', + self.spec.package.__doc__) @property def naming_scheme(self): @@ -271,12 +279,14 @@ class EnvModule(object): @property def use_name(self): """ - Subclasses should implement this to return the name the module command uses to refer to the package. + Subclasses should implement this to return the name the module command + uses to refer to the package. """ naming_tokens = self.tokens naming_scheme = self.naming_scheme name = naming_scheme.format(**naming_tokens) - name += '-' + self.spec.dag_hash() # Always append the hash to make the module file unique + name += '-' + self.spec.dag_hash( + ) # Always append the hash to make the module file unique # Not everybody is working on linux... parts = name.split('/') name = join_path(*parts) @@ -296,8 +306,12 @@ class EnvModule(object): @property def blacklisted(self): configuration = CONFIGURATION.get(self.name, {}) - whitelist_matches = [x for x in configuration.get('whitelist', []) if self.spec.satisfies(x)] - blacklist_matches = [x for x in configuration.get('blacklist', []) if self.spec.satisfies(x)] + whitelist_matches = [x + for x in configuration.get('whitelist', []) + if self.spec.satisfies(x)] + blacklist_matches = [x + for x in configuration.get('blacklist', []) + if self.spec.satisfies(x)] if whitelist_matches: message = '\tWHITELIST : %s [matches : ' % self.spec.cshort_spec for rule in whitelist_matches: @@ -327,7 +341,8 @@ class EnvModule(object): """ if self.blacklisted: return - tty.debug("\tWRITE : %s [%s]" % (self.spec.cshort_spec, self.file_name)) + tty.debug("\tWRITE : %s [%s]" % + (self.spec.cshort_spec, self.file_name)) module_dir = os.path.dirname(self.file_name) if not os.path.exists(module_dir): @@ -337,11 +352,12 @@ class EnvModule(object): # installation prefix env = inspect_path(self.spec.prefix) - # Let the extendee/dependency modify their extensions/dependencies before asking for - # package-specific modifications + # Let the extendee/dependency modify their extensions/dependencies + # before asking for package-specific modifications spack_env = EnvironmentModifications() - # TODO : the code down below is quite similar to build_environment.setup_package and needs to be - # TODO : factored out to a single place + # TODO : the code down below is quite similar to + # TODO : build_environment.setup_package and needs to be factored out + # TODO : to a single place for item in dependencies(self.spec, 'all'): package = self.spec[item.name].package modules = parent_class_modules(package.__class__) @@ -358,14 +374,18 @@ class EnvModule(object): # Parse configuration file module_configuration, conf_env = parse_config_options(self) env.extend(conf_env) - filters = module_configuration.get('filter', {}).get('environment_blacklist',{}) + filters = module_configuration.get('filter', {}).get( + 'environment_blacklist', {}) # Build up the module file content module_file_content = self.header - for x in filter_blacklisted(module_configuration.pop('autoload', []), self.name): + for x in filter_blacklisted( + module_configuration.pop('autoload', []), self.name): module_file_content += self.autoload(x) - for x in filter_blacklisted(module_configuration.pop('prerequisites', []), self.name): + for x in filter_blacklisted( + module_configuration.pop('prerequisites', []), self.name): module_file_content += self.prerequisite(x) - for line in self.process_environment_command(filter_environment_blacklist(env, filters)): + for line in self.process_environment_command( + filter_environment_blacklist(env, filters)): module_file_content += line for line in self.module_specific_content(module_configuration): module_file_content += line @@ -392,10 +412,13 @@ class EnvModule(object): def process_environment_command(self, env): for command in env: try: - yield self.environment_modifications_formats[type(command)].format(**command.args) + yield self.environment_modifications_formats[type( + command)].format(**command.args) except KeyError: - tty.warn('Cannot handle command of type {command} : skipping request'.format(command=type(command))) - tty.warn('{context} at {filename}:{lineno}'.format(**command.args)) + message = 'Cannot handle command of type {command} : skipping request' # NOQA: ignore=E501 + details = '{context} at {filename}:{lineno}' + tty.warn(message.format(command=type(command))) + tty.warn(details.format(**command.args)) @property def file_name(self): @@ -408,9 +431,12 @@ class EnvModule(object): if os.path.exists(mod_file): try: os.remove(mod_file) # Remove the module file - os.removedirs(os.path.dirname(mod_file)) # Remove all the empty directories from the leaf up + os.removedirs( + os.path.dirname(mod_file) + ) # Remove all the empty directories from the leaf up except OSError: - pass # removedirs throws OSError on first non-empty directory found + # removedirs throws OSError on first non-empty directory found + pass class Dotkit(EnvModule): @@ -424,13 +450,12 @@ class Dotkit(EnvModule): autoload_format = 'dk_op {module_file}\n' - prerequisite_format = None # TODO : does something like prerequisite exist for dotkit? - - default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' + default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' # NOQA: ignore=E501 @property def file_name(self): - return join_path(Dotkit.path, self.spec.architecture, '%s.dk' % self.use_name) + return join_path(Dotkit.path, self.spec.architecture, + '%s.dk' % self.use_name) @property def header(self): @@ -474,7 +499,7 @@ class TclModule(EnvModule): prerequisite_format = 'prereq {module_file}\n' - default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' + default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' # NOQA: ignore=E501 @property def file_name(self): @@ -482,9 +507,10 @@ class TclModule(EnvModule): @property def header(self): + timestamp = datetime.datetime.now() # TCL Modulefile header header = '#%Module1.0\n' - header += '## Module file created by spack (https://github.com/LLNL/spack) on %s\n' % datetime.datetime.now() + header += '## Module file created by spack (https://github.com/LLNL/spack) on %s\n' % timestamp # NOQA: ignore=E501 header += '##\n' header += '## %s\n' % self.spec.short_spec header += '##\n' @@ -509,16 +535,18 @@ class TclModule(EnvModule): f = string.Formatter() for item in conflict_format: line = 'conflict ' + item + '\n' - if len([x for x in f.parse(line)]) > 1: # We do have placeholder to substitute - for naming_dir, conflict_dir in zip(self.naming_scheme.split('/'), item.split('/')): + if len([x for x in f.parse(line) + ]) > 1: # We do have placeholder to substitute + for naming_dir, conflict_dir in zip( + self.naming_scheme.split('/'), item.split('/')): if naming_dir != conflict_dir: - message = 'conflict scheme does not match naming scheme [{spec}]\n\n' + message = 'conflict scheme does not match naming scheme [{spec}]\n\n' # NOQA: ignore=E501 message += 'naming scheme : "{nformat}"\n' message += 'conflict scheme : "{cformat}"\n\n' - message += '** You may want to check your `modules.yaml` configuration file **\n' - tty.error( - message.format(spec=self.spec, nformat=self.naming_scheme, cformat=item) - ) + message += '** You may want to check your `modules.yaml` configuration file **\n' # NOQA: ignore=E501 + tty.error(message.format(spec=self.spec, + nformat=self.naming_scheme, + cformat=item)) raise SystemExit('Module generation aborted.') line = line.format(**naming_tokens) yield line diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 3626a574c8..8e6cf32954 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -37,7 +37,6 @@ import os import re import textwrap import time -import glob import llnl.util.tty as tty import spack @@ -62,7 +61,6 @@ from spack.util.environment import dump_environment from spack.util.executable import ProcessError from spack.version import * from urlparse import urlparse - """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] @@ -305,26 +303,21 @@ class Package(object): # """By default we build in parallel. Subclasses can override this.""" parallel = True - """# jobs to use for parallel make. If set, overrides default of ncpus.""" make_jobs = None - - """Most packages are NOT extendable. Set to True if you want extensions.""" + """Most packages are NOT extendable. Set to True if you want extensions.""" extendable = False - """List of prefix-relative file paths (or a single path). If these do not exist after install, or if they exist but are not files, sanity checks fail. """ sanity_check_is_file = [] - """List of prefix-relative directory paths (or a single path). If these do not exist after install, or if they exist but are not directories, sanity checks will fail. """ sanity_check_is_dir = [] - def __init__(self, spec): # this determines how the package should be built. self.spec = spec @@ -336,23 +329,24 @@ class Package(object): self.name = self.name[self.name.rindex('.') + 1:] # Allow custom staging paths for packages - self.path=None + self.path = None # Sanity check attributes required by Spack directives. spack.directives.ensure_dicts(type(self)) # Check versions in the versions dict. for v in self.versions: - assert(isinstance(v, Version)) + assert (isinstance(v, Version)) # Check version descriptors for v in sorted(self.versions): - assert(isinstance(self.versions[v], dict)) + assert (isinstance(self.versions[v], dict)) # Version-ize the keys in versions dict try: - self.versions = dict((Version(v), h) for v,h in self.versions.items()) - except ValueError, e: + self.versions = dict((Version(v), h) + for v, h in self.versions.items()) + except ValueError as e: raise ValueError("In package %s: %s" % (self.name, e.message)) # stage used to build this package. @@ -366,9 +360,9 @@ class Package(object): # This makes self.url behave sanely. if self.spec.versions.concrete: # TODO: this is a really roundabout way of determining the type - # TODO: of fetch to do. figure out a more sane fetch strategy/package - # TODO: init order (right now it's conflated with stage, package, and - # TODO: the tests make assumptions) + # TODO: of fetch to do. figure out a more sane fetch + # TODO: strategy/package init order (right now it's conflated with + # TODO: stage, package, and the tests make assumptions) f = fs.for_package_version(self, self.version) if isinstance(f, fs.URLFetchStrategy): self.url = self.url_for_version(self.spec.version) @@ -387,14 +381,12 @@ class Package(object): if self.is_extension: spack.repo.get(self.extendee_spec)._check_extendable() - @property def version(self): if not self.spec.versions.concrete: raise ValueError("Can only get of package with concrete version.") return self.spec.versions[0] - @memoized def version_urls(self): """Return a list of URLs for different versions of this @@ -407,7 +399,6 @@ class Package(object): version_urls[v] = args['url'] return version_urls - def nearest_url(self, version): """Finds the URL for the next lowest version with a URL. If there is no lower version with a URL, uses the @@ -424,10 +415,11 @@ class Package(object): url = version_urls[v] return url - # TODO: move this out of here and into some URL extrapolation module? def url_for_version(self, version): - """Returns a URL that you can download a new version of this package from.""" + """ + Returns a URL that you can download a new version of this package from. + """ if not isinstance(version, Version): version = Version(version) @@ -441,14 +433,17 @@ class Package(object): return version_urls[version] # If we have no idea, try to substitute the version. - return spack.url.substitute_version(self.nearest_url(version), - self.url_version(version)) + return spack.url.substitute_version( + self.nearest_url(version), self.url_version(version)) def _make_resource_stage(self, root_stage, fetcher, resource): resource_stage_folder = self._resource_stage(resource) resource_mirror = join_path(self.name, os.path.basename(fetcher.url)) - stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource, - name=resource_stage_folder, mirror_path=resource_mirror, + stage = ResourceStage(resource.fetcher, + root=root_stage, + resource=resource, + name=resource_stage_folder, + mirror_path=resource_mirror, path=self.path) return stage @@ -474,7 +469,8 @@ class Package(object): else: # Construct resource stage resource = resources[ii - 1] # ii == 0 is root! - stage = self._make_resource_stage(composite_stage[0], fetcher, resource) + stage = self._make_resource_stage(composite_stage[0], fetcher, + resource) # Append the item to the composite composite_stage.append(stage) @@ -492,13 +488,11 @@ class Package(object): self._stage = self._make_stage() return self._stage - @stage.setter def stage(self, stage): """Allow a stage object to be set to override the default.""" self._stage = stage - def _make_fetcher(self): # Construct a composite fetcher that always contains at least # one element (the root package). In case there are resources @@ -515,7 +509,8 @@ class Package(object): @property def fetcher(self): if not self.spec.versions.concrete: - raise ValueError("Can only get a fetcher for a package with concrete versions.") + raise ValueError( + "Can only get a fetcher for a package with concrete versions.") if not self._fetcher: self._fetcher = self._make_fetcher() return self._fetcher @@ -524,10 +519,11 @@ class Package(object): def fetcher(self, f): self._fetcher = f - @property def extendee_spec(self): - """Spec of the extendee of this package, or None if it is not an extension.""" + """ + Spec of the extendee of this package, or None if it is not an extension + """ if not self.extendees: return None @@ -549,10 +545,11 @@ class Package(object): spec, kwargs = self.extendees[name] return spec - @property def extendee_args(self): - """Spec of the extendee of this package, or None if it is not an extension.""" + """ + Spec of the extendee of this package, or None if it is not an extension + """ if not self.extendees: return None @@ -560,7 +557,6 @@ class Package(object): name = next(iter(self.extendees)) return self.extendees[name][1] - @property def is_extension(self): # if it is concrete, it's only an extension if it actually @@ -571,22 +567,20 @@ class Package(object): # If not, then it's an extension if it *could* be an extension return bool(self.extendees) - def extends(self, spec): - if not spec.name in self.extendees: + if spec.name not in self.extendees: return False s = self.extendee_spec return s and s.satisfies(spec) - @property def activated(self): if not self.is_extension: - raise ValueError("is_extension called on package that is not an extension.") + raise ValueError( + "is_extension called on package that is not an extension.") exts = spack.install_layout.extension_map(self.extendee_spec) return (self.name in exts) and (exts[self.name] == self.spec) - def preorder_traversal(self, visited=None, **kwargs): """This does a preorder traversal of the package's dependence DAG.""" virtual = kwargs.get("virtual", False) @@ -605,36 +599,35 @@ class Package(object): spec = self.dependencies[name] # currently, we do not descend into virtual dependencies, as this - # makes doing a sensible traversal much harder. We just assume that - # ANY of the virtual deps will work, which might not be true (due to - # conflicts or unsatisfiable specs). For now this is ok but we might - # want to reinvestigate if we start using a lot of complicated virtual - # dependencies + # makes doing a sensible traversal much harder. We just assume + # that ANY of the virtual deps will work, which might not be true + # (due to conflicts or unsatisfiable specs). For now this is ok + # but we might want to reinvestigate if we start using a lot of + # complicated virtual dependencies # TODO: reinvestigate this. if spec.virtual: if virtual: yield spec continue - for pkg in spack.repo.get(name).preorder_traversal(visited, **kwargs): + for pkg in spack.repo.get(name).preorder_traversal(visited, + **kwargs): yield pkg - def provides(self, vpkg_name): - """True if this package provides a virtual package with the specified name.""" + """ + True if this package provides a virtual package with the specified name + """ return any(s.name == vpkg_name for s in self.provided) - def virtual_dependencies(self, visited=None): for spec in sorted(set(self.preorder_traversal(virtual=True))): yield spec - @property def installed(self): return os.path.isdir(self.prefix) - @property def installed_dependents(self): """Return a list of the specs of all installed packages that depend @@ -651,60 +644,62 @@ class Package(object): dependents.append(spec) return dependents - @property def prefix(self): """Get the prefix into which this package should be installed.""" return self.spec.prefix - @property def compiler(self): - """Get the spack.compiler.Compiler object used to build this package.""" + """Get the spack.compiler.Compiler object used to build this package""" if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") return spack.compilers.compiler_for_spec(self.spec.compiler) - def url_version(self, version): - """Given a version, this returns a string that should be substituted into the - package's URL to download that version. - By default, this just returns the version string. Subclasses may need to - override this, e.g. for boost versions where you need to ensure that there - are _'s in the download URL. """ - return str(version) + Given a version, this returns a string that should be substituted + into the package's URL to download that version. + By default, this just returns the version string. Subclasses may need + to override this, e.g. for boost versions where you need to ensure that + there are _'s in the download URL. + """ + return str(version) def remove_prefix(self): - """Removes the prefix for a package along with any empty parent directories.""" + """ + Removes the prefix for a package along with any empty parent + directories + """ spack.install_layout.remove_install_directory(self.spec) - def do_fetch(self, mirror_only=False): - """Creates a stage directory and downloads the tarball for this package. - Working directory will be set to the stage directory. + """ + Creates a stage directory and downloads the tarball for this package. + Working directory will be set to the stage directory. """ if not self.spec.concrete: raise ValueError("Can only fetch concrete packages.") start_time = time.time() - if spack.do_checksum and not self.version in self.versions: - tty.warn("There is no checksum on file to fetch %s safely." - % self.spec.format('$_$@')) + if spack.do_checksum and self.version not in self.versions: + tty.warn("There is no checksum on file to fetch %s safely." % + self.spec.format('$_$@')) # Ask the user whether to skip the checksum if we're # interactive, but just fail if non-interactive. - checksum_msg = "Add a checksum or use --no-checksum to skip this check." + checksum_msg = "Add a checksum or use --no-checksum to skip this check." # NOQA: ignore=E501 ignore_checksum = False if sys.stdout.isatty(): - ignore_checksum = tty.get_yes_or_no(" Fetch anyway?", default=False) + ignore_checksum = tty.get_yes_or_no(" Fetch anyway?", + default=False) if ignore_checksum: tty.msg("Fetching with no checksum.", checksum_msg) if not ignore_checksum: - raise FetchError( - "Will not fetch %s" % self.spec.format('$_$@'), checksum_msg) + raise FetchError("Will not fetch %s" % + self.spec.format('$_$@'), checksum_msg) self.stage.fetch(mirror_only) @@ -723,7 +718,6 @@ class Package(object): self.stage.expand_archive() self.stage.chdir_to_source() - def do_patch(self): """Calls do_stage(), then applied patches to the expanded tarball if they haven't been applied already.""" @@ -743,10 +737,10 @@ class Package(object): # Construct paths to special files in the archive dir used to # keep track of whether patches were successfully applied. - archive_dir = self.stage.source_path - good_file = join_path(archive_dir, '.spack_patched') + archive_dir = self.stage.source_path + good_file = join_path(archive_dir, '.spack_patched') no_patches_file = join_path(archive_dir, '.spack_no_patches') - bad_file = join_path(archive_dir, '.spack_patch_failed') + bad_file = join_path(archive_dir, '.spack_patch_failed') # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. @@ -801,13 +795,11 @@ class Package(object): else: touch(no_patches_file) - @property def namespace(self): namespace, dot, module = self.__module__.rpartition('.') return namespace - def do_fake_install(self): """Make a fake install directory contaiing a 'fake' file in bin.""" mkdirp(self.prefix.bin) @@ -815,15 +807,15 @@ class Package(object): mkdirp(self.prefix.lib) mkdirp(self.prefix.man1) - def _get_needed_resources(self): resources = [] # Select the resources that are needed for this build for when_spec, resource_list in self.resources.items(): if when_spec in self.spec: resources.extend(resource_list) - # Sorts the resources by the length of the string representing their destination. Since any nested resource - # must contain another resource's name in its path, it seems that should work + # Sorts the resources by the length of the string representing their + # destination. Since any nested resource must contain another + # resource's name in its path, it seems that should work resources = sorted(resources, key=lambda res: len(res.destination)) return resources @@ -832,10 +824,14 @@ class Package(object): resource_stage_folder = '-'.join(pieces) return resource_stage_folder - def do_install(self, - keep_prefix=False, keep_stage=False, ignore_deps=False, - skip_patch=False, verbose=False, make_jobs=None, fake=False): + keep_prefix=False, + keep_stage=False, + ignore_deps=False, + skip_patch=False, + verbose=False, + make_jobs=None, + fake=False): """Called by commands to install a package and its dependencies. Package implementations should override install() to describe @@ -846,18 +842,20 @@ class Package(object): keep_stage -- By default, stage is destroyed only if there are no exceptions during build. Set to True to keep the stage even with exceptions. - ignore_deps -- Do not install dependencies before installing this package. + ignore_deps -- Don't install dependencies before installing this + package fake -- Don't really build -- install fake stub files instead. skip_patch -- Skip patch stage of build if True. verbose -- Display verbose build output (by default, suppresses it) - make_jobs -- Number of make jobs to use for install. Default is ncpus. + make_jobs -- Number of make jobs to use for install. Default is ncpus """ if not self.spec.concrete: raise ValueError("Can only install concrete packages.") # No installation needed if package is external if self.spec.external: - tty.msg("%s is externally installed in %s" % (self.name, self.spec.external)) + tty.msg("%s is externally installed in %s" % + (self.name, self.spec.external)) return # Ensure package is not already installed @@ -869,9 +867,13 @@ class Package(object): # First, install dependencies recursively. if not ignore_deps: - self.do_install_dependencies( - keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps, - fake=fake, skip_patch=skip_patch, verbose=verbose, make_jobs=make_jobs) + self.do_install_dependencies(keep_prefix=keep_prefix, + keep_stage=keep_stage, + ignore_deps=ignore_deps, + fake=fake, + skip_patch=skip_patch, + verbose=verbose, + make_jobs=make_jobs) # Set parallelism before starting build. self.make_jobs = make_jobs @@ -899,35 +901,41 @@ class Package(object): self.do_fake_install() else: # Do the real install in the source directory. - self.stage.chdir_to_source() + self.stage.chdir_to_source() - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') - try: - # Redirect I/O to a build log (and optionally to the terminal) + try: + # Redirect I/O to a build log (and optionally to + # the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), True): + with log_output(log_file, verbose, sys.stdout.isatty(), + True): dump_environment(env_path) self.install(self.spec, self.prefix) - except ProcessError as e: - # Annotate ProcessErrors with the location of the build log. - e.build_log = log_path - raise e + except ProcessError as e: + # Annotate ProcessErrors with the location of + # the build log + e.build_log = log_path + raise e - # Ensure that something was actually installed. - self.sanity_check_prefix() + # Ensure that something was actually installed. + self.sanity_check_prefix() - # Copy provenance into the install directory on success - log_install_path = spack.install_layout.build_log_path(self.spec) - env_install_path = spack.install_layout.build_env_path(self.spec) - packages_dir = spack.install_layout.build_packages_path(self.spec) + # Copy provenance into the install directory on success + log_install_path = spack.install_layout.build_log_path( + self.spec) + env_install_path = spack.install_layout.build_env_path( + self.spec) + packages_dir = spack.install_layout.build_packages_path( + self.spec) - install(log_path, log_install_path) - install(env_path, env_install_path) - dump_packages(self.spec, packages_dir) + install(log_path, log_install_path) + install(env_path, env_install_path) + dump_packages(self.spec, packages_dir) # Run post install hooks before build stage is removed. spack.hooks.post_install(self) @@ -937,8 +945,9 @@ class Package(object): build_time = self._total_time - self._fetch_time tty.msg("Successfully installed %s" % self.name, - "Fetch: %s. Build: %s. Total: %s." - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + "Fetch: %s. Build: %s. Total: %s." % + (_hms(self._fetch_time), _hms(build_time), + _hms(self._total_time))) print_pkg(self.prefix) try: @@ -953,16 +962,17 @@ class Package(object): tty.warn("Keeping install prefix in place despite error.", "Spack will think this package is installed. " + "Manually remove this directory to fix:", - self.prefix, wrap=True) + self.prefix, + wrap=True) raise # note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. spack.installed_db.add(self.spec, self.prefix) - def sanity_check_prefix(self): """This function checks whether install succeeded.""" + def check_paths(path_list, filetype, predicate): if isinstance(path_list, basestring): path_list = [path_list] @@ -970,8 +980,9 @@ class Package(object): for path in path_list: abs_path = os.path.join(self.prefix, path) if not predicate(abs_path): - raise InstallError("Install failed for %s. No such %s in prefix: %s" - % (self.name, filetype, path)) + raise InstallError( + "Install failed for %s. No such %s in prefix: %s" % + (self.name, filetype, path)) check_paths(self.sanity_check_is_file, 'file', os.path.isfile) check_paths(self.sanity_check_is_dir, 'directory', os.path.isdir) @@ -982,13 +993,11 @@ class Package(object): raise InstallError( "Install failed for %s. Nothing was installed!" % self.name) - def do_install_dependencies(self, **kwargs): # Pass along paths of dependencies here for dep in self.spec.dependencies.values(): dep.package.do_install(**kwargs) - @property def build_log_path(self): if self.installed: @@ -996,7 +1005,6 @@ class Package(object): else: return join_path(self.stage.source_path, 'spack-build.out') - @property def module(self): """Use this to add variables to the class's module's scope. @@ -1037,7 +1045,6 @@ class Package(object): """ pass - def setup_dependent_environment(self, spack_env, run_env, dependent_spec): """Set up the environment of packages that depend on this one. @@ -1077,7 +1084,6 @@ class Package(object): """ self.setup_environment(spack_env, run_env) - def setup_dependent_package(self, module, dependent_spec): """Set up Python module-scope variables for dependent packages. @@ -1123,8 +1129,11 @@ class Package(object): pass def install(self, spec, prefix): - """Package implementations override this with their own build configuration.""" - raise InstallError("Package %s provides no install method!" % self.name) + """ + Package implementations override this with their own configuration + """ + raise InstallError("Package %s provides no install method!" % + self.name) def do_uninstall(self, force=False): if not self.installed: @@ -1146,12 +1155,10 @@ class Package(object): # Once everything else is done, run post install hooks spack.hooks.post_uninstall(self) - def _check_extendable(self): if not self.extendable: raise ValueError("Package %s is not extendable!" % self.name) - def _sanity_check_extension(self): if not self.is_extension: raise ActivationError("This package is not an extension.") @@ -1160,12 +1167,13 @@ class Package(object): extendee_package._check_extendable() if not extendee_package.installed: - raise ActivationError("Can only (de)activate extensions for installed packages.") + raise ActivationError( + "Can only (de)activate extensions for installed packages.") if not self.installed: raise ActivationError("Extensions must first be installed.") - if not self.extendee_spec.name in self.extendees: - raise ActivationError("%s does not extend %s!" % (self.name, self.extendee.name)) - + if self.extendee_spec.name not in self.extendees: + raise ActivationError("%s does not extend %s!" % + (self.name, self.extendee.name)) def do_activate(self, force=False): """Called on an etension to invoke the extendee's activate method. @@ -1175,8 +1183,8 @@ class Package(object): """ self._sanity_check_extension() - spack.install_layout.check_extension_conflict( - self.extendee_spec, self.spec) + spack.install_layout.check_extension_conflict(self.extendee_spec, + self.spec) # Activate any package dependencies that are also extensions. if not force: @@ -1188,9 +1196,8 @@ class Package(object): self.extendee_spec.package.activate(self, **self.extendee_args) spack.install_layout.add_extension(self.extendee_spec, self.spec) - tty.msg("Activated extension %s for %s" - % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) - + tty.msg("Activated extension %s for %s" % + (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) def activate(self, extension, **kwargs): """Symlinks all files from the extension into extendee's install dir. @@ -1201,6 +1208,7 @@ class Package(object): always executed. """ + def ignore(filename): return (filename in spack.install_layout.hidden_file_paths or kwargs.get('ignore', lambda f: False)(filename)) @@ -1212,7 +1220,6 @@ class Package(object): tree.merge(self.prefix, ignore=ignore) - def do_deactivate(self, **kwargs): """Called on the extension to invoke extendee's deactivate() method.""" self._sanity_check_extension() @@ -1230,7 +1237,7 @@ class Package(object): for dep in aspec.traverse(): if self.spec == dep: raise ActivationError( - "Cannot deactivate %s beacuse %s is activated and depends on it." + "Cannot deactivate %s because %s is activated and depends on it." # NOQA: ignore=E501 % (self.spec.short_spec, aspec.short_spec)) self.extendee_spec.package.deactivate(self, **self.extendee_args) @@ -1238,11 +1245,11 @@ class Package(object): # redundant activation check -- makes SURE the spec is not # still activated even if something was wrong above. if self.activated: - spack.install_layout.remove_extension(self.extendee_spec, self.spec) - - tty.msg("Deactivated extension %s for %s" - % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) + spack.install_layout.remove_extension(self.extendee_spec, + self.spec) + tty.msg("Deactivated extension %s for %s" % + (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) def deactivate(self, extension, **kwargs): """Unlinks all files from extension out of this package's install dir. @@ -1253,6 +1260,7 @@ class Package(object): always executed. """ + def ignore(filename): return (filename in spack.install_layout.hidden_file_paths or kwargs.get('ignore', lambda f: False)(filename)) @@ -1260,17 +1268,14 @@ class Package(object): tree = LinkTree(extension.prefix) tree.unmerge(self.prefix, ignore=ignore) - def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() - def do_clean(self): """Removes the package's build stage and source tarball.""" self.stage.destroy() - def format_doc(self, **kwargs): """Wrap doc string at 72 characters and format nicely""" indent = kwargs.get('indent', 0) @@ -1285,7 +1290,6 @@ class Package(object): results.write((" " * indent) + line + "\n") return results.getvalue() - @property def all_urls(self): urls = [] @@ -1297,7 +1301,6 @@ class Package(object): urls.append(args['url']) return urls - def fetch_remote_versions(self): """Try to find remote versions of this package using the list_url and any other URLs described in the package file.""" @@ -1306,26 +1309,30 @@ class Package(object): try: return spack.util.web.find_versions_of_archive( - *self.all_urls, list_url=self.list_url, list_depth=self.list_depth) + *self.all_urls, + list_url=self.list_url, + list_depth=self.list_depth) except spack.error.NoNetworkConnectionError as e: - tty.die("Package.fetch_versions couldn't connect to:", - e.url, e.message) - + tty.die("Package.fetch_versions couldn't connect to:", e.url, + e.message) @property def rpath(self): """Get the rpath this package links with, as a list of paths.""" rpaths = [self.prefix.lib, self.prefix.lib64] - rpaths.extend(d.prefix.lib for d in self.spec.traverse(root=False) + rpaths.extend(d.prefix.lib + for d in self.spec.traverse(root=False) if os.path.isdir(d.prefix.lib)) - rpaths.extend(d.prefix.lib64 for d in self.spec.traverse(root=False) + rpaths.extend(d.prefix.lib64 + for d in self.spec.traverse(root=False) if os.path.isdir(d.prefix.lib64)) return rpaths - @property def rpath_args(self): - """Get the rpath args as a string, with -Wl,-rpath, for each element.""" + """ + Get the rpath args as a string, with -Wl,-rpath, for each element + """ return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) @@ -1333,6 +1340,7 @@ def install_dependency_symlinks(pkg, spec, prefix): """Execute a dummy install and flatten dependencies""" flatten_dependencies(spec, prefix) + def flatten_dependencies(spec, flat_dir): """Make each dependency of spec present in dir via symlink.""" for dep in spec.traverse(root=False): @@ -1341,13 +1349,13 @@ def flatten_dependencies(spec, flat_dir): dep_path = spack.install_layout.path_for_spec(dep) dep_files = LinkTree(dep_path) - os.mkdir(flat_dir+'/'+name) + os.mkdir(flat_dir + '/' + name) - conflict = dep_files.find_conflict(flat_dir+'/'+name) + conflict = dep_files.find_conflict(flat_dir + '/' + name) if conflict: raise DependencyConflictError(conflict) - dep_files.merge(flat_dir+'/'+name) + dep_files.merge(flat_dir + '/' + name) def validate_package_url(url_string): @@ -1388,9 +1396,11 @@ def dump_packages(spec, path): # Create a source repo and get the pkg directory out of it. try: source_repo = spack.repository.Repo(source_repo_root) - source_pkg_dir = source_repo.dirname_for_package_name(node.name) - except RepoError as e: - tty.warn("Warning: Couldn't copy in provenance for %s" % node.name) + source_pkg_dir = source_repo.dirname_for_package_name( + node.name) + except RepoError: + tty.warn("Warning: Couldn't copy in provenance for %s" % + node.name) # Create a destination repository dest_repo_root = join_path(path, node.namespace) @@ -1410,7 +1420,7 @@ def print_pkg(message): """Outputs a message with a package icon.""" from llnl.util.tty.color import cwrite cwrite('@*g{[+]} ') - print message + print(message) def _hms(seconds): @@ -1419,20 +1429,25 @@ def _hms(seconds): h, m = divmod(m, 60) parts = [] - if h: parts.append("%dh" % h) - if m: parts.append("%dm" % m) - if s: parts.append("%.2fs" % s) + if h: + parts.append("%dh" % h) + if m: + parts.append("%dm" % m) + if s: + parts.append("%.2fs" % s) return ' '.join(parts) class FetchError(spack.error.SpackError): """Raised when something goes wrong during fetch.""" + def __init__(self, message, long_msg=None): super(FetchError, self).__init__(message, long_msg) class InstallError(spack.error.SpackError): """Raised when something goes wrong during install or uninstall.""" + def __init__(self, message, long_msg=None): super(InstallError, self).__init__(message, long_msg) @@ -1443,21 +1458,24 @@ class ExternalPackageError(InstallError): class PackageStillNeededError(InstallError): """Raised when package is still needed by another on uninstall.""" + def __init__(self, spec, dependents): - super(PackageStillNeededError, self).__init__( - "Cannot uninstall %s" % spec) + super(PackageStillNeededError, self).__init__("Cannot uninstall %s" % + spec) self.spec = spec self.dependents = dependents class PackageError(spack.error.SpackError): """Raised when something is wrong with a package definition.""" + def __init__(self, message, long_msg=None): super(PackageError, self).__init__(message, long_msg) class PackageVersionError(PackageError): """Raised when a version URL cannot automatically be determined.""" + def __init__(self, version): super(PackageVersionError, self).__init__( "Cannot determine a URL automatically for version %s" % version, @@ -1466,6 +1484,7 @@ class PackageVersionError(PackageError): class VersionFetchError(PackageError): """Raised when a version URL cannot automatically be determined.""" + def __init__(self, cls): super(VersionFetchError, self).__init__( "Cannot fetch versions for package %s " % cls.__name__ + @@ -1474,12 +1493,15 @@ class VersionFetchError(PackageError): class NoURLError(PackageError): """Raised when someone tries to build a URL for a package with no URLs.""" + def __init__(self, cls): super(NoURLError, self).__init__( "Package %s has no version with a URL." % cls.__name__) -class ExtensionError(PackageError): pass +class ExtensionError(PackageError): + + pass class ExtensionConflictError(ExtensionError): @@ -1495,7 +1517,8 @@ class ActivationError(ExtensionError): class DependencyConflictError(spack.error.SpackError): """Raised when the dependencies cannot be flattened as asked for.""" + def __init__(self, conflict): super(DependencyConflictError, self).__init__( - "%s conflicts with another file in the flattened directory." %( + "%s conflicts with another file in the flattened directory." % ( conflict)) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 05f58ab7b1..10eaac1344 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -23,53 +23,23 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -import unittest -import nose -from spack.test.tally_plugin import Tally -from llnl.util.filesystem import join_path import llnl.util.tty as tty -from llnl.util.tty.colify import colify - +import nose import spack - +from llnl.util.filesystem import join_path +from llnl.util.tty.colify import colify +from spack.test.tally_plugin import Tally """Names of tests to be included in Spack's test suite""" -test_names = ['versions', - 'url_parse', - 'url_substitution', - 'packages', - 'stage', - 'spec_syntax', - 'spec_semantics', - 'spec_dag', - 'concretize', - 'multimethod', - 'install', - 'package_sanity', - 'config', - 'directory_layout', - 'pattern', - 'python_version', - 'git_fetch', - 'svn_fetch', - 'hg_fetch', - 'mirror', - 'modules', - 'url_extrapolate', - 'cc', - 'link_tree', - 'spec_yaml', - 'optional_deps', - 'make_executable', - 'configure_guess', - 'lock', - 'database', - 'namespace_trie', - 'yaml', - 'sbang', - 'environment', - 'cmd.uninstall', - 'cmd.test_install'] +test_names = ['versions', 'url_parse', 'url_substitution', 'packages', 'stage', + 'spec_syntax', 'spec_semantics', 'spec_dag', 'concretize', + 'multimethod', 'install', 'package_sanity', 'config', + 'directory_layout', 'pattern', 'python_version', 'git_fetch', + 'svn_fetch', 'hg_fetch', 'mirror', 'modules', 'url_extrapolate', + 'cc', 'link_tree', 'spec_yaml', 'optional_deps', + 'make_executable', 'configure_guess', 'lock', 'database', + 'namespace_trie', 'yaml', 'sbang', 'environment', + 'cmd.uninstall', 'cmd.test_install'] def list_tests(): @@ -80,8 +50,6 @@ def list_tests(): def run(names, outputDir, verbose=False): """Run tests with the supplied names. Names should be a list. If it's empty, run ALL of Spack's tests.""" - verbosity = 1 if not verbose else 2 - if not names: names = test_names else: @@ -95,7 +63,7 @@ def run(names, outputDir, verbose=False): tally = Tally() for test in names: module = 'spack.test.' + test - print module + print(module) tty.msg("Running test: %s" % test) @@ -105,15 +73,13 @@ def run(names, outputDir, verbose=False): xmlOutputFname = "unittests-{0}.xml".format(test) xmlOutputPath = join_path(outputDir, xmlOutputFname) runOpts += ["--with-xunit", - "--xunit-file={0}".format(xmlOutputPath)] + "--xunit-file={0}".format(xmlOutputPath)] argv = [""] + runOpts + [module] - result = nose.run(argv=argv, addplugins=[tally]) + nose.run(argv=argv, addplugins=[tally]) succeeded = not tally.failCount and not tally.errorCount - tty.msg("Tests Complete.", - "%5d tests run" % tally.numberOfTestsRun, - "%5d failures" % tally.failCount, - "%5d errors" % tally.errorCount) + tty.msg("Tests Complete.", "%5d tests run" % tally.numberOfTestsRun, + "%5d failures" % tally.failCount, "%5d errors" % tally.errorCount) if succeeded: tty.info("OK", format='g') diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index b8b0d6fc6a..c65d663250 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -2,14 +2,18 @@ import collections from contextlib import contextmanager import StringIO +import spack.modules +from spack.test.mock_packages_test import MockPackagesTest FILE_REGISTRY = collections.defaultdict(StringIO.StringIO) + # Monkey-patch open to write module files to a StringIO instance @contextmanager def mock_open(filename, mode): if not mode == 'w': - raise RuntimeError('test.modules : unexpected opening mode for monkey-patched open') + raise RuntimeError( + 'test.modules : unexpected opening mode for monkey-patched open') FILE_REGISTRY[filename] = StringIO.StringIO() @@ -20,7 +24,6 @@ def mock_open(filename, mode): FILE_REGISTRY[filename] = handle.getvalue() handle.close() -import spack.modules configuration_autoload_direct = { 'enable': ['tcl'], @@ -47,7 +50,8 @@ configuration_alter_environment = { 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']} }, '=x86-linux': { - 'environment': {'set': {'FOO': 'foo'}, 'unset': ['BAR']} + 'environment': {'set': {'FOO': 'foo'}, + 'unset': ['BAR']} } } } @@ -72,15 +76,14 @@ configuration_conflicts = { } } -from spack.test.mock_packages_test import MockPackagesTest - class TclTests(MockPackagesTest): def setUp(self): super(TclTests, self).setUp() self.configuration_obj = spack.modules.CONFIGURATION spack.modules.open = mock_open - spack.modules.CONFIGURATION = None # Make sure that a non-mocked configuration will trigger an error + # Make sure that a non-mocked configuration will trigger an error + spack.modules.CONFIGURATION = None def tearDown(self): del spack.modules.open @@ -98,7 +101,7 @@ class TclTests(MockPackagesTest): spack.modules.CONFIGURATION = configuration_autoload_direct spec = spack.spec.Spec('mpich@3.0.4=x86-linux') content = self.get_modulefile_content(spec) - self.assertTrue('module-whatis "mpich @3.0.4"' in content ) + self.assertTrue('module-whatis "mpich @3.0.4"' in content) def test_autoload(self): spack.modules.CONFIGURATION = configuration_autoload_direct @@ -117,14 +120,22 @@ class TclTests(MockPackagesTest): spack.modules.CONFIGURATION = configuration_alter_environment spec = spack.spec.Spec('mpileaks=x86-linux') content = self.get_modulefile_content(spec) - self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) - self.assertEqual(len([x for x in content if 'setenv FOO "foo"' in x]), 1) + self.assertEqual( + len([x + for x in content + if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) + self.assertEqual( + len([x for x in content if 'setenv FOO "foo"' in x]), 1) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 1) spec = spack.spec.Spec('libdwarf=x64-linux') content = self.get_modulefile_content(spec) - self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) - self.assertEqual(len([x for x in content if 'setenv FOO "foo"' in x]), 0) + self.assertEqual( + len([x + for x in content + if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) + self.assertEqual( + len([x for x in content if 'setenv FOO "foo"' in x]), 0) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 0) def test_blacklist(self): @@ -138,6 +149,9 @@ class TclTests(MockPackagesTest): spack.modules.CONFIGURATION = configuration_conflicts spec = spack.spec.Spec('mpileaks=x86-linux') content = self.get_modulefile_content(spec) - self.assertEqual(len([x for x in content if x.startswith('conflict')]), 2) - self.assertEqual(len([x for x in content if x == 'conflict mpileaks']), 1) - self.assertEqual(len([x for x in content if x == 'conflict intel/14.0.1']), 1) + self.assertEqual( + len([x for x in content if x.startswith('conflict')]), 2) + self.assertEqual( + len([x for x in content if x == 'conflict mpileaks']), 1) + self.assertEqual( + len([x for x in content if x == 'conflict intel/14.0.1']), 1) |