summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/spack/config.py78
-rw-r--r--lib/spack/spack/environment.py16
-rw-r--r--lib/spack/spack/modules.py184
3 files changed, 241 insertions, 37 deletions
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index 14e5aaf4fb..6b5b3dfd62 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -146,7 +146,7 @@ section_schemas = {
'type': 'object',
'additionalProperties': False,
'patternProperties': {
- 'compilers:?': { # optional colon for overriding site config.
+ 'compilers:?': { # optional colon for overriding site config.
'type': 'object',
'default': {},
'additionalProperties': False,
@@ -195,6 +195,7 @@ section_schemas = {
'default': [],
'items': {
'type': 'string'},},},},
+
'packages': {
'$schema': 'http://json-schema.org/schema#',
'title': 'Spack package configuration file schema',
@@ -238,11 +239,69 @@ section_schemas = {
'default' : {},
}
},},},},},},
+
'modules': {
'$schema': 'http://json-schema.org/schema#',
'title': 'Spack module file configuration file schema',
'type': 'object',
'additionalProperties': False,
+ 'definitions': {
+ 'array_of_strings': {
+ 'type': 'array',
+ 'default': [],
+ 'items': {
+ 'type': 'string'
+ }
+ },
+ 'dependency_selection': {
+ 'type': 'string',
+ 'enum': ['none', 'direct', 'all']
+ },
+ 'module_file_configuration': {
+ 'type': 'object',
+ 'default': {},
+ 'additionalProperties': False,
+ 'properties': {
+ 'filter': {
+ 'type': 'object',
+ 'default': {},
+ 'additionalProperties': False,
+ 'properties': {
+ 'environment_blacklist': {
+ 'type': 'array',
+ 'default': [],
+ 'items': {
+ 'type': 'string'
+ }
+ }
+ }
+ },
+ 'autoload': {'$ref': '#/definitions/dependency_selection'},
+ 'prerequisites': {'$ref': '#/definitions/dependency_selection'},
+ 'environment': {
+ 'type': 'object',
+ 'default': {},
+ 'additionalProperties': False,
+ 'properties': {
+ 'set': {'$ref': '#/definitions/array_of_strings'},
+ 'unset': {'$ref': '#/definitions/array_of_strings'},
+ 'prepend_path': {'$ref': '#/definitions/array_of_strings'},
+ 'append_path': {'$ref': '#/definitions/array_of_strings'}
+ }
+ }
+ }
+ },
+ 'module_type_configuration': {
+ 'type': 'object',
+ 'default': {},
+ 'properties': {
+ 'all': {'$ref': '#/definitions/module_file_configuration'}
+ },
+ 'patternProperties': {
+ r'\w[\w-]*': {'$ref': '#/definitions/module_file_configuration'}
+ }
+ }
+ },
'patternProperties': {
r'modules:?': {
'type': 'object',
@@ -253,9 +312,22 @@ section_schemas = {
'type': 'array',
'default': [],
'items': {
- 'type': 'string'
+ 'type': 'string',
+ 'enum': ['tcl', 'dotkit']
}
- }
+ },
+ 'tcl': {
+ 'allOf': [
+ {'$ref': '#/definitions/module_type_configuration'}, # Base configuration
+ {} # Specific tcl extensions
+ ]
+ },
+ 'dotkit': {
+ 'allOf': [
+ {'$ref': '#/definitions/module_type_configuration'}, # Base configuration
+ {} # Specific dotkit extensions
+ ]
+ },
}
},
},
diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py
index 72aafa4e2d..92ab4e6bea 100644
--- a/lib/spack/spack/environment.py
+++ b/lib/spack/spack/environment.py
@@ -250,3 +250,19 @@ def validate(env, errstream):
modifications = env.group_by_name()
for variable, list_of_changes in sorted(modifications.items()):
set_or_unset_not_first(variable, list_of_changes, errstream)
+
+
+def filter_environment_blacklist(env, variables):
+ """
+ Generator that filters out any change to environment variables present in the input list
+
+ Args:
+ env: list of environment modifications
+ variables: list of variable names to be filtered
+
+ Yields:
+ items in env if they are not in variables
+ """
+ for item in env:
+ if item.name not in variables:
+ yield item
diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py
index 61624fbd70..747c0d3be5 100644
--- a/lib/spack/spack/modules.py
+++ b/lib/spack/spack/modules.py
@@ -45,11 +45,14 @@ import os.path
import re
import shutil
import textwrap
+import copy
import llnl.util.tty as tty
import spack
import spack.config
+
from llnl.util.filesystem import join_path, mkdirp
+from spack.build_environment import parent_class_modules, set_module_variables_for_package
from spack.environment import *
__all__ = ['EnvModule', 'Dotkit', 'TclModule']
@@ -111,6 +114,81 @@ def inspect_path(prefix):
return env
+def dependencies(spec, request='All'):
+ if request == 'None':
+ return []
+
+ l = [xx for xx in
+ sorted(spec.traverse(order='post', depth=True, cover='nodes', root=False), reverse=True)]
+
+ if request == 'Direct':
+ return [xx for ii, xx in l if ii == 1]
+
+ # FIXME : during module file creation nodes seem to be visited multiple times even if cover='nodes'
+ # FIXME : is given. This work around permits to get a unique list of spec anyhow.
+ # FIXME : Possibly we miss a merge step among nodes that refer to the same package.
+ seen = set()
+ seen_add = seen.add
+ return [xx for ii, xx in l if not (xx in seen or seen_add(xx))]
+
+
+def parse_config_options(module_generator):
+ autoloads, prerequisites, filters = [], [], []
+ env = EnvironmentModifications()
+ # Get the configuration for this kind of generator
+ try:
+ module_configuration = copy.copy(CONFIGURATION[module_generator.name])
+ except KeyError:
+ return autoloads, prerequisites, filters, env
+
+ # Get the defaults for all packages
+ all_conf = module_configuration.pop('all', {})
+
+ update_single(module_generator.spec, all_conf, autoloads, prerequisites, filters, env)
+
+ for spec, conf in module_configuration.items():
+ override = False
+ if spec.endswith(':'):
+ spec = spec.strip(':')
+ override = True
+ if module_generator.spec.satisfies(spec):
+ if override:
+ autoloads, prerequisites, filters = [], [], []
+ env = EnvironmentModifications()
+ update_single(module_generator.spec, conf, autoloads, prerequisites, filters, env)
+
+ return autoloads, prerequisites, filters, env
+
+
+def update_single(spec, configuration, autoloads, prerequisites, filters, env):
+ # Get list of modules that will be loaded automatically
+ try:
+ autoloads.extend(dependencies(spec, configuration['autoload']))
+ except KeyError:
+ pass
+ try:
+ prerequisites.extend(dependencies(spec, configuration['prerequisites']))
+ except KeyError:
+ pass
+
+ # Filter modifications to environment variables
+ try:
+ filters.extend(configuration['filter']['environment_blacklist'])
+ except KeyError:
+ pass
+
+ try:
+ for method, arglist in configuration['environment'].items():
+ for item in arglist:
+ if method == 'unset':
+ args = [item]
+ else:
+ args = item.split(',')
+ getattr(env, method)(*args)
+ except KeyError:
+ pass
+
+
class EnvModule(object):
name = 'env_module'
formats = {}
@@ -136,7 +214,6 @@ class EnvModule(object):
if self.spec.package.__doc__:
self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__)
-
@property
def category(self):
# Anything defined at the package level takes precedence
@@ -148,9 +225,14 @@ class EnvModule(object):
# Not very descriptive fallback
return 'spack installed package'
-
def write(self):
- """Write out a module file for this object."""
+ """
+ Writes out a module file for this object.
+
+ This method employs a template pattern and expects derived classes to:
+ - override the header property
+ - provide formats for autoload, prerequisites and environment changes
+ """
module_dir = os.path.dirname(self.file_name)
if not os.path.exists(module_dir):
mkdirp(module_dir)
@@ -159,43 +241,61 @@ class EnvModule(object):
# installation prefix
env = inspect_path(self.spec.prefix)
- # Let the extendee modify their extensions before asking for
+ # Let the extendee/dependency modify their extensions/dependencies before asking for
# package-specific modifications
spack_env = EnvironmentModifications()
- for item in self.pkg.extendees:
- try:
- package = self.spec[item].package
- package.setup_dependent_package(self.pkg.module, self.spec)
- package.setup_dependent_environment(spack_env, env, self.spec)
- except:
- # The extends was conditional, so it doesn't count here
- # eg: extends('python', when='+python')
- pass
+ # TODO : the code down below is quite similar to build_environment.setup_package and needs to be
+ # TODO : factored out to a single place
+ for item in dependencies(self.spec, 'All'):
+ package = self.spec[item.name].package
+ modules = parent_class_modules(package.__class__)
+ for mod in modules:
+ set_module_variables_for_package(package, mod)
+ set_module_variables_for_package(package, package.module)
+ package.setup_dependent_package(self.pkg.module, self.spec)
+ package.setup_dependent_environment(spack_env, env, self.spec)
# Package-specific environment modifications
+ set_module_variables_for_package(self.pkg, self.pkg.module)
self.spec.package.setup_environment(spack_env, env)
- # TODO : implement site-specific modifications and filters
- if not env:
- return
+ # Parse configuration file
+ autoloads, prerequisites, filters, conf_env = parse_config_options(self)
+ env.extend(conf_env)
+
+ # Build up the module file content
+ module_file_content = self.header
+ for x in autoloads:
+ module_file_content += self.autoload(x)
+ for x in prerequisites:
+ module_file_content += self.prerequisite(x)
+ for line in self.process_environment_command(filter_environment_blacklist(env, filters)):
+ module_file_content += line
+ # Dump to file
with open(self.file_name, 'w') as f:
- self.write_header(f)
- for line in self.process_environment_command(env):
- f.write(line)
+ f.write(module_file_content)
- def write_header(self, stream):
+ @property
+ def header(self):
raise NotImplementedError()
+ def autoload(self, spec):
+ m = TclModule(spec)
+ return self.autoload_format.format(module_file=m.use_name)
+
+ def prerequisite(self, spec):
+ m = TclModule(spec)
+ return self.prerequisite_format.format(module_file=m.use_name)
+
def process_environment_command(self, env):
for command in env:
try:
- yield self.formats[type(command)].format(**command.args)
+ yield self.environment_modifications_formats[type(command)].format(**command.args)
except KeyError:
tty.warn('Cannot handle command of type {command} : skipping request'.format(command=type(command)))
tty.warn('{context} at {filename}:{lineno}'.format(**command.args))
-
@property
def file_name(self):
"""Subclasses should implement this to return the name of the file
@@ -222,11 +322,15 @@ class Dotkit(EnvModule):
name = 'dotkit'
path = join_path(spack.share_path, "dotkit")
- formats = {
+ environment_modifications_formats = {
PrependPath: 'dk_alter {name} {value}\n',
SetEnv: 'dk_setenv {name} {value}\n'
}
+ autoload_format = 'dk_op {module_file}\n' # TODO : Check this line
+
+ prerequisite_format = None # TODO : does something like prerequisite exist for dotkit?
+
@property
def file_name(self):
return join_path(Dotkit.path, self.spec.architecture, '%s.dk' % self.use_name)
@@ -238,26 +342,29 @@ class Dotkit(EnvModule):
self.spec.compiler.version,
self.spec.dag_hash())
- def write_header(self, dk_file):
+ @property
+ def header(self):
# Category
+ header = ''
if self.category:
- dk_file.write('#c %s\n' % self.category)
+ header += '#c %s\n' % self.category
# Short description
if self.short_description:
- dk_file.write('#d %s\n' % self.short_description)
+ header += '#d %s\n' % self.short_description
# Long description
if self.long_description:
for line in textwrap.wrap(self.long_description, 72):
- dk_file.write("#h %s\n" % line)
+ header += '#h %s\n' % line
+ return header
class TclModule(EnvModule):
name = 'tcl'
path = join_path(spack.share_path, "modules")
- formats = {
+ environment_modifications_formats = {
PrependPath: 'prepend-path {name} \"{value}\"\n',
AppendPath: 'append-path {name} \"{value}\"\n',
RemovePath: 'remove-path {name} \"{value}\"\n',
@@ -265,6 +372,13 @@ class TclModule(EnvModule):
UnsetEnv: 'unsetenv {name}\n'
}
+ autoload_format = ('if ![ is-loaded {module_file} ] {{'
+ ' puts stderr "Autoloading {module_file}"'
+ ' module load {module_file}'
+ '}}')
+
+ prerequisite_format = 'prereq {module_file}\n'
+
@property
def file_name(self):
return join_path(TclModule.path, self.spec.architecture, self.use_name)
@@ -276,17 +390,19 @@ class TclModule(EnvModule):
self.spec.compiler.version,
self.spec.dag_hash())
- def write_header(self, module_file):
+ @property
+ def header(self):
# TCL Modulefile header
- module_file.write('#%Module1.0\n')
+ header = '#%Module1.0\n'
# TODO : category ?
# Short description
if self.short_description:
- module_file.write('module-whatis \"%s\"\n\n' % self.short_description)
+ header += 'module-whatis \"%s\"\n\n' % self.short_description
# Long description
if self.long_description:
- module_file.write('proc ModulesHelp { } {\n')
+ header += 'proc ModulesHelp { } {\n'
for line in textwrap.wrap(self.long_description, 72):
- module_file.write("puts stderr \"%s\"\n" % line)
- module_file.write('}\n\n')
+ header += 'puts stderr "%s"\n' % line
+ header += '}\n\n'
+ return header
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-10-23 01:07:03 +0000