diff options
| -rw-r--r-- | etc/spack/defaults/config.yaml | 13 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/module.py | 3 | ||||
| -rw-r--r-- | lib/spack/spack/modules.py | 36 | ||||
| -rw-r--r-- | lib/spack/spack/schema/config.py | 10 | ||||
| -rw-r--r-- | lib/spack/spack/test/modules.py | 48 |
5 files changed, 69 insertions, 41 deletions
diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 6a749b5f6d..85019ede61 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -14,17 +14,18 @@ # ~/.spack/config.yaml # ------------------------------------------------------------------------- config: - - # Precedence of configuration scopes, high to low. - # The user can override this paradoxically. - # scopes: [user, spack, default] - - # This is the path to the root of the Spack install tree. # You can use $spack here to refer to the root of the spack instance. install_tree: $spack/opt/spack + # Locations where different types of modules should be installed. + module_roots: + tcl: $spack/share/spack/modules + lmod: $spack/share/spack/lmod + dotkit: $spack/share/spack/dotkit + + # Temporary locations Spack can try to use for builds. # # Spack will use the first one it finds that exists and is writable. diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index c6fa84109e..d7abe0fa87 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -210,7 +210,8 @@ def refresh(mtype, specs, args): cls = module_types[mtype] # Detect name clashes - writers = [cls(spec) for spec in specs] + writers = [cls(spec) for spec in specs + if spack.repo.exists(spec.name)] # skip unknown packages. file2writer = collections.defaultdict(list) for item in writers: file2writer[item.file_name].append(item) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index cf67ca905d..2c2804aea1 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -48,10 +48,12 @@ import string import textwrap import llnl.util.tty as tty +from llnl.util.filesystem import join_path, mkdirp + import spack import spack.compilers # Needed by LmodModules import spack.config -from llnl.util.filesystem import join_path, mkdirp +from spack.util.path import canonicalize_path from spack.build_environment import parent_class_modules from spack.build_environment import set_module_variables_for_package from spack.environment import * @@ -62,7 +64,11 @@ __all__ = ['EnvModule', 'Dotkit', 'TclModule'] metaclass.""" module_types = {} -CONFIGURATION = spack.config.get_config('modules') +"""Module install roots are in config.yaml.""" +_roots = spack.config.get_config('config').get('module_roots', {}) + +"""Specifics about modules are in modules.yaml""" +_module_config = spack.config.get_config('modules') def print_help(): @@ -92,7 +98,7 @@ def inspect_path(prefix): """ env = EnvironmentModifications() # Inspect the prefix to check for the existence of common directories - prefix_inspections = CONFIGURATION.get('prefix_inspections', {}) + prefix_inspections = _module_config.get('prefix_inspections', {}) for relative_path, variables in prefix_inspections.items(): expected = join_path(prefix, relative_path) if os.path.isdir(expected): @@ -168,7 +174,7 @@ def parse_config_options(module_generator): module file """ # Get the configuration for this kind of generator - module_configuration = copy.deepcopy(CONFIGURATION.get( + module_configuration = copy.deepcopy(_module_config.get( module_generator.name, {})) ##### @@ -248,7 +254,7 @@ class EnvModule(object): def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) - if cls.name != 'env_module' and cls.name in CONFIGURATION[ + if cls.name != 'env_module' and cls.name in _module_config[ 'enable']: module_types[cls.name] = cls @@ -271,7 +277,7 @@ class EnvModule(object): @property def naming_scheme(self): try: - naming_scheme = CONFIGURATION[self.name]['naming_scheme'] + naming_scheme = _module_config[self.name]['naming_scheme'] except KeyError: naming_scheme = self.default_naming_format return naming_scheme @@ -314,7 +320,7 @@ class EnvModule(object): @property def blacklisted(self): - configuration = CONFIGURATION.get(self.name, {}) + configuration = _module_config.get(self.name, {}) whitelist_matches = [x for x in configuration.get('whitelist', []) if self.spec.satisfies(x)] @@ -473,7 +479,9 @@ class EnvModule(object): class Dotkit(EnvModule): name = 'dotkit' - path = join_path(spack.share_path, 'dotkit') + path = canonicalize_path( + _roots.get(name, join_path(spack.share_path, name))) + environment_modifications_formats = { PrependPath: 'dk_alter {name} {value}\n', RemovePath: 'dk_unalter {name} {value}\n', @@ -516,7 +524,8 @@ class Dotkit(EnvModule): class TclModule(EnvModule): name = 'tcl' - path = join_path(spack.share_path, "modules") + path = canonicalize_path( + _roots.get(name, join_path(spack.share_path, 'modules'))) autoload_format = ('if ![ is-loaded {module_file} ] {{\n' ' puts stderr "Autoloading {module_file}"\n' @@ -635,7 +644,8 @@ class TclModule(EnvModule): class LmodModule(EnvModule): name = 'lmod' - path = join_path(spack.share_path, "lmod") + path = canonicalize_path( + _roots.get(name, join_path(spack.share_path, name))) environment_modifications_formats = { PrependPath: 'prepend_path("{name}", "{value}")\n', @@ -656,6 +666,12 @@ class LmodModule(EnvModule): path_part_without_hash = join_path('{token.name}', '{token.version}') + # TODO : Check that extra tokens specified in configuration file + # TODO : are actually virtual dependencies + configuration = _module_config.get('lmod', {}) + hierarchy_tokens = configuration.get('hierarchical_scheme', []) + hierarchy_tokens = hierarchy_tokens + ['mpi', 'compiler'] + def __init__(self, spec=None): super(LmodModule, self).__init__(spec) diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index 7ae3b75c3e..a8057f303c 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -42,6 +42,16 @@ schema = { {'type': 'array', 'items': {'type': 'string'}}], }, + 'module_roots': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'properties': { + 'tcl': {'type': 'string'}, + 'lmod': {'type': 'string'}, + 'dotkit': {'type': 'string'}, + }, + }, 'source_cache': {'type': 'string'}, 'misc_cache': {'type': 'string'}, 'verify_ssl': { diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index cdaeef3339..cb3a26e62b 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -102,18 +102,18 @@ class ModuleFileGeneratorTests(MockPackagesTest): def setUp(self): super(ModuleFileGeneratorTests, self).setUp() - self.configuration_instance = spack.modules.CONFIGURATION + self.configuration_instance = spack.modules._module_config self.module_types_instance = spack.modules.module_types spack.modules.open = mock_open spack.modules.mkdirp = lambda x: None # Make sure that a non-mocked configuration will trigger an error - spack.modules.CONFIGURATION = None + spack.modules._module_config = None spack.modules.module_types = {self.factory.name: self.factory} def tearDown(self): del spack.modules.open spack.modules.module_types = self.module_types_instance - spack.modules.CONFIGURATION = self.configuration_instance + spack.modules._module_config = self.configuration_instance spack.modules.mkdirp = llnl.util.filesystem.mkdirp super(ModuleFileGeneratorTests, self).tearDown() @@ -231,7 +231,7 @@ class TclTests(ModuleFileGeneratorTests): } def test_simple_case(self): - spack.modules.CONFIGURATION = self.configuration_autoload_direct + spack.modules._module_config = self.configuration_autoload_direct spec = spack.spec.Spec(mpich_spec_string) content = self.get_modulefile_content(spec) self.assertTrue('module-whatis "mpich @3.0.4"' in content) @@ -239,13 +239,13 @@ class TclTests(ModuleFileGeneratorTests): spec, 'non-existing-tag') def test_autoload(self): - spack.modules.CONFIGURATION = self.configuration_autoload_direct + spack.modules._module_config = self.configuration_autoload_direct spec = spack.spec.Spec(mpileaks_spec_string) content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2) self.assertEqual(len([x for x in content if 'module load ' in x]), 2) - spack.modules.CONFIGURATION = self.configuration_autoload_all + spack.modules._module_config = self.configuration_autoload_all spec = spack.spec.Spec(mpileaks_spec_string) content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5) @@ -256,7 +256,7 @@ class TclTests(ModuleFileGeneratorTests): # - 1 ('build','link') dependency # - 1 ('build',) dependency # Just make sure the 'build' dependency is not there - spack.modules.CONFIGURATION = self.configuration_autoload_direct + spack.modules._module_config = self.configuration_autoload_direct spec = spack.spec.Spec('dtbuild1') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2) @@ -267,25 +267,25 @@ class TclTests(ModuleFileGeneratorTests): # - 1 ('build','link') dependency # - 1 ('build',) dependency # Just make sure the 'build' dependency is not there - spack.modules.CONFIGURATION = self.configuration_autoload_all + spack.modules._module_config = self.configuration_autoload_all spec = spack.spec.Spec('dtbuild1') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2) self.assertEqual(len([x for x in content if 'module load ' in x]), 2) def test_prerequisites(self): - spack.modules.CONFIGURATION = self.configuration_prerequisites_direct + spack.modules._module_config = self.configuration_prerequisites_direct spec = spack.spec.Spec('mpileaks arch=x86-linux') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'prereq' in x]), 2) - spack.modules.CONFIGURATION = self.configuration_prerequisites_all + spack.modules._module_config = self.configuration_prerequisites_all spec = spack.spec.Spec('mpileaks arch=x86-linux') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'prereq' in x]), 5) def test_alter_environment(self): - spack.modules.CONFIGURATION = self.configuration_alter_environment + spack.modules._module_config = self.configuration_alter_environment spec = spack.spec.Spec('mpileaks platform=test target=x86_64') content = self.get_modulefile_content(spec) self.assertEqual( @@ -315,7 +315,7 @@ class TclTests(ModuleFileGeneratorTests): len([x for x in content if 'setenv LIBDWARF_ROOT' in x]), 1) def test_blacklist(self): - spack.modules.CONFIGURATION = self.configuration_blacklist + spack.modules._module_config = self.configuration_blacklist spec = spack.spec.Spec('mpileaks ^zmpi') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) @@ -329,7 +329,7 @@ class TclTests(ModuleFileGeneratorTests): self.assertEqual(len([x for x in content if 'module load ' in x]), 1) def test_conflicts(self): - spack.modules.CONFIGURATION = self.configuration_conflicts + spack.modules._module_config = self.configuration_conflicts spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual( @@ -339,11 +339,11 @@ class TclTests(ModuleFileGeneratorTests): self.assertEqual( len([x for x in content if x == 'conflict intel/14.0.1']), 1) - spack.modules.CONFIGURATION = self.configuration_wrong_conflicts + spack.modules._module_config = self.configuration_wrong_conflicts self.assertRaises(SystemExit, self.get_modulefile_content, spec) def test_suffixes(self): - spack.modules.CONFIGURATION = self.configuration_suffix + spack.modules._module_config = self.configuration_suffix spec = spack.spec.Spec('mpileaks+debug arch=x86-linux') spec.concretize() generator = spack.modules.TclModule(spec) @@ -412,7 +412,7 @@ class LmodTests(ModuleFileGeneratorTests): } def test_simple_case(self): - spack.modules.CONFIGURATION = self.configuration_autoload_direct + spack.modules._module_config = self.configuration_autoload_direct spec = spack.spec.Spec(mpich_spec_string) content = self.get_modulefile_content(spec) self.assertTrue('-- -*- lua -*-' in content) @@ -420,14 +420,14 @@ class LmodTests(ModuleFileGeneratorTests): self.assertTrue('whatis([[Version : 3.0.4]])' in content) def test_autoload(self): - spack.modules.CONFIGURATION = self.configuration_autoload_direct + spack.modules._module_config = self.configuration_autoload_direct spec = spack.spec.Spec(mpileaks_spec_string) content = self.get_modulefile_content(spec) self.assertEqual( len([x for x in content if 'if not isloaded(' in x]), 2) self.assertEqual(len([x for x in content if 'load(' in x]), 2) - spack.modules.CONFIGURATION = self.configuration_autoload_all + spack.modules._module_config = self.configuration_autoload_all spec = spack.spec.Spec(mpileaks_spec_string) content = self.get_modulefile_content(spec) self.assertEqual( @@ -435,7 +435,7 @@ class LmodTests(ModuleFileGeneratorTests): self.assertEqual(len([x for x in content if 'load(' in x]), 5) def test_alter_environment(self): - spack.modules.CONFIGURATION = self.configuration_alter_environment + spack.modules._module_config = self.configuration_alter_environment spec = spack.spec.Spec('mpileaks platform=test target=x86_64') content = self.get_modulefile_content(spec) self.assertEqual( @@ -460,7 +460,7 @@ class LmodTests(ModuleFileGeneratorTests): len([x for x in content if 'unsetenv("BAR")' in x]), 0) def test_blacklist(self): - spack.modules.CONFIGURATION = self.configuration_blacklist + spack.modules._module_config = self.configuration_blacklist spec = spack.spec.Spec(mpileaks_spec_string) content = self.get_modulefile_content(spec) self.assertEqual( @@ -499,14 +499,14 @@ class DotkitTests(MockPackagesTest): def setUp(self): super(DotkitTests, self).setUp() - self.configuration_obj = spack.modules.CONFIGURATION + self.configuration_obj = spack.modules._module_config spack.modules.open = mock_open # Make sure that a non-mocked configuration will trigger an error - spack.modules.CONFIGURATION = None + spack.modules._module_config = None def tearDown(self): del spack.modules.open - spack.modules.CONFIGURATION = self.configuration_obj + spack.modules._module_config = self.configuration_obj super(DotkitTests, self).tearDown() def get_modulefile_content(self, spec): @@ -517,7 +517,7 @@ class DotkitTests(MockPackagesTest): return content def test_dotkit(self): - spack.modules.CONFIGURATION = self.configuration_dotkit + spack.modules._module_config = self.configuration_dotkit spec = spack.spec.Spec('mpileaks arch=x86-linux') content = self.get_modulefile_content(spec) self.assertTrue('#c spack' in content) |