Add module_roots to config.yaml

4 files changed, 62 insertions, 35 deletions

diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py
index c6fa84109e..d7abe0fa87 100644
--- a/lib/spack/spack/cmd/module.py
+++ b/lib/spack/spack/cmd/module.py
@@ -210,7 +210,8 @@ def refresh(mtype, specs, args):
     cls = module_types[mtype]
 
     # Detect name clashes
-    writers = [cls(spec) for spec in specs]
+    writers = [cls(spec) for spec in specs
+               if spack.repo.exists(spec.name)]  # skip unknown packages.
     file2writer = collections.defaultdict(list)
     for item in writers:
         file2writer[item.file_name].append(item)
diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py
index cf67ca905d..2c2804aea1 100644
--- a/lib/spack/spack/modules.py
+++ b/lib/spack/spack/modules.py
@@ -48,10 +48,12 @@ import string
 import textwrap
 
 import llnl.util.tty as tty
+from llnl.util.filesystem import join_path, mkdirp
+
 import spack
 import spack.compilers  # Needed by LmodModules
 import spack.config
-from llnl.util.filesystem import join_path, mkdirp
+from spack.util.path import canonicalize_path
 from spack.build_environment import parent_class_modules
 from spack.build_environment import set_module_variables_for_package
 from spack.environment import *
@@ -62,7 +64,11 @@ __all__ = ['EnvModule', 'Dotkit', 'TclModule']
    metaclass."""
 module_types = {}
 
-CONFIGURATION = spack.config.get_config('modules')
+"""Module install roots are in config.yaml."""
+_roots = spack.config.get_config('config').get('module_roots', {})
+
+"""Specifics about modules are in modules.yaml"""
+_module_config = spack.config.get_config('modules')
 
 
 def print_help():
@@ -92,7 +98,7 @@ def inspect_path(prefix):
     """
     env = EnvironmentModifications()
     # Inspect the prefix to check for the existence of common directories
-    prefix_inspections = CONFIGURATION.get('prefix_inspections', {})
+    prefix_inspections = _module_config.get('prefix_inspections', {})
     for relative_path, variables in prefix_inspections.items():
         expected = join_path(prefix, relative_path)
         if os.path.isdir(expected):
@@ -168,7 +174,7 @@ def parse_config_options(module_generator):
         module file
     """
     # Get the configuration for this kind of generator
-    module_configuration = copy.deepcopy(CONFIGURATION.get(
+    module_configuration = copy.deepcopy(_module_config.get(
         module_generator.name, {}))
 
     #####
@@ -248,7 +254,7 @@ class EnvModule(object):
 
         def __init__(cls, name, bases, dict):
             type.__init__(cls, name, bases, dict)
-            if cls.name != 'env_module' and cls.name in CONFIGURATION[
+            if cls.name != 'env_module' and cls.name in _module_config[
                     'enable']:
                 module_types[cls.name] = cls
 
@@ -271,7 +277,7 @@ class EnvModule(object):
     @property
     def naming_scheme(self):
         try:
-            naming_scheme = CONFIGURATION[self.name]['naming_scheme']
+            naming_scheme = _module_config[self.name]['naming_scheme']
         except KeyError:
             naming_scheme = self.default_naming_format
         return naming_scheme
@@ -314,7 +320,7 @@ class EnvModule(object):
 
     @property
     def blacklisted(self):
-        configuration = CONFIGURATION.get(self.name, {})
+        configuration = _module_config.get(self.name, {})
         whitelist_matches = [x
                              for x in configuration.get('whitelist', [])
                              if self.spec.satisfies(x)]
@@ -473,7 +479,9 @@ class EnvModule(object):
 
 class Dotkit(EnvModule):
     name = 'dotkit'
-    path = join_path(spack.share_path, 'dotkit')
+    path = canonicalize_path(
+        _roots.get(name, join_path(spack.share_path, name)))
+
     environment_modifications_formats = {
         PrependPath: 'dk_alter {name} {value}\n',
         RemovePath: 'dk_unalter {name} {value}\n',
@@ -516,7 +524,8 @@ class Dotkit(EnvModule):
 
 class TclModule(EnvModule):
     name = 'tcl'
-    path = join_path(spack.share_path, "modules")
+    path = canonicalize_path(
+        _roots.get(name, join_path(spack.share_path, 'modules')))
 
     autoload_format = ('if ![ is-loaded {module_file} ] {{\n'
                        '    puts stderr "Autoloading {module_file}"\n'
@@ -635,7 +644,8 @@ class TclModule(EnvModule):
 
 class LmodModule(EnvModule):
     name = 'lmod'
-    path = join_path(spack.share_path, "lmod")
+    path = canonicalize_path(
+        _roots.get(name, join_path(spack.share_path, name)))
 
     environment_modifications_formats = {
         PrependPath: 'prepend_path("{name}", "{value}")\n',
@@ -656,6 +666,12 @@ class LmodModule(EnvModule):
 
     path_part_without_hash = join_path('{token.name}', '{token.version}')
 
+    # TODO : Check that extra tokens specified in configuration file
+    # TODO : are actually virtual dependencies
+    configuration = _module_config.get('lmod', {})
+    hierarchy_tokens = configuration.get('hierarchical_scheme', [])
+    hierarchy_tokens = hierarchy_tokens + ['mpi', 'compiler']
+
     def __init__(self, spec=None):
         super(LmodModule, self).__init__(spec)
 
diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py
index 7ae3b75c3e..a8057f303c 100644
--- a/lib/spack/spack/schema/config.py
+++ b/lib/spack/spack/schema/config.py
@@ -42,6 +42,16 @@ schema = {
                         {'type': 'array',
                          'items': {'type': 'string'}}],
                 },
+                'module_roots': {
+                    'type': 'object',
+                    'default': {},
+                    'additionalProperties': False,
+                    'properties': {
+                        'tcl': {'type': 'string'},
+                        'lmod': {'type': 'string'},
+                        'dotkit': {'type': 'string'},
+                    },
+                },
                 'source_cache': {'type': 'string'},
                 'misc_cache': {'type': 'string'},
                 'verify_ssl': {
diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py
index cdaeef3339..cb3a26e62b 100644
--- a/lib/spack/spack/test/modules.py
+++ b/lib/spack/spack/test/modules.py
@@ -102,18 +102,18 @@ class ModuleFileGeneratorTests(MockPackagesTest):
 
     def setUp(self):
         super(ModuleFileGeneratorTests, self).setUp()
-        self.configuration_instance = spack.modules.CONFIGURATION
+        self.configuration_instance = spack.modules._module_config
         self.module_types_instance = spack.modules.module_types
         spack.modules.open = mock_open
         spack.modules.mkdirp = lambda x: None
         # Make sure that a non-mocked configuration will trigger an error
-        spack.modules.CONFIGURATION = None
+        spack.modules._module_config = None
         spack.modules.module_types = {self.factory.name: self.factory}
 
     def tearDown(self):
         del spack.modules.open
         spack.modules.module_types = self.module_types_instance
-        spack.modules.CONFIGURATION = self.configuration_instance
+        spack.modules._module_config = self.configuration_instance
         spack.modules.mkdirp = llnl.util.filesystem.mkdirp
         super(ModuleFileGeneratorTests, self).tearDown()
 
@@ -231,7 +231,7 @@ class TclTests(ModuleFileGeneratorTests):
     }
 
     def test_simple_case(self):
-        spack.modules.CONFIGURATION = self.configuration_autoload_direct
+        spack.modules._module_config = self.configuration_autoload_direct
         spec = spack.spec.Spec(mpich_spec_string)
         content = self.get_modulefile_content(spec)
         self.assertTrue('module-whatis "mpich @3.0.4"' in content)
@@ -239,13 +239,13 @@ class TclTests(ModuleFileGeneratorTests):
                           spec, 'non-existing-tag')
 
     def test_autoload(self):
-        spack.modules.CONFIGURATION = self.configuration_autoload_direct
+        spack.modules._module_config = self.configuration_autoload_direct
         spec = spack.spec.Spec(mpileaks_spec_string)
         content = self.get_modulefile_content(spec)
         self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2)
         self.assertEqual(len([x for x in content if 'module load ' in x]), 2)
 
-        spack.modules.CONFIGURATION = self.configuration_autoload_all
+        spack.modules._module_config = self.configuration_autoload_all
         spec = spack.spec.Spec(mpileaks_spec_string)
         content = self.get_modulefile_content(spec)
         self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5)
@@ -256,7 +256,7 @@ class TclTests(ModuleFileGeneratorTests):
         # - 1 ('build','link') dependency
         # - 1 ('build',) dependency
         # Just make sure the 'build' dependency is not there
-        spack.modules.CONFIGURATION = self.configuration_autoload_direct
+        spack.modules._module_config = self.configuration_autoload_direct
         spec = spack.spec.Spec('dtbuild1')
         content = self.get_modulefile_content(spec)
         self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2)
@@ -267,25 +267,25 @@ class TclTests(ModuleFileGeneratorTests):
         # - 1 ('build','link') dependency
         # - 1 ('build',) dependency
         # Just make sure the 'build' dependency is not there
-        spack.modules.CONFIGURATION = self.configuration_autoload_all
+        spack.modules._module_config = self.configuration_autoload_all
         spec = spack.spec.Spec('dtbuild1')
         content = self.get_modulefile_content(spec)
         self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2)
         self.assertEqual(len([x for x in content if 'module load ' in x]), 2)
 
     def test_prerequisites(self):
-        spack.modules.CONFIGURATION = self.configuration_prerequisites_direct
+        spack.modules._module_config = self.configuration_prerequisites_direct
         spec = spack.spec.Spec('mpileaks arch=x86-linux')
         content = self.get_modulefile_content(spec)
         self.assertEqual(len([x for x in content if 'prereq' in x]), 2)
 
-        spack.modules.CONFIGURATION = self.configuration_prerequisites_all
+        spack.modules._module_config = self.configuration_prerequisites_all
         spec = spack.spec.Spec('mpileaks arch=x86-linux')
         content = self.get_modulefile_content(spec)
         self.assertEqual(len([x for x in content if 'prereq' in x]), 5)
 
     def test_alter_environment(self):
-        spack.modules.CONFIGURATION = self.configuration_alter_environment
+        spack.modules._module_config = self.configuration_alter_environment
         spec = spack.spec.Spec('mpileaks platform=test target=x86_64')
         content = self.get_modulefile_content(spec)
         self.assertEqual(
@@ -315,7 +315,7 @@ class TclTests(ModuleFileGeneratorTests):
             len([x for x in content if 'setenv LIBDWARF_ROOT' in x]), 1)
 
     def test_blacklist(self):
-        spack.modules.CONFIGURATION = self.configuration_blacklist
+        spack.modules._module_config = self.configuration_blacklist
         spec = spack.spec.Spec('mpileaks ^zmpi')
         content = self.get_modulefile_content(spec)
         self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1)
@@ -329,7 +329,7 @@ class TclTests(ModuleFileGeneratorTests):
         self.assertEqual(len([x for x in content if 'module load ' in x]), 1)
 
     def test_conflicts(self):
-        spack.modules.CONFIGURATION = self.configuration_conflicts
+        spack.modules._module_config = self.configuration_conflicts
         spec = spack.spec.Spec('mpileaks')
         content = self.get_modulefile_content(spec)
         self.assertEqual(
@@ -339,11 +339,11 @@ class TclTests(ModuleFileGeneratorTests):
         self.assertEqual(
             len([x for x in content if x == 'conflict intel/14.0.1']), 1)
 
-        spack.modules.CONFIGURATION = self.configuration_wrong_conflicts
+        spack.modules._module_config = self.configuration_wrong_conflicts
         self.assertRaises(SystemExit, self.get_modulefile_content, spec)
 
     def test_suffixes(self):
-        spack.modules.CONFIGURATION = self.configuration_suffix
+        spack.modules._module_config = self.configuration_suffix
         spec = spack.spec.Spec('mpileaks+debug arch=x86-linux')
         spec.concretize()
         generator = spack.modules.TclModule(spec)
@@ -412,7 +412,7 @@ class LmodTests(ModuleFileGeneratorTests):
     }
 
     def test_simple_case(self):
-        spack.modules.CONFIGURATION = self.configuration_autoload_direct
+        spack.modules._module_config = self.configuration_autoload_direct
         spec = spack.spec.Spec(mpich_spec_string)
         content = self.get_modulefile_content(spec)
         self.assertTrue('-- -*- lua -*-' in content)
@@ -420,14 +420,14 @@ class LmodTests(ModuleFileGeneratorTests):
         self.assertTrue('whatis([[Version : 3.0.4]])' in content)
 
     def test_autoload(self):
-        spack.modules.CONFIGURATION = self.configuration_autoload_direct
+        spack.modules._module_config = self.configuration_autoload_direct
         spec = spack.spec.Spec(mpileaks_spec_string)
         content = self.get_modulefile_content(spec)
         self.assertEqual(
             len([x for x in content if 'if not isloaded(' in x]), 2)
         self.assertEqual(len([x for x in content if 'load(' in x]), 2)
 
-        spack.modules.CONFIGURATION = self.configuration_autoload_all
+        spack.modules._module_config = self.configuration_autoload_all
         spec = spack.spec.Spec(mpileaks_spec_string)
         content = self.get_modulefile_content(spec)
         self.assertEqual(
@@ -435,7 +435,7 @@ class LmodTests(ModuleFileGeneratorTests):
         self.assertEqual(len([x for x in content if 'load(' in x]), 5)
 
     def test_alter_environment(self):
-        spack.modules.CONFIGURATION = self.configuration_alter_environment
+        spack.modules._module_config = self.configuration_alter_environment
         spec = spack.spec.Spec('mpileaks platform=test target=x86_64')
         content = self.get_modulefile_content(spec)
         self.assertEqual(
@@ -460,7 +460,7 @@ class LmodTests(ModuleFileGeneratorTests):
             len([x for x in content if 'unsetenv("BAR")' in x]), 0)
 
     def test_blacklist(self):
-        spack.modules.CONFIGURATION = self.configuration_blacklist
+        spack.modules._module_config = self.configuration_blacklist
         spec = spack.spec.Spec(mpileaks_spec_string)
         content = self.get_modulefile_content(spec)
         self.assertEqual(
@@ -499,14 +499,14 @@ class DotkitTests(MockPackagesTest):
 
     def setUp(self):
         super(DotkitTests, self).setUp()
-        self.configuration_obj = spack.modules.CONFIGURATION
+        self.configuration_obj = spack.modules._module_config
         spack.modules.open = mock_open
         # Make sure that a non-mocked configuration will trigger an error
-        spack.modules.CONFIGURATION = None
+        spack.modules._module_config = None
 
     def tearDown(self):
         del spack.modules.open
-        spack.modules.CONFIGURATION = self.configuration_obj
+        spack.modules._module_config = self.configuration_obj
         super(DotkitTests, self).tearDown()
 
     def get_modulefile_content(self, spec):
@@ -517,7 +517,7 @@ class DotkitTests(MockPackagesTest):
         return content
 
     def test_dotkit(self):
-        spack.modules.CONFIGURATION = self.configuration_dotkit
+        spack.modules._module_config = self.configuration_dotkit
         spec = spack.spec.Spec('mpileaks arch=x86-linux')
         content = self.get_modulefile_content(spec)
         self.assertTrue('#c spack' in content)