From 445cae5c81a1f02e830f0f52613855ae27c51f22 Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Thu, 14 May 2020 11:27:37 -0700 Subject: Feature: Allow lmod configuration to set core specs (#16517) Packages built with lmod core_compiler are placed in `Core`. Other packages may belong in `Core`. For example, python may be built with a proprietary compiler for performance, but belong on the `Core` directory. With this PR, lmod config can include a `core_specs` list. Any package that satisfies a spec in that list is placed in `Core`, regardless of its compiler or dependencies. --- lib/spack/docs/module_file_support.rst | 11 +++++++++++ lib/spack/spack/modules/lmod.py | 10 ++++++++++ lib/spack/spack/schema/modules.py | 5 +++-- lib/spack/spack/test/data/modules/lmod/complex_hierarchy.yaml | 3 +++ lib/spack/spack/test/modules/lmod.py | 4 +++- 5 files changed, 30 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/module_file_support.rst b/lib/spack/docs/module_file_support.rst index 4ddd42139c..406715b07e 100644 --- a/lib/spack/docs/module_file_support.rst +++ b/lib/spack/docs/module_file_support.rst @@ -538,6 +538,8 @@ most likely via the ``+blas`` variant specification. lmod: core_compilers: - 'gcc@4.8' + core_specs: + - 'python' hierarchy: - 'mpi' - 'lapack' @@ -547,6 +549,15 @@ most likely via the ``+blas`` variant specification. implementations of ``mpi`` and ``lapack``, and let LMod switch safely from one to the other. + All packages built with a compiler in ``core_compilers`` and all + packages that satisfy a spec in ``core_specs`` will be put in the + ``Core`` hierarchy of the lua modules. + +.. warning:: + Consistency of Core packages + The user is responsible for maintining consistency among core packages, as ``core_specs`` + bypasses the hierarchy that allows LMod to safely switch between coherent software stacks. + .. warning:: Deep hierarchies and ``lmod spider`` For hierarchies that are deeper than three layers ``lmod spider`` may have some issues. diff --git a/lib/spack/spack/modules/lmod.py b/lib/spack/spack/modules/lmod.py index a9ebb0bba9..d0d14dac5b 100644 --- a/lib/spack/spack/modules/lmod.py +++ b/lib/spack/spack/modules/lmod.py @@ -110,6 +110,11 @@ class LmodConfiguration(BaseConfiguration): raise CoreCompilersNotFoundError(msg) return value + @property + def core_specs(self): + """Returns the list of "Core" specs""" + return configuration().get('core_specs', []) + @property def hierarchy_tokens(self): """Returns the list of tokens that are part of the modulefile @@ -140,6 +145,11 @@ class LmodConfiguration(BaseConfiguration): to the actual provider. 'compiler' is always present among the requirements. """ + # If it's a core_spec, lie and say it requires a core compiler + if any(self.spec.satisfies(core_spec) + for core_spec in self.core_specs): + return {'compiler': self.core_compilers[0]} + # Keep track of the requirements that this package has in terms # of virtual packages that participate in the hierarchical structure requirements = {'compiler': self.spec.compiler} diff --git a/lib/spack/spack/schema/modules.py b/lib/spack/spack/schema/modules.py index 1fbbf614c8..4e11071757 100644 --- a/lib/spack/spack/schema/modules.py +++ b/lib/spack/spack/schema/modules.py @@ -16,7 +16,7 @@ import spack.schema.environment #: #: THIS NEEDS TO BE UPDATED FOR EVERY NEW KEYWORD THAT #: IS ADDED IMMEDIATELY BELOW THE MODULE TYPE ATTRIBUTE -spec_regex = r'(?!hierarchy|verbose|hash_length|whitelist|' \ +spec_regex = r'(?!hierarchy|core_specs|verbose|hash_length|whitelist|' \ r'blacklist|naming_scheme|core_compilers|all)(^\w[\w-]*)' #: Matches an anonymous spec, i.e. a spec without a root name @@ -145,7 +145,8 @@ properties = { 'type': 'object', 'properties': { 'core_compilers': array_of_strings, - 'hierarchy': array_of_strings + 'hierarchy': array_of_strings, + 'core_specs': array_of_strings, }, } # Specific lmod extensions ] diff --git a/lib/spack/spack/test/data/modules/lmod/complex_hierarchy.yaml b/lib/spack/spack/test/data/modules/lmod/complex_hierarchy.yaml index ada3c691cb..5cad95c7fa 100644 --- a/lib/spack/spack/test/data/modules/lmod/complex_hierarchy.yaml +++ b/lib/spack/spack/test/data/modules/lmod/complex_hierarchy.yaml @@ -6,6 +6,9 @@ lmod: core_compilers: - 'clang@3.3' + core_specs: + - 'mpich@3.0.1' + hierarchy: - lapack - blas diff --git a/lib/spack/spack/test/modules/lmod.py b/lib/spack/spack/test/modules/lmod.py index 12b6d71aa5..3daeeb068d 100644 --- a/lib/spack/spack/test/modules/lmod.py +++ b/lib/spack/spack/test/modules/lmod.py @@ -27,6 +27,7 @@ def compiler(request): @pytest.fixture(params=[ ('mpich@3.0.4', ('mpi',)), + ('mpich@3.0.1', []), ('openblas@0.2.15', ('blas',)), ('openblas-with-lapack@0.2.15', ('blas', 'lapack')) ]) @@ -54,7 +55,8 @@ class TestLmod(object): # Check that the compiler part of the path has no hash and that it # is transformed to r"Core" if the compiler is listed among core # compilers - if compiler == 'clang@3.3': + # Check that specs listed as core_specs are transformed to "Core" + if compiler == 'clang@3.3' or spec_string == 'mpich@3.0.1': assert 'Core' in layout.available_path_parts else: assert compiler.replace('@', '/') in layout.available_path_parts -- cgit v1.2.3-70-g09d2