From f3d62f1d3b2d53e88bcea8e6ea8b1c5abe5018b1 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 19 Jan 2016 18:29:39 +0100 Subject: code checking : added pep8 and coveralls --- .coveragerc | 27 +++++++++++++++++++++++++++ .gitignore | 2 ++ .travis.yml | 19 +++++++++++++++---- 3 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000000..37410a3677 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,27 @@ +# .coveragerc to control coverage.py +[run] +branch = True + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if False: + if __name__ == .__main__.: + +ignore_errors = True + +[html] +directory = htmlcov diff --git a/.gitignore b/.gitignore index 4b97de5d50..643e5d9b03 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ /share/spack/dotkit /share/spack/modules /TAGS +/htmlcov +.coverage diff --git a/.travis.yml b/.travis.yml index ab379be486..4207f8d08b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,21 +6,32 @@ python: # Use new Travis infrastructure (Docker can't sudo yet) sudo: false -# No need to install any deps. -install: true +# Install coveralls to obtain code coverage +install: + - "pip install coveralls" + - "pip install pep8" before_install: # Need this for the git tests to succeed. - git config --global user.email "spack@example.com" - git config --global user.name "Test User" + # Need this to be able to compute the list of changed files + - git fetch origin develop:develop script: - . share/spack/setup-env.sh - - spack test + # Run unit tests with code coverage + - coverage run --source=lib --omit=lib/spack/spack/test/*,lib/spack/env/*,lib/spack/docs/* bin/spack test + # Checks if the file that have been changed are pep8 conformant + - CHANGED_PYTHON_FILES=`git diff develop... --name-only | perl -ne 'print if /\.py/g'` + - if [[ ${CHANGED_PYTHON_FILES} ]] ; then pep8 --max-line-length=120 ${CHANGED_PYTHON_FILES} ; fi + +after_success: + - coveralls notifications: email: recipients: - - tgamblin@llnl.gov + - massimiliano.culpo@gmail.com on_success: change on_failure: always -- cgit v1.2.3-70-g09d2 From ae7c999712167ed68a6e1a45c0f8f8dfc9fb5b17 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 19 Jan 2016 18:55:18 +0100 Subject: code checking : added flake8 --- .travis.yml | 3 +++ flake8.ini | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 flake8.ini diff --git a/.travis.yml b/.travis.yml index 4207f8d08b..2d9bbabe10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ sudo: false install: - "pip install coveralls" - "pip install pep8" + - "pip install flake8" before_install: # Need this for the git tests to succeed. @@ -25,6 +26,8 @@ script: # Checks if the file that have been changed are pep8 conformant - CHANGED_PYTHON_FILES=`git diff develop... --name-only | perl -ne 'print if /\.py/g'` - if [[ ${CHANGED_PYTHON_FILES} ]] ; then pep8 --max-line-length=120 ${CHANGED_PYTHON_FILES} ; fi + - if [[ ${CHANGED_PYTHON_FILES} ]] ; then flake8 --format pylint --config flake8.ini ${CHANGED_PYTHON_FILES} ; fi + after_success: - coveralls diff --git a/flake8.ini b/flake8.ini new file mode 100644 index 0000000000..757c71705e --- /dev/null +++ b/flake8.ini @@ -0,0 +1,3 @@ +[flake8] +ignore = W391,F403 +max-line-length = 120 \ No newline at end of file -- cgit v1.2.3-70-g09d2 From f092e69d12fa98d60989d6b827aacfe95cc297aa Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 19 Jan 2016 19:10:40 +0100 Subject: yapf : added style file for convenience --- .style.yapf | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .style.yapf diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 0000000000..b4a1f688b8 --- /dev/null +++ b/.style.yapf @@ -0,0 +1,3 @@ +[style] +based_on_style = pep8 +column_limit = 120 \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 00185a39d9fe0155016e36c47b951f908bb7919f Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 19 Jan 2016 19:28:38 +0100 Subject: reverted email notification --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2d9bbabe10..f935062b02 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,6 @@ after_success: notifications: email: recipients: - - massimiliano.culpo@gmail.com + - tgamblin@llnl.gov on_success: change on_failure: always -- cgit v1.2.3-70-g09d2 From 4b22c2406c163c83998de8938b5fefa18886057e Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 5 Feb 2016 08:56:20 +0100 Subject: removed redundant references to pep8 --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f935062b02..082b747bc2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,6 @@ sudo: false # Install coveralls to obtain code coverage install: - "pip install coveralls" - - "pip install pep8" - "pip install flake8" before_install: @@ -23,9 +22,8 @@ script: - . share/spack/setup-env.sh # Run unit tests with code coverage - coverage run --source=lib --omit=lib/spack/spack/test/*,lib/spack/env/*,lib/spack/docs/* bin/spack test - # Checks if the file that have been changed are pep8 conformant + # Checks if the file that have been changed are flake8 conformant - CHANGED_PYTHON_FILES=`git diff develop... --name-only | perl -ne 'print if /\.py/g'` - - if [[ ${CHANGED_PYTHON_FILES} ]] ; then pep8 --max-line-length=120 ${CHANGED_PYTHON_FILES} ; fi - if [[ ${CHANGED_PYTHON_FILES} ]] ; then flake8 --format pylint --config flake8.ini ${CHANGED_PYTHON_FILES} ; fi -- cgit v1.2.3-70-g09d2 From 1664d1b5034a6ca7bf9901df91a22087d609f5bf Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 22 Mar 2016 15:59:58 +0100 Subject: travis : removed external directory from the list under coverage --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 465a86faf7..30dfebf783 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ script: - spack config get compilers - spack install -v libdwarf # Run unit tests with code coverage - - coverage run --source=lib --omit=lib/spack/spack/test/*,lib/spack/env/*,lib/spack/docs/* bin/spack test + - coverage run --source=lib --omit=lib/spack/spack/test/*,lib/spack/env/*,lib/spack/docs/*,lib/spack/external/* bin/spack test # Checks if the file that have been changed are flake8 conformant - CHANGED_PYTHON_FILES=`git diff develop... --name-only | perl -ne 'print if /\.py/g'` - if [[ ${CHANGED_PYTHON_FILES} ]] ; then flake8 --format pylint --config flake8.ini ${CHANGED_PYTHON_FILES} ; fi -- cgit v1.2.3-70-g09d2 From d546d828d322d54bb61c10dbf1fa42831049001e Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 5 Apr 2016 13:20:28 +0200 Subject: module file : added filtering based on environment variable name --- etc/spack/modules.yaml | 4 ++++ lib/spack/spack/config.py | 44 +++++++++++++++++++++++++++++++++++++++--- lib/spack/spack/environment.py | 16 +++++++++++++++ lib/spack/spack/modules.py | 10 +++++++++- 4 files changed, 70 insertions(+), 4 deletions(-) diff --git a/etc/spack/modules.yaml b/etc/spack/modules.yaml index aa2a2c3fe2..395cf9c2cd 100644 --- a/etc/spack/modules.yaml +++ b/etc/spack/modules.yaml @@ -6,3 +6,7 @@ # ------------------------------------------------------------------------- modules: enable: ['tcl', 'dotkit'] + + dotkit: + filter: + environment_modifications: ['CPATH', 'LIBRARY_PATH'] # Exclude changes to any of these variables diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 14e5aaf4fb..4fca735fc9 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -146,7 +146,7 @@ section_schemas = { 'type': 'object', 'additionalProperties': False, 'patternProperties': { - 'compilers:?': { # optional colon for overriding site config. + 'compilers:?': { # optional colon for overriding site config. 'type': 'object', 'default': {}, 'additionalProperties': False, @@ -195,6 +195,7 @@ section_schemas = { 'default': [], 'items': { 'type': 'string'},},},}, + 'packages': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack package configuration file schema', @@ -238,11 +239,35 @@ section_schemas = { 'default' : {}, } },},},},},}, + 'modules': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack module file configuration file schema', 'type': 'object', 'additionalProperties': False, + 'definitions': { + 'module_type_configuration': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'properties': { + 'filter': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'properties': { + 'environment_modifications': { + 'type': 'array', + 'default': [], + 'items': { + 'type': 'string' + } + } + } + } + } + } + }, 'patternProperties': { r'modules:?': { 'type': 'object', @@ -253,9 +278,22 @@ section_schemas = { 'type': 'array', 'default': [], 'items': { - 'type': 'string' + 'type': 'string', + 'enum': ['tcl', 'dotkit'] } - } + }, + 'tcl': { + 'allOf': [ + {'$ref': '#/definitions/module_type_configuration'}, # Base configuration + {} # Specific tcl extensions + ] + }, + 'dotkit': { + 'allOf': [ + {'$ref': '#/definitions/module_type_configuration'}, # Base configuration + {} # Specific dotkit extensions + ] + }, } }, }, diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 72aafa4e2d..3d18d3a63f 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -250,3 +250,19 @@ def validate(env, errstream): modifications = env.group_by_name() for variable, list_of_changes in sorted(modifications.items()): set_or_unset_not_first(variable, list_of_changes, errstream) + + +def filter_environment_modifications(env, variables): + """ + Generator that filters out any change to environment variables present in the input list + + Args: + env: list of environment modifications + variables: list of variable names to be filtered + + Yields: + items in env if they are not in variables + """ + for item in env: + if item.name not in variables: + yield item diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index d797af287d..aca37ae14b 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -179,9 +179,17 @@ class EnvModule(object): if not env: return + # Filter modifications to the environment according to configuration files + try: + filter_list = CONFIGURATION[self.name]['filter']['environment_modifications'] + except KeyError: + filter_list = [] + with open(self.file_name, 'w') as f: self.write_header(f) - for line in self.process_environment_command(env): + for line in self.process_environment_command( + filter_environment_modifications(env, filter_list) + ): f.write(line) def write_header(self, stream): -- cgit v1.2.3-70-g09d2 From 7fc0b1ebd2218bae90ec47c0290a23ffae8b54d3 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 5 Apr 2016 13:33:11 +0200 Subject: module file : reverted to sensible default --- etc/spack/modules.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/etc/spack/modules.yaml b/etc/spack/modules.yaml index 395cf9c2cd..aa2a2c3fe2 100644 --- a/etc/spack/modules.yaml +++ b/etc/spack/modules.yaml @@ -6,7 +6,3 @@ # ------------------------------------------------------------------------- modules: enable: ['tcl', 'dotkit'] - - dotkit: - filter: - environment_modifications: ['CPATH', 'LIBRARY_PATH'] # Exclude changes to any of these variables -- cgit v1.2.3-70-g09d2 From c352249e305c00d951586bc8c19a13ceadaa1887 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 5 Apr 2016 14:55:05 +0200 Subject: EnvModules : narrowing the type of exceptions being "handled" --- lib/spack/spack/modules.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index aca37ae14b..0d486f6430 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -167,10 +167,11 @@ class EnvModule(object): package = self.spec[item].package package.setup_dependent_package(self.pkg.module, self.spec) package.setup_dependent_environment(spack_env, env, self.spec) - except: + except KeyError as e: # The extends was conditional, so it doesn't count here # eg: extends('python', when='+python') - pass + tty.debug(str(e)) + # Package-specific environment modifications self.spec.package.setup_environment(spack_env, env) -- cgit v1.2.3-70-g09d2 From ca7d7010076a520fa20f02ab0609bbc3e6216d2f Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 5 Apr 2016 16:10:04 +0200 Subject: module file : all the dependencies can affect the run_time environment, not only extendees --- lib/spack/spack/modules.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 0d486f6430..04317f53b1 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -50,6 +50,7 @@ import llnl.util.tty as tty import spack import spack.config from llnl.util.filesystem import join_path, mkdirp +from spack.build_environment import parent_class_modules, set_module_variables_for_package from spack.environment import * __all__ = ['EnvModule', 'Dotkit', 'TclModule'] @@ -159,12 +160,25 @@ class EnvModule(object): # installation prefix env = inspect_path(self.spec.prefix) - # Let the extendee modify their extensions before asking for + # Let the extendee/dependency modify their extensions/dependencies before asking for # package-specific modifications spack_env = EnvironmentModifications() - for item in self.pkg.extendees: + + def dependencies(): + # FIXME : during module file creation nodes seem to be visited multiple times even if cover='nodes' + # FIXME : is given. This work around permits to get a unique list of spec anyhow. + # FIXME : Possibly we miss a merge step among nodes that refer to the same package. + l = [x for x in sorted(self.spec.traverse(order='post', depth=True, cover='nodes'),reverse=True)] + seen = set() + return [x for ii, x in l if not (x in seen or seen.add(x))] + + for item in dependencies(): try: - package = self.spec[item].package + package = self.spec[item.name].package + modules = parent_class_modules(package.__class__) + for mod in modules: + set_module_variables_for_package(package, mod) + set_module_variables_for_package(package, package.module) package.setup_dependent_package(self.pkg.module, self.spec) package.setup_dependent_environment(spack_env, env, self.spec) except KeyError as e: @@ -172,8 +186,8 @@ class EnvModule(object): # eg: extends('python', when='+python') tty.debug(str(e)) - # Package-specific environment modifications + set_module_variables_for_package(self.pkg, self.pkg.module) self.spec.package.setup_environment(spack_env, env) # TODO : implement site-specific modifications and filters -- cgit v1.2.3-70-g09d2 From 2968b60ee0b6b8d649e3d0c1a091857bfde28b89 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 5 Apr 2016 18:03:09 +0200 Subject: module file : added autoload and prereq --- lib/spack/spack/config.py | 8 +++++- lib/spack/spack/modules.py | 65 ++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 4fca735fc9..ff5fba24f8 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -246,6 +246,10 @@ section_schemas = { 'type': 'object', 'additionalProperties': False, 'definitions': { + 'dependency_selection': { + 'type': 'string', + 'enum': ['None', 'Direct', 'All'] + }, 'module_type_configuration': { 'type': 'object', 'default': {}, @@ -264,7 +268,9 @@ section_schemas = { } } } - } + }, + 'autoload': {'$ref': '#/definitions/dependency_selection'}, + 'prerequisites': {'$ref': '#/definitions/dependency_selection'} } } }, diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 04317f53b1..d9a846f281 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -164,15 +164,25 @@ class EnvModule(object): # package-specific modifications spack_env = EnvironmentModifications() - def dependencies(): + def dependencies(request='All'): + if request == 'None': + return [] + + l = [x for x in sorted(self.spec.traverse(order='post', depth=True, cover='nodes', root=False), reverse=True)] + + if request == 'Direct': + return [x for ii, x in l if ii == 1] + # FIXME : during module file creation nodes seem to be visited multiple times even if cover='nodes' # FIXME : is given. This work around permits to get a unique list of spec anyhow. # FIXME : Possibly we miss a merge step among nodes that refer to the same package. - l = [x for x in sorted(self.spec.traverse(order='post', depth=True, cover='nodes'),reverse=True)] seen = set() - return [x for ii, x in l if not (x in seen or seen.add(x))] + seen_add = seen.add + return [x for ii, x in l if not (x in seen or seen_add(x))] - for item in dependencies(): + # TODO : the code down below is quite similar to build_environment.setup_package and needs to be + # TODO : factored out to a single place + for item in dependencies('All'): try: package = self.spec[item.name].package modules = parent_class_modules(package.__class__) @@ -190,11 +200,18 @@ class EnvModule(object): set_module_variables_for_package(self.pkg, self.pkg.module) self.spec.package.setup_environment(spack_env, env) - # TODO : implement site-specific modifications and filters - if not env: - return + # Get list of modules that will be loaded automatically + try: + autoload_list = dependencies(CONFIGURATION[self.name]['autoload']) + except KeyError: + autoload_list = [] - # Filter modifications to the environment according to configuration files + try: + prerequisites_list = dependencies(CONFIGURATION[self.name]['prerequisites']) + except KeyError: + prerequisites_list = [] + + # Filter modifications to environment variables try: filter_list = CONFIGURATION[self.name]['filter']['environment_modifications'] except KeyError: @@ -202,14 +219,26 @@ class EnvModule(object): with open(self.file_name, 'w') as f: self.write_header(f) - for line in self.process_environment_command( - filter_environment_modifications(env, filter_list) - ): + # Automatic loads + for x in autoload_list: + self.write_autoload(f, x) + # Prerequisites + for x in prerequisites_list: + self.write_prerequisite(f, x) + # Modifications to the environment + iterable = self.process_environment_command( filter_environment_modifications(env, filter_list)) + for line in iterable: f.write(line) def write_header(self, stream): raise NotImplementedError() + def write_autoload(self, stream, spec): + raise NotImplementedError() + + def write_prerequisite(self, stream, spec): + raise NotImplementedError() + def process_environment_command(self, env): for command in env: try: @@ -309,3 +338,17 @@ class TclModule(EnvModule): for line in textwrap.wrap(self.long_description, 72): module_file.write("puts stderr \"%s\"\n" % line) module_file.write('}\n\n') + + def write_autoload(self, module_file, spec): + autoload_format = ''' +if ![ is-loaded {module_file} ] {{ + puts stderr "Autoloading {module_file}" + module load {module_file} +}} +''''' + m = TclModule(spec) + module_file.write(autoload_format.format(module_file=m.use_name)) + + def write_prerequisite(self, module_file, spec): + m = TclModule(spec) + module_file.write('prereq {module_file}\n'.format(module_file=m.use_name)) -- cgit v1.2.3-70-g09d2 From 5da37c573f13f4c16021f1314a87e4f0c995ba0f Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 5 Apr 2016 18:41:38 +0200 Subject: modules : don't pass stream around --- lib/spack/spack/modules.py | 50 ++++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index d9a846f281..9a021a7c49 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -149,7 +149,6 @@ class EnvModule(object): # Not very descriptive fallback return 'spack installed package' - def write(self): """Write out a module file for this object.""" module_dir = os.path.dirname(self.file_name) @@ -218,25 +217,27 @@ class EnvModule(object): filter_list = [] with open(self.file_name, 'w') as f: - self.write_header(f) + # Header + f.write(self.header) # Automatic loads for x in autoload_list: - self.write_autoload(f, x) + f.write(self.autoload(x)) # Prerequisites for x in prerequisites_list: - self.write_prerequisite(f, x) + f.write(self.prerequisite(x)) # Modifications to the environment - iterable = self.process_environment_command( filter_environment_modifications(env, filter_list)) + iterable = self.process_environment_command(filter_environment_modifications(env, filter_list)) for line in iterable: f.write(line) - def write_header(self, stream): + @property + def header(self): raise NotImplementedError() - def write_autoload(self, stream, spec): + def autoload(self, spec): raise NotImplementedError() - def write_prerequisite(self, stream, spec): + def prerequisite(self, spec): raise NotImplementedError() def process_environment_command(self, env): @@ -286,19 +287,22 @@ class Dotkit(EnvModule): self.spec.compiler.version, self.spec.dag_hash()) - def write_header(self, dk_file): + @property + def header(self): # Category + header = '' if self.category: - dk_file.write('#c %s\n' % self.category) + header += '#c %s\n' % self.category # Short description if self.short_description: - dk_file.write('#d %s\n' % self.short_description) + header += '#d %s\n' % self.short_description # Long description if self.long_description: for line in textwrap.wrap(self.long_description, 72): - dk_file.write("#h %s\n" % line) + header += '#h %s\n' % line + return header class TclModule(EnvModule): @@ -324,22 +328,24 @@ class TclModule(EnvModule): self.spec.compiler.version, self.spec.dag_hash()) - def write_header(self, module_file): + @property + def header(self): # TCL Modulefile header - module_file.write('#%Module1.0\n') + header = '#%Module1.0\n' # TODO : category ? # Short description if self.short_description: - module_file.write('module-whatis \"%s\"\n\n' % self.short_description) + header += 'module-whatis \"%s\"\n\n' % self.short_description # Long description if self.long_description: - module_file.write('proc ModulesHelp { } {\n') + header += 'proc ModulesHelp { } {\n' for line in textwrap.wrap(self.long_description, 72): - module_file.write("puts stderr \"%s\"\n" % line) - module_file.write('}\n\n') + header += 'puts stderr "%s"\n' % line + header += '}\n\n' + return header - def write_autoload(self, module_file, spec): + def autoload(self, spec): autoload_format = ''' if ![ is-loaded {module_file} ] {{ puts stderr "Autoloading {module_file}" @@ -347,8 +353,8 @@ if ![ is-loaded {module_file} ] {{ }} ''''' m = TclModule(spec) - module_file.write(autoload_format.format(module_file=m.use_name)) + return autoload_format.format(module_file=m.use_name) - def write_prerequisite(self, module_file, spec): + def prerequisite(self, spec): m = TclModule(spec) - module_file.write('prereq {module_file}\n'.format(module_file=m.use_name)) + return 'prereq {module_file}\n'.format(module_file=m.use_name) -- cgit v1.2.3-70-g09d2 From 670cb423f91b17b9757b387a9579be4f248d208b Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 5 Apr 2016 18:44:40 +0200 Subject: modules : iterating on all the dependencies doesn't reuire try/except KeyError --- lib/spack/spack/modules.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 9a021a7c49..115d4d9a37 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -182,18 +182,13 @@ class EnvModule(object): # TODO : the code down below is quite similar to build_environment.setup_package and needs to be # TODO : factored out to a single place for item in dependencies('All'): - try: - package = self.spec[item.name].package - modules = parent_class_modules(package.__class__) - for mod in modules: - set_module_variables_for_package(package, mod) - set_module_variables_for_package(package, package.module) - package.setup_dependent_package(self.pkg.module, self.spec) - package.setup_dependent_environment(spack_env, env, self.spec) - except KeyError as e: - # The extends was conditional, so it doesn't count here - # eg: extends('python', when='+python') - tty.debug(str(e)) + package = self.spec[item.name].package + modules = parent_class_modules(package.__class__) + for mod in modules: + set_module_variables_for_package(package, mod) + set_module_variables_for_package(package, package.module) + package.setup_dependent_package(self.pkg.module, self.spec) + package.setup_dependent_environment(spack_env, env, self.spec) # Package-specific environment modifications set_module_variables_for_package(self.pkg, self.pkg.module) -- cgit v1.2.3-70-g09d2 From 872f049b30cef3395900f57cd98575b8cacfbb36 Mon Sep 17 00:00:00 2001 From: Dhanannjay Deo Date: Tue, 29 Mar 2016 22:27:16 -0400 Subject: create visit package --- var/spack/repos/builtin/packages/visit/package.py | 40 +++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 var/spack/repos/builtin/packages/visit/package.py diff --git a/var/spack/repos/builtin/packages/visit/package.py b/var/spack/repos/builtin/packages/visit/package.py new file mode 100644 index 0000000000..ff80815ae6 --- /dev/null +++ b/var/spack/repos/builtin/packages/visit/package.py @@ -0,0 +1,40 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install visit +# +# You can always get back here to change things with: +# +# spack edit visit +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + + +class Visit(Package): + """VisIt is an Open Source, interactive, scalable, visualization, animation and analysis tool.""" + homepage = "https://wci.llnl.gov/simulation/computer-codes/visit/" + url = "http://portal.nersc.gov/project/visit/releases/2.10.1/visit2.10.1.tar.gz" + + version('2.10.1', '3cbca162fdb0249f17c4456605c4211e') + + depends_on("vtk@7.0") + depends_on("qt@4.8.6") + # FIXME: Add dependencies if this package requires them. + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + # FIXME: Spack couldn't guess one, so here are some options: + # configure('--prefix=%s' % prefix) + std_cmake_args = [] + cmake('.', *std_cmake_args) + + # FIXME: Add logic to build and install here + make() + make("install") -- cgit v1.2.3-70-g09d2 From 2e05830eb1078778eddcd702a9a50fd3d2b775d5 Mon Sep 17 00:00:00 2001 From: Dhanannjay Deo Date: Thu, 31 Mar 2016 14:55:35 -0400 Subject: Constrain to vtk6.1 qt4.8.6 and opengl1 rendering backend --- var/spack/repos/builtin/packages/visit/package.py | 36 ++++++++--------------- 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/var/spack/repos/builtin/packages/visit/package.py b/var/spack/repos/builtin/packages/visit/package.py index ff80815ae6..ade86631cf 100644 --- a/var/spack/repos/builtin/packages/visit/package.py +++ b/var/spack/repos/builtin/packages/visit/package.py @@ -1,19 +1,3 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install visit -# -# You can always get back here to change things with: -# -# spack edit visit -# -# See the spack documentation for more information on building -# packages. -# from spack import * @@ -24,17 +8,21 @@ class Visit(Package): version('2.10.1', '3cbca162fdb0249f17c4456605c4211e') - depends_on("vtk@7.0") + depends_on("vtk@6.1.0~opengl2") depends_on("qt@4.8.6") - # FIXME: Add dependencies if this package requires them. + depends_on("python") + # TODO: Other package dependencies from spack def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. - # FIXME: Spack couldn't guess one, so here are some options: - # configure('--prefix=%s' % prefix) - std_cmake_args = [] - cmake('.', *std_cmake_args) - # FIXME: Add logic to build and install here + feature_args = std_cmake_args[:] + feature_args = ["-DVTK_MAJOR_VERSION=6", + "-DVTK_MINOR_VERSION=1", + "-DCMAKE_INSTALL_PREFIX:PATH=%s" % spec.prefix, + "-DVISIT_LOC_QMAKE_EXE:FILEPATH=%s/qmake-qt4" % spec['qt'].prefix.bin, + "-DPYTHON_EXECUTABLE:FILEPATH=%s/python" % spec['python'].prefix.bin] + + cmake('./src', *feature_args) + make() make("install") -- cgit v1.2.3-70-g09d2 From c3c70cf704be1e174391e2d64d5491db72289a57 Mon Sep 17 00:00:00 2001 From: Dhanannjay Deo Date: Tue, 5 Apr 2016 14:35:33 -0400 Subject: Install lite pdb headers --- var/spack/repos/builtin/packages/silo/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/silo/package.py b/var/spack/repos/builtin/packages/silo/package.py index b7894e4d2b..245acc0973 100644 --- a/var/spack/repos/builtin/packages/silo/package.py +++ b/var/spack/repos/builtin/packages/silo/package.py @@ -1,5 +1,6 @@ from spack import * + class Silo(Package): """Silo is a library for reading and writing a wide variety of scientific data to binary, disk files.""" @@ -30,6 +31,7 @@ class Silo(Package): '--prefix=%s' % prefix, '--with-hdf5=%s,%s' % (spec['hdf5'].prefix.include, spec['hdf5'].prefix.lib), '--with-zlib=%s,%s' % (spec['zlib'].prefix.include, spec['zlib'].prefix.lib), + '--enable-install-lite-headers', *config_args) make() -- cgit v1.2.3-70-g09d2 From f221f645091bb0175185e8c20fe19be2cc42639a Mon Sep 17 00:00:00 2001 From: Dhanannjay Deo Date: Tue, 5 Apr 2016 22:31:08 -0400 Subject: Add variant for shared library --- var/spack/repos/builtin/packages/silo/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/silo/package.py b/var/spack/repos/builtin/packages/silo/package.py index 245acc0973..638a894b7b 100644 --- a/var/spack/repos/builtin/packages/silo/package.py +++ b/var/spack/repos/builtin/packages/silo/package.py @@ -13,6 +13,7 @@ class Silo(Package): version('4.8', 'b1cbc0e7ec435eb656dc4b53a23663c9') variant('fortran', default=True, description='Enable Fortran support') + variant('shared', default=True, description='Build shared libraries') variant('silex', default=False, description='Builds Silex, a GUI for viewing Silo files') depends_on('hdf5') @@ -22,6 +23,8 @@ class Silo(Package): config_args = [ '--enable-fortran' if '+fortran' in spec else '--disable-fortran', '--enable-silex' if '+silex' in spec else '--disable-silex', + '--enable-shared' if '+shared' in spec else '--disable-shared', + '--disable-static' if '+shared' in spec else '--enable-static', ] if '+silex' in spec: -- cgit v1.2.3-70-g09d2 From 071548a62f8dd13d09915c1ed985063e6e372d74 Mon Sep 17 00:00:00 2001 From: Dhanannjay 'Djay' Deo Date: Tue, 5 Apr 2016 23:16:57 -0400 Subject: Building VisIt with silo --- var/spack/repos/builtin/packages/visit/package.py | 24 +++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/visit/package.py b/var/spack/repos/builtin/packages/visit/package.py index ade86631cf..9b21370fa3 100644 --- a/var/spack/repos/builtin/packages/visit/package.py +++ b/var/spack/repos/builtin/packages/visit/package.py @@ -11,18 +11,22 @@ class Visit(Package): depends_on("vtk@6.1.0~opengl2") depends_on("qt@4.8.6") depends_on("python") - # TODO: Other package dependencies from spack + depends_on("hdf5") # silo seems to need it + depends_on("silo+shared") def install(self, spec, prefix): + with working_dir('spack-build', create=True): - feature_args = std_cmake_args[:] - feature_args = ["-DVTK_MAJOR_VERSION=6", - "-DVTK_MINOR_VERSION=1", - "-DCMAKE_INSTALL_PREFIX:PATH=%s" % spec.prefix, - "-DVISIT_LOC_QMAKE_EXE:FILEPATH=%s/qmake-qt4" % spec['qt'].prefix.bin, - "-DPYTHON_EXECUTABLE:FILEPATH=%s/python" % spec['python'].prefix.bin] + feature_args = std_cmake_args[:] + feature_args = ["-DVTK_MAJOR_VERSION=6", + "-DVTK_MINOR_VERSION=1", + "-DCMAKE_INSTALL_PREFIX:PATH=%s" % spec.prefix, + "-DVISIT_LOC_QMAKE_EXE:FILEPATH=%s/qmake-qt4" % spec['qt'].prefix.bin, + "-DPYTHON_EXECUTABLE:FILEPATH=%s/python" % spec['python'].prefix.bin, + "-DVISIT_SILO_DIR:PATH=%s" % spec['silo'].prefix, + "-DVISIT_HDF5_DIR:PATH=%s" % spec['hdf5'].prefix] - cmake('./src', *feature_args) + cmake('../src', *feature_args) - make() - make("install") + make() + make("install") -- cgit v1.2.3-70-g09d2 From d636b4fdde5b9a37dff9a354368d36fa6969cec9 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 6 Apr 2016 13:42:47 +0200 Subject: modules : more sensible name to blacklist environment variables modules : added skeleton to permit modifications based on specs --- lib/spack/spack/config.py | 20 ++++++++-- lib/spack/spack/environment.py | 2 +- lib/spack/spack/modules.py | 90 +++++++++++++++++++++--------------------- 3 files changed, 62 insertions(+), 50 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index ff5fba24f8..2067c6146e 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -250,7 +250,7 @@ section_schemas = { 'type': 'string', 'enum': ['None', 'Direct', 'All'] }, - 'module_type_configuration': { + 'module_file_configuration': { 'type': 'object', 'default': {}, 'additionalProperties': False, @@ -260,7 +260,7 @@ section_schemas = { 'default': {}, 'additionalProperties': False, 'properties': { - 'environment_modifications': { + 'environment_blacklist': { 'type': 'array', 'default': [], 'items': { @@ -270,7 +270,21 @@ section_schemas = { } }, 'autoload': {'$ref': '#/definitions/dependency_selection'}, - 'prerequisites': {'$ref': '#/definitions/dependency_selection'} + 'prerequisites': {'$ref': '#/definitions/dependency_selection'}, + 'environment': { + 'type': 'object', + 'default': {} + } + } + }, + 'module_type_configuration': { + 'type': 'object', + 'default': {}, + 'properties': { + 'all': {'$ref': '#/definitions/module_file_configuration'} + }, + 'patternProperties': { + r'\w[\w-]*': {'$ref': '#/definitions/module_file_configuration'} } } }, diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 3d18d3a63f..92ab4e6bea 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -252,7 +252,7 @@ def validate(env, errstream): set_or_unset_not_first(variable, list_of_changes, errstream) -def filter_environment_modifications(env, variables): +def filter_environment_blacklist(env, variables): """ Generator that filters out any change to environment variables present in the input list diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 115d4d9a37..62cd2a8a8b 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -49,6 +49,7 @@ import textwrap import llnl.util.tty as tty import spack import spack.config + from llnl.util.filesystem import join_path, mkdirp from spack.build_environment import parent_class_modules, set_module_variables_for_package from spack.environment import * @@ -137,7 +138,6 @@ class EnvModule(object): if self.spec.package.__doc__: self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) - @property def category(self): # Anything defined at the package level takes precedence @@ -150,35 +150,40 @@ class EnvModule(object): return 'spack installed package' def write(self): - """Write out a module file for this object.""" + """ + Writes out a module file for this object. + + This method employs a template pattern and expects derived classes to: + - override the header property + - provide formats for autoload, prerequisites and environment changes + """ module_dir = os.path.dirname(self.file_name) if not os.path.exists(module_dir): mkdirp(module_dir) - # Environment modifications guessed by inspecting the - # installation prefix - env = inspect_path(self.spec.prefix) - - # Let the extendee/dependency modify their extensions/dependencies before asking for - # package-specific modifications - spack_env = EnvironmentModifications() - def dependencies(request='All'): if request == 'None': return [] - l = [x for x in sorted(self.spec.traverse(order='post', depth=True, cover='nodes', root=False), reverse=True)] + l = [xx for xx in sorted(self.spec.traverse(order='post', depth=True, cover='nodes', root=False), reverse=True)] if request == 'Direct': - return [x for ii, x in l if ii == 1] + return [xx for ii, xx in l if ii == 1] # FIXME : during module file creation nodes seem to be visited multiple times even if cover='nodes' # FIXME : is given. This work around permits to get a unique list of spec anyhow. # FIXME : Possibly we miss a merge step among nodes that refer to the same package. seen = set() seen_add = seen.add - return [x for ii, x in l if not (x in seen or seen_add(x))] + return [xx for ii, xx in l if not (xx in seen or seen_add(xx))] + + # Environment modifications guessed by inspecting the + # installation prefix + env = inspect_path(self.spec.prefix) + # Let the extendee/dependency modify their extensions/dependencies before asking for + # package-specific modifications + spack_env = EnvironmentModifications() # TODO : the code down below is quite similar to build_environment.setup_package and needs to be # TODO : factored out to a single place for item in dependencies('All'): @@ -207,43 +212,43 @@ class EnvModule(object): # Filter modifications to environment variables try: - filter_list = CONFIGURATION[self.name]['filter']['environment_modifications'] + filter_list = CONFIGURATION[self.name]['filter']['environment_blacklist'] except KeyError: filter_list = [] + # Build up the module file content + module_file_content = self.header + for x in autoload_list: + module_file_content += self.autoload(x) + for x in prerequisites_list: + module_file_content += self.prerequisite(x) + for line in self.process_environment_command(filter_environment_blacklist(env, filter_list)): + module_file_content += line + + # Dump to file with open(self.file_name, 'w') as f: - # Header - f.write(self.header) - # Automatic loads - for x in autoload_list: - f.write(self.autoload(x)) - # Prerequisites - for x in prerequisites_list: - f.write(self.prerequisite(x)) - # Modifications to the environment - iterable = self.process_environment_command(filter_environment_modifications(env, filter_list)) - for line in iterable: - f.write(line) + f.write(module_file_content) @property def header(self): raise NotImplementedError() def autoload(self, spec): - raise NotImplementedError() + m = TclModule(spec) + return self.autoload_format.format(module_file=m.use_name) def prerequisite(self, spec): - raise NotImplementedError() + m = TclModule(spec) + return self.prerequisite_format.format(module_file=m.use_name) def process_environment_command(self, env): for command in env: try: - yield self.formats[type(command)].format(**command.args) + yield self.environment_modifications_formats[type(command)].format(**command.args) except KeyError: tty.warn('Cannot handle command of type {command} : skipping request'.format(command=type(command))) tty.warn('{context} at {filename}:{lineno}'.format(**command.args)) - @property def file_name(self): """Subclasses should implement this to return the name of the file @@ -266,7 +271,7 @@ class Dotkit(EnvModule): name = 'dotkit' path = join_path(spack.share_path, "dotkit") - formats = { + environment_modifications_formats = { PrependPath: 'dk_alter {name} {value}\n', SetEnv: 'dk_setenv {name} {value}\n' } @@ -304,7 +309,7 @@ class TclModule(EnvModule): name = 'tcl' path = join_path(spack.share_path, "modules") - formats = { + environment_modifications_formats = { PrependPath: 'prepend-path {name} \"{value}\"\n', AppendPath: 'append-path {name} \"{value}\"\n', RemovePath: 'remove-path {name} \"{value}\"\n', @@ -312,6 +317,13 @@ class TclModule(EnvModule): UnsetEnv: 'unsetenv {name}\n' } + autoload_format = ('if ![ is-loaded {module_file} ] {{' + ' puts stderr "Autoloading {module_file}"' + ' module load {module_file}' + '}}') + + prerequisite_format = 'prereq {module_file}\n' + @property def file_name(self): return join_path(TclModule.path, self.spec.architecture, self.use_name) @@ -339,17 +351,3 @@ class TclModule(EnvModule): header += 'puts stderr "%s"\n' % line header += '}\n\n' return header - - def autoload(self, spec): - autoload_format = ''' -if ![ is-loaded {module_file} ] {{ - puts stderr "Autoloading {module_file}" - module load {module_file} -}} -''''' - m = TclModule(spec) - return autoload_format.format(module_file=m.use_name) - - def prerequisite(self, spec): - m = TclModule(spec) - return 'prereq {module_file}\n'.format(module_file=m.use_name) -- cgit v1.2.3-70-g09d2 From e993c17f89fd44dbf22b19b6669b79b5c79d85a4 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 6 Apr 2016 14:02:27 +0200 Subject: modules : added environment modifications from configuration file --- lib/spack/spack/config.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 2067c6146e..8babb356b4 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -246,6 +246,13 @@ section_schemas = { 'type': 'object', 'additionalProperties': False, 'definitions': { + 'array_of_strings': { + 'type': 'array', + 'default': [], + 'items': { + 'type': 'string' + } + }, 'dependency_selection': { 'type': 'string', 'enum': ['None', 'Direct', 'All'] @@ -273,7 +280,14 @@ section_schemas = { 'prerequisites': {'$ref': '#/definitions/dependency_selection'}, 'environment': { 'type': 'object', - 'default': {} + 'default': {}, + 'additionalProperties': False, + 'properties': { + 'set-env': {'$ref': '#/definitions/array_of_strings'}, + 'unset-env': {'$ref': '#/definitions/array_of_strings'}, + 'prepend-path': {'$ref': '#/definitions/array_of_strings'}, + 'append-path': {'$ref': '#/definitions/array_of_strings'} + } } } }, -- cgit v1.2.3-70-g09d2 From e4cfe7488717eb94b824e9846ebf571c62e625fb Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 6 Apr 2016 16:42:56 +0200 Subject: modules : customization based on specs --- lib/spack/spack/config.py | 8 +-- lib/spack/spack/modules.py | 122 ++++++++++++++++++++++++++++++++------------- 2 files changed, 90 insertions(+), 40 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 8babb356b4..3565f17a77 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -283,10 +283,10 @@ section_schemas = { 'default': {}, 'additionalProperties': False, 'properties': { - 'set-env': {'$ref': '#/definitions/array_of_strings'}, - 'unset-env': {'$ref': '#/definitions/array_of_strings'}, - 'prepend-path': {'$ref': '#/definitions/array_of_strings'}, - 'append-path': {'$ref': '#/definitions/array_of_strings'} + 'set': {'$ref': '#/definitions/array_of_strings'}, + 'unset': {'$ref': '#/definitions/array_of_strings'}, + 'prepend_path': {'$ref': '#/definitions/array_of_strings'}, + 'append_path': {'$ref': '#/definitions/array_of_strings'} } } } diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 62cd2a8a8b..3d5775d03f 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -45,6 +45,7 @@ import os.path import re import shutil import textwrap +import copy import llnl.util.tty as tty import spack @@ -113,6 +114,84 @@ def inspect_path(prefix): return env +def dependencies(spec, request='All'): + if request == 'None': + return [] + + l = [xx for xx in + sorted(spec.traverse(order='post', depth=True, cover='nodes', root=False), reverse=True)] + + if request == 'Direct': + return [xx for ii, xx in l if ii == 1] + + # FIXME : during module file creation nodes seem to be visited multiple times even if cover='nodes' + # FIXME : is given. This work around permits to get a unique list of spec anyhow. + # FIXME : Possibly we miss a merge step among nodes that refer to the same package. + seen = set() + seen_add = seen.add + return [xx for ii, xx in l if not (xx in seen or seen_add(xx))] + + +def parse_config_options(module_generator): + autoloads, prerequisites, filters = [], [], [] + env = EnvironmentModifications() + # Get the configuration for this kind of generator + try: + module_configuration = copy.copy(CONFIGURATION[module_generator.name]) + except KeyError: + return autoloads, prerequisites, filters, env + + # Get the defaults for all packages + all_conf = module_configuration.pop('all', {}) + + update_single(module_generator.spec, all_conf, autoloads, prerequisites, filters, env) + + for spec, conf in module_configuration.items(): + override = False + if spec.endswith(':'): + spec = spec.strip(':') + override = True + if module_generator.spec.satisfies(spec): + if override: + autoloads, prerequisites, filters = [], [], [] + env = EnvironmentModifications() + update_single(module_generator.spec, conf, autoloads, prerequisites, filters, env) + tty.msg('{request}, {spec}'.format(request=spec, spec=module_generator.spec)) + + return autoloads, prerequisites, filters, env + + +def update_single(spec, configuration, autoloads, prerequisites, filters, env): + # Get list of modules that will be loaded automatically + try: + autoloads.extend(dependencies(spec, configuration['autoload'])) + except KeyError: + pass + try: + prerequisites.extend(dependencies(spec, configuration['prerequisites'])) + except KeyError: + pass + + # Filter modifications to environment variables + try: + filters.extend(configuration['filter']['environment_blacklist']) + except KeyError: + pass + + try: + for method, arglist in configuration['environment'].items(): + for item in arglist: + if method == 'unset': + args = [item] + else: + args = item.split(',') + getattr(env, method)(*args) + except KeyError: + pass + + + + class EnvModule(object): name = 'env_module' formats = {} @@ -161,22 +240,6 @@ class EnvModule(object): if not os.path.exists(module_dir): mkdirp(module_dir) - def dependencies(request='All'): - if request == 'None': - return [] - - l = [xx for xx in sorted(self.spec.traverse(order='post', depth=True, cover='nodes', root=False), reverse=True)] - - if request == 'Direct': - return [xx for ii, xx in l if ii == 1] - - # FIXME : during module file creation nodes seem to be visited multiple times even if cover='nodes' - # FIXME : is given. This work around permits to get a unique list of spec anyhow. - # FIXME : Possibly we miss a merge step among nodes that refer to the same package. - seen = set() - seen_add = seen.add - return [xx for ii, xx in l if not (xx in seen or seen_add(xx))] - # Environment modifications guessed by inspecting the # installation prefix env = inspect_path(self.spec.prefix) @@ -186,7 +249,7 @@ class EnvModule(object): spack_env = EnvironmentModifications() # TODO : the code down below is quite similar to build_environment.setup_package and needs to be # TODO : factored out to a single place - for item in dependencies('All'): + for item in dependencies(self.spec, 'All'): package = self.spec[item.name].package modules = parent_class_modules(package.__class__) for mod in modules: @@ -199,30 +262,17 @@ class EnvModule(object): set_module_variables_for_package(self.pkg, self.pkg.module) self.spec.package.setup_environment(spack_env, env) - # Get list of modules that will be loaded automatically - try: - autoload_list = dependencies(CONFIGURATION[self.name]['autoload']) - except KeyError: - autoload_list = [] - - try: - prerequisites_list = dependencies(CONFIGURATION[self.name]['prerequisites']) - except KeyError: - prerequisites_list = [] - - # Filter modifications to environment variables - try: - filter_list = CONFIGURATION[self.name]['filter']['environment_blacklist'] - except KeyError: - filter_list = [] + # Parse configuration file + autoloads, prerequisites, filters, conf_env = parse_config_options(self) + env.extend(conf_env) # Build up the module file content module_file_content = self.header - for x in autoload_list: + for x in autoloads: module_file_content += self.autoload(x) - for x in prerequisites_list: + for x in prerequisites: module_file_content += self.prerequisite(x) - for line in self.process_environment_command(filter_environment_blacklist(env, filter_list)): + for line in self.process_environment_command(filter_environment_blacklist(env, filters)): module_file_content += line # Dump to file -- cgit v1.2.3-70-g09d2 From 7904f75782a3ba65c6096b22e61f8f789d7fc001 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 6 Apr 2016 17:05:47 +0200 Subject: modules : all the other strings are lower case --- lib/spack/spack/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3565f17a77..6b5b3dfd62 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -255,7 +255,7 @@ section_schemas = { }, 'dependency_selection': { 'type': 'string', - 'enum': ['None', 'Direct', 'All'] + 'enum': ['none', 'direct', 'all'] }, 'module_file_configuration': { 'type': 'object', -- cgit v1.2.3-70-g09d2 From 21b7734d92d210f3287588ef2fd49062b3c727c7 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 6 Apr 2016 17:08:31 +0200 Subject: modules : removed debug leftover --- lib/spack/spack/modules.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 3d5775d03f..b12b93e32d 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -156,7 +156,6 @@ def parse_config_options(module_generator): autoloads, prerequisites, filters = [], [], [] env = EnvironmentModifications() update_single(module_generator.spec, conf, autoloads, prerequisites, filters, env) - tty.msg('{request}, {spec}'.format(request=spec, spec=module_generator.spec)) return autoloads, prerequisites, filters, env -- cgit v1.2.3-70-g09d2 From fbabfc593d10a7fc35944e72e2eb8b1ebf4818eb Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 6 Apr 2016 16:44:22 -0500 Subject: Make R extensable and add a couple of packages for verification. Added R as a build system so that the create template will have the correct configure line. Added a regex for version parsing as the R URLs are a little odd. --- lib/spack/spack/cmd/create.py | 8 +++- lib/spack/spack/url.py | 3 ++ var/spack/repos/builtin/packages/R/package.py | 54 ++++++++++++++++++++++ .../repos/builtin/packages/r-abind/package.py | 15 ++++++ .../repos/builtin/packages/r-magic/package.py | 15 ++++++ 5 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 var/spack/repos/builtin/packages/r-abind/package.py create mode 100644 var/spack/repos/builtin/packages/r-magic/package.py diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index f0cd50b8df..e3a31806ab 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -124,10 +124,12 @@ class ConfigureGuesser(object): autotools = "configure('--prefix=%s' % prefix)" cmake = "cmake('.', *std_cmake_args)" python = "python('setup.py', 'install', '--prefix=%s' % prefix)" + r = "R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file)" config_lines = ((r'/configure$', 'autotools', autotools), (r'/CMakeLists.txt$', 'cmake', cmake), - (r'/setup.py$', 'python', python)) + (r'/setup.py$', 'python', python), + (r'/NAMESPACE$', 'r', r)) # Peek inside the tarball. tar = which('tar') @@ -272,6 +274,10 @@ def create(parser, args): if guesser.build_system == 'python': name = 'py-%s' % name + # Prepend 'r-' to R package names, by convention. + if guesser.build_system == 'r': + name = 'r-%s' % name + # Create a directory for the new package. pkg_path = repo.filename_for_package_name(name) if os.path.exists(pkg_path) and not args.force: diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index f51f05cad7..b4fb70d6fb 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -206,6 +206,9 @@ def parse_version_offset(path): # e.g. lame-398-1 (r'-((\d)+-\d)', stem), + # e.g. foobar_1.2-3 + (r'_((\d+\.)+\d+(-\d+)?)', stem), + # e.g. foobar-4.5.1 (r'-((\d+\.)*\d+)$', stem), diff --git a/var/spack/repos/builtin/packages/R/package.py b/var/spack/repos/builtin/packages/R/package.py index 2471dff09b..3a76416f27 100644 --- a/var/spack/repos/builtin/packages/R/package.py +++ b/var/spack/repos/builtin/packages/R/package.py @@ -1,4 +1,14 @@ +import functools +import glob +import inspect +import os +import re +from contextlib import closing + +import spack +from llnl.util.lang import match_predicate from spack import * +from spack.util.environment import * class R(Package): @@ -9,6 +19,8 @@ class R(Package): """ homepage = "https://www.r-project.org" url = "http://cran.cnr.berkeley.edu/src/base/R-3/R-3.1.2.tar.gz" + + extendable = True version('3.2.3', '1ba3dac113efab69e706902810cc2970') version('3.2.2', '57cef5c2e210a5454da1979562a10e5b') @@ -47,3 +59,45 @@ class R(Package): configure(*options) make() make('install') + + # ======================================================================== + # Set up environment to make install easy for R extensions. + # ======================================================================== + + @property + def r_lib_dir(self): + return os.path.join('lib64', 'R', 'library') + + def setup_dependent_environment(self, spack_env, run_env, extension_spec): + # Set R_LIBS to include the library dir for the + # extension and any other R extensions it depends on. + r_libs_path = [] + for d in extension_spec.traverse(): + if d.package.extends(self.spec): + r_libs_path.append(os.path.join(d.prefix, self.r_lib_dir)) + + r_libs_path = ':'.join(r_libs_path) + spack_env.set('R_LIBS', r_libs_path) + + # For run time environment set only the path for extension_spec and prepend it to R_LIBS + if extension_spec.package.extends(self.spec): + run_env.prepend_path('R_LIBS', os.path.join(extension_spec.prefix, self.r_lib_dir)) + + + def setup_dependent_package(self, module, ext_spec): + """ + Called before R modules' install() methods. + + In most cases, extensions will only need to have one line:: + + R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) + """ + # R extension builds can have a global R executable function + module.R = Executable(join_path(self.spec.prefix.bin, 'R')) + + # Add variable for library directry + module.r_lib_dir = os.path.join(ext_spec.prefix, self.r_lib_dir) + + # Make the site packages directory for extensions, if it does not exist already. + if ext_spec.package.is_extension: + mkdirp(module.r_lib_dir) diff --git a/var/spack/repos/builtin/packages/r-abind/package.py b/var/spack/repos/builtin/packages/r-abind/package.py new file mode 100644 index 0000000000..54d399c432 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-abind/package.py @@ -0,0 +1,15 @@ +from spack import * + +class RAbind(Package): + """Combine multidimensional arrays into a single array. This is a generalization of 'cbind' and 'rbind'. Works with vectors, matrices, and higher-dimensional arrays. Also provides functions 'adrop', 'asub', and 'afill' for manipulating, extracting and replacing data in arrays.""" + + homepage = "https://cran.r-project.org/" + url = "https://cran.r-project.org/src/contrib/abind_1.4-3.tar.gz" + + version('1.4-3', '10fcf80c677b991bf263d38be35a1fc5', expand=False) + + extends('R') + + def install(self, spec, prefix): + + R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) diff --git a/var/spack/repos/builtin/packages/r-magic/package.py b/var/spack/repos/builtin/packages/r-magic/package.py new file mode 100644 index 0000000000..5f25b2a162 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-magic/package.py @@ -0,0 +1,15 @@ +from spack import * + +class RMagic(Package): + """A collection of efficient, vectorized algorithms for the creation and investigation of magic squares and hypercubes, including a variety of functions for the manipulation and analysis of arbitrarily dimensioned arrays.""" + homepage = "https://cran.r-project.org/" + url = "https://cran.r-project.org/src/contrib/magic_1.5-6.tar.gz" + + version('1.5-6', 'a68e5ced253b2196af842e1fc84fd029', expand=False) + + extends('R') + + depends_on('r-abind') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) -- cgit v1.2.3-70-g09d2 From 32779ab1f6c2658771e69e58f8d8a451b23043b0 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 6 Apr 2016 17:02:34 -0500 Subject: Add r-filehash to test version naming. --- var/spack/repos/builtin/packages/r-filehash/package.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-filehash/package.py diff --git a/var/spack/repos/builtin/packages/r-filehash/package.py b/var/spack/repos/builtin/packages/r-filehash/package.py new file mode 100644 index 0000000000..a3b688da10 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-filehash/package.py @@ -0,0 +1,14 @@ +from spack import * + +class RFilehash(Package): + """Implements a simple key-value style database where character string keys are associated with data values that are stored on the disk. A simple interface is provided for inserting, retrieving, and deleting data from the database. Utilities are provided that allow 'filehash' databases to be treated much like environments and lists are already used in R. These utilities are provided to encourage interactive and exploratory analysis on large datasets. Three different file formats for representing the database are currently available and new formats can easily be incorporated by third parties for use in the 'filehash' framework.""" + + homepage = 'https://cran.r-project.org/' + url = "https://cran.r-project.org/src/contrib/filehash_2.3.tar.gz" + + version('2.3', '01fffafe09b148ccadc9814c103bdc2f', expand=False) + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) -- cgit v1.2.3-70-g09d2 From b19d6ab9c5e4b7a8289e057826b498dd93ea7e08 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 6 Apr 2016 19:08:01 -0500 Subject: Check for non-numeric bits in the stem. --- lib/spack/spack/url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index b4fb70d6fb..ad51da9d47 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -207,7 +207,7 @@ def parse_version_offset(path): (r'-((\d)+-\d)', stem), # e.g. foobar_1.2-3 - (r'_((\d+\.)+\d+(-\d+)?)', stem), + (r'_((\d+\.)+\d+(-\d+)?[a-z]?)', stem), # e.g. foobar-4.5.1 (r'-((\d+\.)*\d+)$', stem), -- cgit v1.2.3-70-g09d2 From 8a7f34665b0b8b95f273401082d862989bb4443c Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 6 Apr 2016 23:05:49 -0500 Subject: Make sure base bioconductor package can be installed. --- var/spack/repos/builtin/packages/r-BiocGenerics/package.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-BiocGenerics/package.py diff --git a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py new file mode 100644 index 0000000000..e2d9bb9594 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py @@ -0,0 +1,13 @@ +from spack import * + +class RBiocgenerics(Package): + """S4 generic functions needed by many Bioconductor packages.""" + homepage = 'https://www.bioconductor.org/packages/release/bioc/html/BiocGenerics.html' + url = "https://www.bioconductor.org/packages/release/bioc/src/contrib/BiocGenerics_0.16.1.tar.gz" + + version('0.16.1', 'c2148ffd86fc6f1f819c7f68eb2c744f', expand=False) + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) -- cgit v1.2.3-70-g09d2 From 7dc1524c0807ea8cb6cf990d1811e0ca2218444a Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 7 Apr 2016 10:48:05 +0200 Subject: modules : added stub for dotkit (to be verified) --- lib/spack/spack/modules.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index b12b93e32d..4f7f67e910 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -189,8 +189,6 @@ def update_single(spec, configuration, autoloads, prerequisites, filters, env): pass - - class EnvModule(object): name = 'env_module' formats = {} @@ -325,6 +323,10 @@ class Dotkit(EnvModule): SetEnv: 'dk_setenv {name} {value}\n' } + autoload_format = 'dk_op {module_file}\n' # TODO : Check this line + + prerequisite_format = None # TODO : does something like prerequisite exist for dotkit? + @property def file_name(self): return join_path(Dotkit.path, self.spec.architecture, '%s.dk' % self.use_name) -- cgit v1.2.3-70-g09d2 From 6cffac79bd39443047d380eb46121a43a256737e Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Sat, 9 Apr 2016 11:21:29 +0200 Subject: openmpi : added a few variants --- .../repos/builtin/packages/openmpi/package.py | 53 ++++++++++++++-------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 9a127f1812..1b498eb49c 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -28,10 +28,20 @@ class Openmpi(Package): patch('configure.patch', when="@1.10.0:1.10.1") variant('psm', default=False, description='Build support for the PSM library.') + variant('psm2', default=False, description='Build support for the Intel PSM2 library.') variant('verbs', default=False, description='Build support for OpenFabrics verbs.') + variant('mxm', default=False, description='Build Mellanox Messaging support') - # TODO : variant support for other schedulers is missing + variant('thread_multiple', default=False, description='Enable MPI_THREAD_MULTIPLE support') + + # TODO : variant support for alps, loadleveler is missing variant('tm', default=False, description='Build TM (Torque, PBSPro, and compatible) support') + variant('slurm', default=False, description='Build SLURM scheduler component') + + variant('pmi', default=False, description='Build PMI support, optionally adding DIR to the search path') + variant('sqlite3', default=False, description='Build sqlite3 support') + + # TODO : support for CUDA is missing provides('mpi@:2.2', when='@1.6.5') provides('mpi@:3.0', when='@1.7.5:') @@ -48,27 +58,35 @@ class Openmpi(Package): spack_env.set('OMPI_FC', spack_fc) spack_env.set('OMPI_F77', spack_f77) + @property + def verbs(self): + # Up through version 1.6, this option was previously named --with-openib + if self.spec.satisfies('@:1.6'): + return 'openib' + # In version 1.7, it was renamed to be --with-verbs + elif self.spec.satisfies('@1.7:'): + return 'verbs' def install(self, spec, prefix): config_args = ["--prefix=%s" % prefix, "--with-hwloc=%s" % spec['hwloc'].prefix, "--enable-shared", "--enable-static"] - - # Variants - if '+tm' in spec: - config_args.append("--with-tm") # necessary for Torque support - - if '+psm' in spec: - config_args.append("--with-psm") - - if '+verbs' in spec: - # Up through version 1.6, this option was previously named --with-openib - if spec.satisfies('@:1.6'): - config_args.append("--with-openib") - # In version 1.7, it was renamed to be --with-verbs - elif spec.satisfies('@1.7:'): - config_args.append("--with-verbs") + # Variant based arguments + config_args.extend([ + # Schedulers + '--with-tm' if '+tm' in spec else '--without-tm', + '--with-slurm' if '+slurm' in spec else '--without-slurm', + # Fabrics + '--with-psm' if '+psm' in spec else '--without-psm', + '--with-psm2' if '+psm2' in spec else '--without-psm2', + ('--with-%s' % self.verbs) if '+verbs' in spec else ('--without-%s' % self.verbs), + '--with-mxm' if '+mxm' in spec else '--without-mxm', + # Other options + '--enable-mpi-thread-multiple' if '+thread_multiple' in spec else '--disable-mpi-thread-multiple', + '--with-pmi' if '+pmi' in spec else '--without-pmi', + '--with-sqlite3' if '+sqlite3' in spec else '--without-sqlite3' + ]) # TODO: use variants for this, e.g. +lanl, +llnl, etc. # use this for LANL builds, but for LLNL builds, we need: @@ -76,9 +94,6 @@ class Openmpi(Package): if self.version == ver("1.6.5") and '+lanl' in spec: config_args.append("--with-platform=contrib/platform/lanl/tlcc2/optimized-nopanasas") - # TODO: Spack should make it so that you can't actually find - # these compilers if they're "disabled" for the current - # compiler configuration. if not self.compiler.f77 and not self.compiler.fc: config_args.append("--enable-mpi-fortran=no") -- cgit v1.2.3-70-g09d2 From a5a5dbc4081ca13f579dce111394fe77d2c54a53 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Sun, 10 Apr 2016 13:21:41 +0200 Subject: tcl module file : added new-lines in autoload (per @glennpj bug report) --- lib/spack/spack/modules.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 747c0d3be5..291dd775c5 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -372,10 +372,10 @@ class TclModule(EnvModule): UnsetEnv: 'unsetenv {name}\n' } - autoload_format = ('if ![ is-loaded {module_file} ] {{' - ' puts stderr "Autoloading {module_file}"' - ' module load {module_file}' - '}}') + autoload_format = ('if ![ is-loaded {module_file} ] {{\n' + ' puts stderr "Autoloading {module_file}"\n' + ' module load {module_file}\n' + '}}\n') prerequisite_format = 'prereq {module_file}\n' -- cgit v1.2.3-70-g09d2 From 0e2b1359e32e2948dc3ef831b83ae34483909fd0 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 11 Apr 2016 11:01:47 +0200 Subject: modules : fixed bug preventing the creation of modulefiles autoloading only direct dependencies --- lib/spack/spack/config.py | 16 +++++++--------- lib/spack/spack/modules.py | 6 +++--- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 6b5b3dfd62..e696a62e96 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -117,22 +117,20 @@ Will make Spack take compilers *only* from the user configuration, and the site configuration will be ignored. """ +import copy import os import re import sys -import copy -import jsonschema -from jsonschema import Draft4Validator, validators -import yaml -from yaml.error import MarkedYAMLError -from ordereddict_backport import OrderedDict +import jsonschema import llnl.util.tty as tty -from llnl.util.filesystem import mkdirp -import copy - import spack +import yaml +from jsonschema import Draft4Validator, validators +from llnl.util.filesystem import mkdirp +from ordereddict_backport import OrderedDict from spack.error import SpackError +from yaml.error import MarkedYAMLError # Hacked yaml for configuration files preserves line numbers. import spack.util.spack_yaml as syaml diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 291dd775c5..8a96d49144 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -114,14 +114,14 @@ def inspect_path(prefix): return env -def dependencies(spec, request='All'): - if request == 'None': +def dependencies(spec, request='all'): + if request == 'none': return [] l = [xx for xx in sorted(spec.traverse(order='post', depth=True, cover='nodes', root=False), reverse=True)] - if request == 'Direct': + if request == 'direct': return [xx for ii, xx in l if ii == 1] # FIXME : during module file creation nodes seem to be visited multiple times even if cover='nodes' -- cgit v1.2.3-70-g09d2 From 3959ca627046ff7d6519da9c5375a0ae6da50b74 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 11 Apr 2016 18:10:06 +0200 Subject: modules : added possibility to blacklist or whitelist module files --- lib/spack/spack/cmd/module.py | 1 - lib/spack/spack/config.py | 17 +++++++++++------ lib/spack/spack/modules.py | 32 +++++++++++++++++++++++++++++--- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index a67f5c0c13..f996f4eb84 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -89,7 +89,6 @@ def module_refresh(): shutil.rmtree(cls.path, ignore_errors=False) mkdirp(cls.path) for spec in specs: - tty.debug(" Writing file for %s" % spec) cls(spec).write() diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index e696a62e96..da4e787f18 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -292,12 +292,17 @@ section_schemas = { 'module_type_configuration': { 'type': 'object', 'default': {}, - 'properties': { - 'all': {'$ref': '#/definitions/module_file_configuration'} - }, - 'patternProperties': { - r'\w[\w-]*': {'$ref': '#/definitions/module_file_configuration'} - } + 'oneOf': [ + { + 'properties': { + 'whitelist': {'$ref': '#/definitions/array_of_strings'}, + 'blacklist': {'$ref': '#/definitions/array_of_strings'}, + } + }, + { + 'patternProperties': {r'\w[\w-]*': {'$ref': '#/definitions/module_file_configuration'}} + } + ] } }, 'patternProperties': { diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 8a96d49144..04f43b9605 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -40,17 +40,15 @@ various shell-support files in $SPACK/share/spack/setup-env.*. Each hook in hooks/ implements the logic for writing its specific type of module file. """ +import copy import os import os.path import re -import shutil import textwrap -import copy import llnl.util.tty as tty import spack import spack.config - from llnl.util.filesystem import join_path, mkdirp from spack.build_environment import parent_class_modules, set_module_variables_for_package from spack.environment import * @@ -225,6 +223,30 @@ class EnvModule(object): # Not very descriptive fallback return 'spack installed package' + @property + def blacklisted(self): + configuration = CONFIGURATION.get(self.name, {}) + whitelist_matches = [x for x in configuration.get('whitelist', []) if self.spec.satisfies(x)] + blacklist_matches = [x for x in configuration.get('blacklist', []) if self.spec.satisfies(x)] + if whitelist_matches: + message = '\t%s is whitelisted [matches : ' % self.spec.cshort_spec + for rule in whitelist_matches: + message += '%s ' % rule + message += ' ]' + tty.debug(message) + + if blacklist_matches: + message = '\t%s is blacklisted [matches : ' % self.spec.cshort_spec + for rule in blacklist_matches: + message += '%s ' % rule + message += ' ]' + tty.debug(message) + + if not whitelist_matches and blacklist_matches: + return True + + return False + def write(self): """ Writes out a module file for this object. @@ -233,6 +255,10 @@ class EnvModule(object): - override the header property - provide formats for autoload, prerequisites and environment changes """ + if self.blacklisted: + return + tty.debug("\t%s : writing module file" % self.spec.cshort_spec) + module_dir = os.path.dirname(self.file_name) if not os.path.exists(module_dir): mkdirp(module_dir) -- cgit v1.2.3-70-g09d2 From 41f365112c07dc6e144da92611018ee1fe6bf30b Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 12 Apr 2016 09:13:50 +0200 Subject: modules : added provenance comment in tcl header --- lib/spack/spack/modules.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 04f43b9605..0f921d7bf2 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -45,6 +45,7 @@ import os import os.path import re import textwrap +import datetime import llnl.util.tty as tty import spack @@ -401,7 +402,7 @@ class TclModule(EnvModule): autoload_format = ('if ![ is-loaded {module_file} ] {{\n' ' puts stderr "Autoloading {module_file}"\n' ' module load {module_file}\n' - '}}\n') + '}}\n\n') prerequisite_format = 'prereq {module_file}\n' @@ -420,6 +421,12 @@ class TclModule(EnvModule): def header(self): # TCL Modulefile header header = '#%Module1.0\n' + header += '## Module file created by spack (https://github.com/LLNL/spack)' + header += ' on %s\n' % datetime.datetime.now() + header += '##\n' + header += '## %s\n' % self.spec.short_spec + header += '##\n' + # TODO : category ? # Short description if self.short_description: -- cgit v1.2.3-70-g09d2 From 80678b2188acb30638e33b86a894d133ce4b4796 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 12 Apr 2016 16:54:51 +0200 Subject: fix : proper update of config file (before it was discarding architectures that were not the current one) fixes #774 --- bin/spack | 2 +- lib/spack/spack/cmd/compiler.py | 13 ++++++------- lib/spack/spack/config.py | 10 ++++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/bin/spack b/bin/spack index 31165bba9d..f51cb8a4ec 100755 --- a/bin/spack +++ b/bin/spack @@ -152,7 +152,7 @@ def main(): command = spack.cmd.get_command(args.command) try: return_val = command(parser, args) - except SpackError, e: + except SpackError as e: e.die() except KeyboardInterrupt: sys.stderr.write('\n') diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 3e58e82184..786b3b8eb4 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -22,19 +22,18 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys import argparse +import sys import llnl.util.tty as tty -from llnl.util.tty.color import colorize -from llnl.util.tty.colify import colify -from llnl.util.lang import index_by - import spack.compilers -import spack.spec import spack.config -from spack.util.environment import get_path +import spack.spec +from llnl.util.lang import index_by +from llnl.util.tty.colify import colify +from llnl.util.tty.color import colorize from spack.spec import CompilerSpec +from spack.util.environment import get_path description = "Manage compilers" diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 14e5aaf4fb..336d47cbb7 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -539,14 +539,16 @@ def update_config(section, update_data, scope=None): other yaml-ish structure. """ + validate_section_name(section) # validate section name + scope = validate_scope(scope) # get ConfigScope object from string. + # read in the config to ensure we've got current data - get_config(section) + configuration = get_config(section) - validate_section_name(section) # validate section name - scope = validate_scope(scope) # get ConfigScope object from string. + configuration.update(update_data) # read only the requested section's data. - scope.sections[section] = { section : update_data } + scope.sections[section] = {section: configuration} scope.write_section(section) -- cgit v1.2.3-70-g09d2 From 27280ea8be8e108b8d128ab518d4562ec0368684 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 12 Apr 2016 17:21:54 +0200 Subject: fix : added regression tests --- lib/spack/spack/test/config.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 0562d2d620..3977f0e7d4 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -33,7 +33,7 @@ from spack.test.mock_packages_test import * # Some sample compiler config data a_comps = { - "all": { + "x86_64_E5v2_IntelIB": { "gcc@4.7.3" : { "cc" : "/gcc473", "cxx": "/g++473", @@ -53,7 +53,7 @@ a_comps = { } b_comps = { - "all": { + "x86_64_E5v3": { "icc@10.0" : { "cc" : "/icc100", "cxx": "/icc100", @@ -85,27 +85,24 @@ class ConfigTest(MockPackagesTest): super(ConfigTest, self).tearDown() shutil.rmtree(self.tmp_dir, True) - - def check_config(self, comps, *compiler_names): + def check_config(self, comps, arch, *compiler_names): """Check that named compilers in comps match Spack's config.""" config = spack.config.get_config('compilers') compiler_list = ['cc', 'cxx', 'f77', 'fc'] for key in compiler_names: for c in compiler_list: - expected = comps['all'][key][c] - actual = config['all'][key][c] + expected = comps[arch][key][c] + actual = config[arch][key][c] self.assertEqual(expected, actual) - def test_write_key_in_memory(self): # Write b_comps "on top of" a_comps. spack.config.update_config('compilers', a_comps, 'test_low_priority') spack.config.update_config('compilers', b_comps, 'test_high_priority') # Make sure the config looks how we expect. - self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') - self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3') - + self.check_config(a_comps, 'x86_64_E5v2_IntelIB', 'gcc@4.7.3', 'gcc@4.5.0') + self.check_config(b_comps, 'x86_64_E5v3', 'icc@10.0', 'icc@11.1', 'clang@3.3') def test_write_key_to_disk(self): # Write b_comps "on top of" a_comps. @@ -116,5 +113,17 @@ class ConfigTest(MockPackagesTest): spack.config.clear_config_caches() # Same check again, to ensure consistency. - self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') - self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3') + self.check_config(a_comps, 'x86_64_E5v2_IntelIB', 'gcc@4.7.3', 'gcc@4.5.0') + self.check_config(b_comps, 'x86_64_E5v3', 'icc@10.0', 'icc@11.1', 'clang@3.3') + + def test_write_to_same_priority_file(self): + # Write b_comps in the same file as a_comps. + spack.config.update_config('compilers', a_comps, 'test_low_priority') + spack.config.update_config('compilers', b_comps, 'test_low_priority') + + # Clear caches so we're forced to read from disk. + spack.config.clear_config_caches() + + # Same check again, to ensure consistency. + self.check_config(a_comps, 'x86_64_E5v2_IntelIB', 'gcc@4.7.3', 'gcc@4.5.0') + self.check_config(b_comps, 'x86_64_E5v3', 'icc@10.0', 'icc@11.1', 'clang@3.3') -- cgit v1.2.3-70-g09d2 From c422ce7c1d85e4f934e0ce6e2ffd90d742723a15 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 13 Apr 2016 11:26:22 +0200 Subject: modules : added doc strings, fixed minor style issues, filtered from dependencies blacklisted modules --- lib/spack/spack/config.py | 2 +- lib/spack/spack/modules.py | 85 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 68 insertions(+), 19 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index da4e787f18..cf3b885ae6 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -488,7 +488,7 @@ def _read_config_file(filename, schema): elif not os.path.isfile(filename): raise ConfigFileError( - "Invlaid configuration. %s exists but is not a file." % filename) + "Invalid configuration. %s exists but is not a file." % filename) elif not os.access(filename, os.R_OK): raise ConfigFileError("Config file is not readable: %s" % filename) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 0f921d7bf2..2ec20afa69 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -63,8 +63,9 @@ CONFIGURATION = spack.config.get_config('modules') def print_help(): - """For use by commands to tell user how to activate shell support.""" - + """ + For use by commands to tell user how to activate shell support. + """ tty.msg("This command requires spack's shell integration.", "", "To initialize spack's shell commands, you must run one of", @@ -114,6 +115,20 @@ def inspect_path(prefix): def dependencies(spec, request='all'): + """ + Returns the list of dependent specs for a given spec, according to the given request + + Args: + spec: target spec + request: either 'none', 'direct' or 'all' + + Returns: + empty list if 'none', direct dependency list if 'direct', all dependencies if 'all' + """ + if request not in ('none', 'direct', 'all'): + raise tty.error("Wrong value for argument 'request' : should be one of ('none', 'direct', 'all') " + " [current value is '%s']" % request) + if request == 'none': return [] @@ -132,6 +147,18 @@ def dependencies(spec, request='all'): def parse_config_options(module_generator): + """ + Parse the configuration file and returns a bunch of items that will be needed during module file generation + + Args: + module_generator: module generator for a given spec + + Returns: + autoloads: list of specs to be autoloaded + prerequisites: list of specs to be marked as prerequisite + filters: list of environment variables whose modification is blacklisted in module files + env: list of custom environment modifications to be applied in the module file + """ autoloads, prerequisites, filters = [], [], [] env = EnvironmentModifications() # Get the configuration for this kind of generator @@ -160,15 +187,20 @@ def parse_config_options(module_generator): def update_single(spec, configuration, autoloads, prerequisites, filters, env): + """ + Updates the entries in the arguments according to the configuration + + Args: + spec: [in] target spec + configuration: [in] configuration file for the current type of module file generator + autoloads: [inout] list of dependencies to be automatically loaded + prerequisites: [inout] list of prerequisites + filters: [inout] list of environment variables whose modification is to be blacklisted + env: [inout] list of modifications to the environment + """ # Get list of modules that will be loaded automatically - try: - autoloads.extend(dependencies(spec, configuration['autoload'])) - except KeyError: - pass - try: - prerequisites.extend(dependencies(spec, configuration['prerequisites'])) - except KeyError: - pass + autoloads.extend(dependencies(spec, configuration.get('autoload', 'none'))) + prerequisites.extend(dependencies(spec, configuration.get('prerequisites', 'none'))) # Filter modifications to environment variables try: @@ -188,6 +220,24 @@ def update_single(spec, configuration, autoloads, prerequisites, filters, env): pass +def filter_blacklisted(specs, module_name): + """ + Given a sequence of specs, filters the ones that are blacklisted in the module configuration file. + + Args: + specs: sequence of spec instances + module_name: type of module file objects + + Yields: + non blacklisted specs + """ + for x in specs: + if module_types[module_name](x).blacklisted: + tty.debug('\tFILTER : %s' % x) + continue + yield x + + class EnvModule(object): name = 'env_module' formats = {} @@ -230,14 +280,14 @@ class EnvModule(object): whitelist_matches = [x for x in configuration.get('whitelist', []) if self.spec.satisfies(x)] blacklist_matches = [x for x in configuration.get('blacklist', []) if self.spec.satisfies(x)] if whitelist_matches: - message = '\t%s is whitelisted [matches : ' % self.spec.cshort_spec + message = '\tWHITELIST : %s [matches : ' % self.spec.cshort_spec for rule in whitelist_matches: message += '%s ' % rule message += ' ]' tty.debug(message) if blacklist_matches: - message = '\t%s is blacklisted [matches : ' % self.spec.cshort_spec + message = '\tBLACKLIST : %s [matches : ' % self.spec.cshort_spec for rule in blacklist_matches: message += '%s ' % rule message += ' ]' @@ -258,7 +308,7 @@ class EnvModule(object): """ if self.blacklisted: return - tty.debug("\t%s : writing module file" % self.spec.cshort_spec) + tty.debug("\tWRITE : %s [%s]" % (self.spec.cshort_spec, self.file_name)) module_dir = os.path.dirname(self.file_name) if not os.path.exists(module_dir): @@ -273,7 +323,7 @@ class EnvModule(object): spack_env = EnvironmentModifications() # TODO : the code down below is quite similar to build_environment.setup_package and needs to be # TODO : factored out to a single place - for item in dependencies(self.spec, 'All'): + for item in dependencies(self.spec, 'all'): package = self.spec[item.name].package modules = parent_class_modules(package.__class__) for mod in modules: @@ -292,9 +342,9 @@ class EnvModule(object): # Build up the module file content module_file_content = self.header - for x in autoloads: + for x in filter_blacklisted(autoloads, self.name): module_file_content += self.autoload(x) - for x in prerequisites: + for x in filter_blacklisted(prerequisites, self.name): module_file_content += self.prerequisite(x) for line in self.process_environment_command(filter_environment_blacklist(env, filters)): module_file_content += line @@ -421,8 +471,7 @@ class TclModule(EnvModule): def header(self): # TCL Modulefile header header = '#%Module1.0\n' - header += '## Module file created by spack (https://github.com/LLNL/spack)' - header += ' on %s\n' % datetime.datetime.now() + header += '## Module file created by spack (https://github.com/LLNL/spack) on %s\n' % datetime.datetime.now() header += '##\n' header += '## %s\n' % self.spec.short_spec header += '##\n' -- cgit v1.2.3-70-g09d2 From 1b4c4be151650f28e0e7d796cbfe85d05ea91f49 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 13 Apr 2016 12:44:51 +0200 Subject: modules : category is a single word (as I am not sure how dotkit will react to spaces) --- lib/spack/spack/modules.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 2ec20afa69..2c68c0c170 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -270,9 +270,9 @@ class EnvModule(object): return self.pkg.category # Extensions for extendee in self.pkg.extendees: - return '{extendee} extension'.format(extendee=extendee) + return '{extendee}_extension'.format(extendee=extendee) # Not very descriptive fallback - return 'spack installed package' + return 'spack' @property def blacklisted(self): @@ -404,7 +404,7 @@ class Dotkit(EnvModule): SetEnv: 'dk_setenv {name} {value}\n' } - autoload_format = 'dk_op {module_file}\n' # TODO : Check this line + autoload_format = 'dk_op {module_file}\n' prerequisite_format = None # TODO : does something like prerequisite exist for dotkit? -- cgit v1.2.3-70-g09d2 From c005b5fe9c72c19db92a5459d8c3657d7343aa0a Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 15 Apr 2016 09:24:44 +0200 Subject: Added coverage badge pointing to LLNL/develop --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1977a4fee9..7797da299c 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ ============ [![Build Status](https://travis-ci.org/LLNL/spack.png?branch=develop)](https://travis-ci.org/LLNL/spack) +[![Coverage Status](https://coveralls.io/repos/github/LLNL/spack/badge.svg?branch=develop)](https://coveralls.io/github/LLNL/spack?branch=develop) Spack is a package management tool designed to support multiple versions and configurations of software on a wide variety of platforms -- cgit v1.2.3-70-g09d2 From e03e87b79186c21c3db084056363bea4db8dba04 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 15 Apr 2016 15:57:54 -0400 Subject: 1. Added variants to choose language interfaces. 2. Now produces relocatable code (+fpic) by default --- .../repos/builtin/packages/parallel-netcdf/package.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/parallel-netcdf/package.py b/var/spack/repos/builtin/packages/parallel-netcdf/package.py index e6f8cf026b..1bbd24781e 100644 --- a/var/spack/repos/builtin/packages/parallel-netcdf/package.py +++ b/var/spack/repos/builtin/packages/parallel-netcdf/package.py @@ -11,11 +11,25 @@ class ParallelNetcdf(Package): version('1.7.0', '267eab7b6f9dc78c4d0e6def2def3aea4bc7c9f0') version('1.6.1', '62a094eb952f9d1e15f07d56e535052604f1ac34') + variant('cxx', default=True, description='Build the C++ Interface') + variant('fortran', default=True, description='Build the Fortran Interface') + variant('fpic', default=True, description='Produce position-independent code (for use with shared libraries)') + depends_on("m4") depends_on("mpi") + # See: https://trac.mcs.anl.gov/projects/parallel-netcdf/browser/trunk/INSTALL def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-mpi=%s" % spec['mpi'].prefix) + args = list() + if '+fpic' in spec: + args.extend(['CFLAGS=-fPIC', 'CXXFLAGS=-fPIC', 'FFLAGS=-fPIC']) + if '~cxx' in spec: + args.append('--disable-cxx') + if '~fortran' in spec: + args.append('--disable-fortran') + + args.extend(["--prefix=%s" % prefix, + "--with-mpi=%s" % spec['mpi'].prefix]) + configure(*args) make() make("install") -- cgit v1.2.3-70-g09d2 From 00f44d558ab3b3ee4b9d04534a01ad7decbe0d7b Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 12 Apr 2016 15:42:55 +0200 Subject: modules : started working on naming schemes and conflict --- lib/spack/spack/config.py | 9 +++++++- lib/spack/spack/modules.py | 56 ++++++++++++++++++++++++++++------------------ 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index cf3b885ae6..f9a680d109 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -297,6 +297,9 @@ section_schemas = { 'properties': { 'whitelist': {'$ref': '#/definitions/array_of_strings'}, 'blacklist': {'$ref': '#/definitions/array_of_strings'}, + 'naming_scheme': { + 'type': 'string' # Can we be more specific here? + } } }, { @@ -322,7 +325,11 @@ section_schemas = { 'tcl': { 'allOf': [ {'$ref': '#/definitions/module_type_configuration'}, # Base configuration - {} # Specific tcl extensions + { + 'properties': { + 'conflict': {'type': 'string'} + } + } # Specific tcl extensions ] }, 'dotkit': { diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 2c68c0c170..e6e9be1eff 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -41,11 +41,11 @@ various shell-support files in $SPACK/share/spack/setup-env.*. Each hook in hooks/ implements the logic for writing its specific type of module file. """ import copy +import datetime import os import os.path import re import textwrap -import datetime import llnl.util.tty as tty import spack @@ -75,7 +75,7 @@ def print_help(): " . %s/setup-env.sh" % spack.share_path, "", "For csh and tcsh:", - " setenv SPACK_ROOT %s" % spack.prefix, + " setenv SPACK_ROOT %s" % spack.prefix, " source %s/setup-env.csh" % spack.share_path, "") @@ -263,6 +263,34 @@ class EnvModule(object): if self.spec.package.__doc__: self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) + @property + def naming_scheme(self): + try: + naming_scheme = CONFIGURATION[self.name]['naming_scheme'] + except KeyError: + naming_scheme = self.default_naming_format + return naming_scheme + + @property + def tokens(self): + tokens = { + 'name': self.spec.name, + 'version': self.spec.version, + 'compiler': self.spec.compiler, + 'hash': self.spec.dag_hash() + } + return tokens + + @property + def use_name(self): + """ + Subclasses should implement this to return the name the module command uses to refer to the package. + """ + naming_tokens = self.tokens + naming_scheme = self.naming_scheme + name = naming_scheme.format(**naming_tokens) + return name + @property def category(self): # Anything defined at the package level takes precedence @@ -379,12 +407,6 @@ class EnvModule(object): where this module lives.""" raise NotImplementedError() - @property - def use_name(self): - """Subclasses should implement this to return the name the - module command uses to refer to the package.""" - raise NotImplementedError() - def remove(self): mod_file = self.file_name if os.path.exists(mod_file): @@ -408,17 +430,12 @@ class Dotkit(EnvModule): prerequisite_format = None # TODO : does something like prerequisite exist for dotkit? + default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}-{hash}' + @property def file_name(self): return join_path(Dotkit.path, self.spec.architecture, '%s.dk' % self.use_name) - @property - def use_name(self): - return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version, - self.spec.compiler.name, - self.spec.compiler.version, - self.spec.dag_hash()) - @property def header(self): # Category @@ -456,17 +473,12 @@ class TclModule(EnvModule): prerequisite_format = 'prereq {module_file}\n' + default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}-{hash}' + @property def file_name(self): return join_path(TclModule.path, self.spec.architecture, self.use_name) - @property - def use_name(self): - return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version, - self.spec.compiler.name, - self.spec.compiler.version, - self.spec.dag_hash()) - @property def header(self): # TCL Modulefile header -- cgit v1.2.3-70-g09d2 From c69acfa5c80fc4ea93787c19f9f4a6f3cf833f9c Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 15 Apr 2016 12:21:32 +0200 Subject: naming work correctly --- lib/spack/spack/modules.py | 11 +++++++---- share/spack/setup-env.sh | 6 +++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index e6e9be1eff..a24d5e5d09 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -276,8 +276,7 @@ class EnvModule(object): tokens = { 'name': self.spec.name, 'version': self.spec.version, - 'compiler': self.spec.compiler, - 'hash': self.spec.dag_hash() + 'compiler': self.spec.compiler } return tokens @@ -289,6 +288,10 @@ class EnvModule(object): naming_tokens = self.tokens naming_scheme = self.naming_scheme name = naming_scheme.format(**naming_tokens) + name += '-' + self.spec.dag_hash() # Always append the hash to make the module file unique + # Not everybody is working on linux... + parts = name.split('/') + name = join_path(*parts) return name @property @@ -430,7 +433,7 @@ class Dotkit(EnvModule): prerequisite_format = None # TODO : does something like prerequisite exist for dotkit? - default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}-{hash}' + default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' @property def file_name(self): @@ -473,7 +476,7 @@ class TclModule(EnvModule): prerequisite_format = 'prereq {module_file}\n' - default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}-{hash}' + default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' @property def file_name(self): diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 764af68400..0c9e5b2409 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -41,7 +41,7 @@ # commands. This allows the user to use packages without knowing all # their installation details. # -# e.g., rather than requring a full spec for libelf, the user can type: +# e.g., rather than requiring a full spec for libelf, the user can type: # # spack use libelf # @@ -110,11 +110,11 @@ function spack { unuse $_sp_module_args $_sp_full_spec fi ;; "load") - if _sp_full_spec=$(command spack $_sp_flags module find dotkit $_sp_spec); then + if _sp_full_spec=$(command spack $_sp_flags module find tcl $_sp_spec); then module load $_sp_module_args $_sp_full_spec fi ;; "unload") - if _sp_full_spec=$(command spack $_sp_flags module find dotkit $_sp_spec); then + if _sp_full_spec=$(command spack $_sp_flags module find tcl $_sp_spec); then module unload $_sp_module_args $_sp_full_spec fi ;; esac -- cgit v1.2.3-70-g09d2 From 18a241fe213618a0390a8071480854a5c06e3681 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 15 Apr 2016 15:19:02 +0200 Subject: modules : added hook for module specific extensions --- lib/spack/spack/config.py | 7 +--- lib/spack/spack/modules.py | 100 +++++++++++++++++++++------------------------ 2 files changed, 49 insertions(+), 58 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index f9a680d109..8e4cbba96e 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -276,6 +276,7 @@ section_schemas = { }, 'autoload': {'$ref': '#/definitions/dependency_selection'}, 'prerequisites': {'$ref': '#/definitions/dependency_selection'}, + 'conflict': {'type': 'string'}, 'environment': { 'type': 'object', 'default': {}, @@ -325,11 +326,7 @@ section_schemas = { 'tcl': { 'allOf': [ {'$ref': '#/definitions/module_type_configuration'}, # Base configuration - { - 'properties': { - 'conflict': {'type': 'string'} - } - } # Specific tcl extensions + {} # Specific tcl extensions ] }, 'dotkit': { diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index a24d5e5d09..a81132bcac 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -146,6 +146,17 @@ def dependencies(spec, request='all'): return [xx for ii, xx in l if not (xx in seen or seen_add(xx))] +def update_dictionary_extending_lists(target, update): + for key in update: + value = target.get(key, None) + if isinstance(value, list): + target[key].extend(update[key]) + elif isinstance(value, dict): + update_dictionary_extending_lists(target[key], update[key]) + else: + target[key] = update[key] + + def parse_config_options(module_generator): """ Parse the configuration file and returns a bunch of items that will be needed during module file generation @@ -159,19 +170,13 @@ def parse_config_options(module_generator): filters: list of environment variables whose modification is blacklisted in module files env: list of custom environment modifications to be applied in the module file """ - autoloads, prerequisites, filters = [], [], [] - env = EnvironmentModifications() # Get the configuration for this kind of generator - try: - module_configuration = copy.copy(CONFIGURATION[module_generator.name]) - except KeyError: - return autoloads, prerequisites, filters, env - - # Get the defaults for all packages - all_conf = module_configuration.pop('all', {}) - - update_single(module_generator.spec, all_conf, autoloads, prerequisites, filters, env) + module_configuration = copy.copy(CONFIGURATION.get(module_generator.name, {})) + ##### + # Merge all the rules + ##### + module_file_actions = module_configuration.pop('all', {}) for spec, conf in module_configuration.items(): override = False if spec.endswith(':'): @@ -179,45 +184,29 @@ def parse_config_options(module_generator): override = True if module_generator.spec.satisfies(spec): if override: - autoloads, prerequisites, filters = [], [], [] - env = EnvironmentModifications() - update_single(module_generator.spec, conf, autoloads, prerequisites, filters, env) - - return autoloads, prerequisites, filters, env - - -def update_single(spec, configuration, autoloads, prerequisites, filters, env): - """ - Updates the entries in the arguments according to the configuration + module_file_actions = {} + update_dictionary_extending_lists(module_file_actions, conf) + + ##### + # Process the common rules + ##### + + # Automatic loading loads + module_file_actions['autoload'] = dependencies(module_generator.spec, module_file_actions.get('autoload', 'none')) + # Prerequisites + module_file_actions['prerequisites'] = dependencies(module_generator.spec, module_file_actions.get('prerequisites', 'none')) + # Environment modifications + environment_actions = module_file_actions.pop('environment', {}) + env = EnvironmentModifications() + for method, arglist in environment_actions.items(): + for item in arglist: + if method == 'unset': + args = [item] + else: + args = item.split(',') + getattr(env, method)(*args) - Args: - spec: [in] target spec - configuration: [in] configuration file for the current type of module file generator - autoloads: [inout] list of dependencies to be automatically loaded - prerequisites: [inout] list of prerequisites - filters: [inout] list of environment variables whose modification is to be blacklisted - env: [inout] list of modifications to the environment - """ - # Get list of modules that will be loaded automatically - autoloads.extend(dependencies(spec, configuration.get('autoload', 'none'))) - prerequisites.extend(dependencies(spec, configuration.get('prerequisites', 'none'))) - - # Filter modifications to environment variables - try: - filters.extend(configuration['filter']['environment_blacklist']) - except KeyError: - pass - - try: - for method, arglist in configuration['environment'].items(): - for item in arglist: - if method == 'unset': - args = [item] - else: - args = item.split(',') - getattr(env, method)(*args) - except KeyError: - pass + return module_file_actions, env def filter_blacklisted(specs, module_name): @@ -368,17 +357,19 @@ class EnvModule(object): self.spec.package.setup_environment(spack_env, env) # Parse configuration file - autoloads, prerequisites, filters, conf_env = parse_config_options(self) + module_configuration, conf_env = parse_config_options(self) env.extend(conf_env) - + filters = module_configuration.get('filter', {}).get('environment_blacklist',{}) # Build up the module file content module_file_content = self.header - for x in filter_blacklisted(autoloads, self.name): + for x in filter_blacklisted(module_configuration.pop('autoload', []), self.name): module_file_content += self.autoload(x) - for x in filter_blacklisted(prerequisites, self.name): + for x in filter_blacklisted(module_configuration.pop('prerequisites', []), self.name): module_file_content += self.prerequisite(x) for line in self.process_environment_command(filter_environment_blacklist(env, filters)): module_file_content += line + for line in self.module_specific_content(module_configuration): + module_file_content += line # Dump to file with open(self.file_name, 'w') as f: @@ -388,6 +379,9 @@ class EnvModule(object): def header(self): raise NotImplementedError() + def module_specific_content(self, configuration): + return tuple() + def autoload(self, spec): m = TclModule(spec) return self.autoload_format.format(module_file=m.use_name) -- cgit v1.2.3-70-g09d2 From 50b148ca22b8bc9daeb1cb427eadeefd06e84800 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 18 Apr 2016 13:09:39 +0200 Subject: modules : tcl modules handle 'conflict' directive. This should completely cover the functionality in #498 --- lib/spack/spack/modules.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index a81132bcac..61ecdc10c3 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -497,3 +497,19 @@ class TclModule(EnvModule): header += 'puts stderr "%s"\n' % line header += '}\n\n' return header + + def module_specific_content(self, configuration): + naming_tokens = self.tokens + # Conflict + conflict_format = configuration.get('conflict', '') + if conflict_format: + for naming_dir, conflict_dir in zip(self.naming_scheme.split('/'), conflict_format.split('/')): + if naming_dir != conflict_dir: + message = 'Conflict scheme does not match naming scheme [{spec}]\n\n' + message += 'naming scheme : "{nformat}"\n' + message += 'conflict scheme : "{cformat}"\n' + raise tty.error( + message.format(spec=self.spec, nformat=self.naming_scheme, cformat=conflict_format) + ) + conflict_format = 'conflict ' + conflict_format + yield conflict_format.format(**naming_tokens) -- cgit v1.2.3-70-g09d2 From bce276d57301e97f7d09a902f4caa8a76735e22d Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 18 Apr 2016 17:51:53 +0200 Subject: fix : missing autoload, failing validation --- lib/spack/spack/config.py | 2 +- lib/spack/spack/modules.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 8e4cbba96e..1ac8a01619 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -293,7 +293,7 @@ section_schemas = { 'module_type_configuration': { 'type': 'object', 'default': {}, - 'oneOf': [ + 'anyOf': [ { 'properties': { 'whitelist': {'$ref': '#/definitions/array_of_strings'}, diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 61ecdc10c3..5bf003f9ad 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -171,7 +171,7 @@ def parse_config_options(module_generator): env: list of custom environment modifications to be applied in the module file """ # Get the configuration for this kind of generator - module_configuration = copy.copy(CONFIGURATION.get(module_generator.name, {})) + module_configuration = copy.deepcopy(CONFIGURATION.get(module_generator.name, {})) ##### # Merge all the rules -- cgit v1.2.3-70-g09d2 From 44321c5c2487c62e58f543455569804c95081cfa Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Mon, 18 Apr 2016 16:46:15 -0700 Subject: Update the main Krell Institute and Argo Navis Tech. packages for MPI variant support, get source from github not sourceforge, tested external package usage, and general package clean-up --- .../builtin/packages/cbtf-argonavis/package.py | 90 +++++-- .../repos/builtin/packages/cbtf-krell/package.py | 262 ++++++++++++++---- .../repos/builtin/packages/cbtf-lanl/package.py | 65 +++-- var/spack/repos/builtin/packages/cbtf/package.py | 88 ++++-- .../builtin/packages/openspeedshop/package.py | 300 ++++++++++++++++----- 5 files changed, 631 insertions(+), 174 deletions(-) diff --git a/var/spack/repos/builtin/packages/cbtf-argonavis/package.py b/var/spack/repos/builtin/packages/cbtf-argonavis/package.py index 7b07933911..90789a98f2 100644 --- a/var/spack/repos/builtin/packages/cbtf-argonavis/package.py +++ b/var/spack/repos/builtin/packages/cbtf-argonavis/package.py @@ -1,5 +1,5 @@ ################################################################################ -# Copyright (c) 2015 Krell Institute. All Rights Reserved. +# Copyright (c) 2015-2016 Krell Institute. All Rights Reserved. # # This program is free software; you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free Software @@ -24,43 +24,83 @@ class CbtfArgonavis(Package): homepage = "http://sourceforge.net/p/cbtf/wiki/Home/" # Mirror access template example - #url = "file:/g/g24/jeg/cbtf-argonavis-1.5.tar.gz" - #version('1.5', '1f7f6512f55409ed2135cfceabe26b82') + #url = "file:/home/jeg/OpenSpeedShop_ROOT/SOURCES/cbtf-argonavis-1.6.tar.gz" + #version('1.6', '0fafa0008478405c2c2319450f174ed4') - version('1.6', branch='master', git='http://git.code.sf.net/p/cbtf-argonavis/cbtf-argonavis') + version('1.6', branch='master', git='https://github.com/OpenSpeedShop/cbtf-argonavis.git') - depends_on("cmake@3.0.2:") + depends_on("cmake@3.0.2") + depends_on("boost@1.50.0:") depends_on("papi") + depends_on("mrnet@5.0.1:+lwthreads+krellpatch") depends_on("cbtf") depends_on("cbtf-krell") - depends_on("cuda") + depends_on("cuda@6.0.37") + #depends_on("cuda") parallel = False + def adjustBuildTypeParams_cmakeOptions(self, spec, cmakeOptions): + # Sets build type parameters into cmakeOptions the options that will enable the cbtf-krell built type settings + + compile_flags="-O2 -g" + BuildTypeOptions = [] + + # Set CMAKE_BUILD_TYPE to what cbtf-krell wants it to be, not the stdcmakeargs + for word in cmakeOptions[:]: + if word.startswith('-DCMAKE_BUILD_TYPE'): + cmakeOptions.remove(word) + if word.startswith('-DCMAKE_CXX_FLAGS'): + cmakeOptions.remove(word) + if word.startswith('-DCMAKE_C_FLAGS'): + cmakeOptions.remove(word) + if word.startswith('-DCMAKE_VERBOSE_MAKEFILE'): + cmakeOptions.remove(word) + BuildTypeOptions.extend([ + '-DCMAKE_VERBOSE_MAKEFILE=ON', + '-DCMAKE_BUILD_TYPE=None', + '-DCMAKE_CXX_FLAGS=%s' % compile_flags, + '-DCMAKE_C_FLAGS=%s' % compile_flags + ]) + + cmakeOptions.extend(BuildTypeOptions) + + def install(self, spec, prefix): # Look for package installation information in the cbtf and cbtf-krell prefixes cmake_prefix_path = join_path(spec['cbtf'].prefix) + ':' + join_path(spec['cbtf-krell'].prefix) - # FIXME, hard coded for testing purposes, we will alter when the external package feature is available - cuda_prefix_path = "/usr/local/cudatoolkit-6.0" - cupti_prefix_path = "/usr/local/cudatoolkit-6.0/extras/CUPTI" - - with working_dir('CUDA'): with working_dir('build', create=True): - cmake('..', - '-DCMAKE_INSTALL_PREFIX=%s' % prefix, - '-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, - '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, - '-DCUDA_INSTALL_PATH=%s' % cuda_prefix_path, - '-DCUDA_ROOT=%s' % cuda_prefix_path, - '-DCUPTI_ROOT=%s' % cupti_prefix_path, - '-DCUDA_DIR=%s' % cuda_prefix_path, - '-DPAPI_ROOT=%s' % spec['papi'].prefix, - '-DCBTF_PREFIX=%s' % spec['cbtf'].prefix, - *std_cmake_args) - make("clean") - make() - make("install") + cmakeOptions = [] + cmakeOptions.extend(['-DCMAKE_INSTALL_PREFIX=%s' % prefix, + '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, + '-DCUDA_DIR=%s' % spec['cuda'].prefix, + '-DCUDA_INSTALL_PATH=%s' % spec['cuda'].prefix, + '-DCUDA_TOOLKIT_ROOT_DIR=%s' % spec['cuda'].prefix, + '-DCUPTI_DIR=%s' % join_path(spec['cuda'].prefix + '/extras/CUPTI'), + '-DCUPTI_ROOT=%s' % join_path(spec['cuda'].prefix + '/extras/CUPTI'), + '-DPAPI_ROOT=%s' % spec['papi'].prefix, + '-DCBTF_DIR=%s' % spec['cbtf'].prefix, + '-DCBTF_KRELL_DIR=%s' % spec['cbtf-krell'].prefix, + '-DBOOST_ROOT=%s' % spec['boost'].prefix, + '-DBoost_DIR=%s' % spec['boost'].prefix, + '-DBOOST_LIBRARYDIR=%s' % spec['boost'].prefix.lib, + '-DMRNET_DIR=%s' % spec['mrnet'].prefix, + '-DBoost_NO_SYSTEM_PATHS=ON' + ]) + + # Add in the standard cmake arguments + cmakeOptions.extend(std_cmake_args) + + # Adjust the standard cmake arguments to what we want the build type, etc to be + self.adjustBuildTypeParams_cmakeOptions(spec, cmakeOptions) + + # Invoke cmake + cmake('..', *cmakeOptions) + + make("clean") + make() + make("install") diff --git a/var/spack/repos/builtin/packages/cbtf-krell/package.py b/var/spack/repos/builtin/packages/cbtf-krell/package.py index 9458ac113c..e6050cb4a9 100644 --- a/var/spack/repos/builtin/packages/cbtf-krell/package.py +++ b/var/spack/repos/builtin/packages/cbtf-krell/package.py @@ -1,5 +1,5 @@ ################################################################################ -# Copyright (c) 2015 Krell Institute. All Rights Reserved. +# Copyright (c) 2015-2016 Krell Institute. All Rights Reserved. # # This program is free software; you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free Software @@ -26,21 +26,30 @@ class CbtfKrell(Package): homepage = "http://sourceforge.net/p/cbtf/wiki/Home/" # optional mirror access template - #url = "file:/g/g24/jeg/cbtf-krell-1.5.tar.gz" - #version('1.5', 'b13f6df6a93c44149d977773dd776d2f') + #url = "file:/home/jeg/cbtf-krell-1.6.tar.gz" + #version('1.6', 'edeb61cd488f16e7b124f77db9ce762d') - version('1.6', branch='master', git='http://git.code.sf.net/p/cbtf-krell/cbtf-krell') + version('1.6', branch='master', git='https://github.com/OpenSpeedShop/cbtf-krell.git') + # MPI variants + variant('openmpi', default=False, description="Build mpi experiment collector for openmpi MPI when this variant is enabled.") + variant('mpt', default=False, description="Build mpi experiment collector for SGI MPT MPI when this variant is enabled.") + variant('mvapich2', default=False, description="Build mpi experiment collector for mvapich2 MPI when this variant is enabled.") + variant('mvapich', default=False, description="Build mpi experiment collector for mvapich MPI when this variant is enabled.") + variant('mpich2', default=False, description="Build mpi experiment collector for mpich2 MPI when this variant is enabled.") + variant('mpich', default=False, description="Build mpi experiment collector for mpich MPI when this variant is enabled.") # Dependencies for cbtf-krell + depends_on("cmake@3.0.2") # For binutils service depends_on("binutils@2.24+krellpatch") # collectionTool - depends_on("boost@1.50.0") - depends_on("dyninst@8.2.1") - depends_on("mrnet@4.1.0:+lwthreads") + depends_on("boost@1.50.0:") + depends_on("dyninst@8.2.1:") + depends_on("mrnet@5.0.1:+lwthreads+krellpatch") + depends_on("xerces-c@3.1.1:") depends_on("cbtf") @@ -51,66 +60,207 @@ class CbtfKrell(Package): # MPI Installations # These have not worked either for build or execution, commenting out for now - #depends_on("openmpi") - #depends_on("mvapich2@2.0") - #depends_on("mpich") + depends_on("openmpi", when='+openmpi') + depends_on("mpich", when='+mpich') + depends_on("mpich2", when='+mpich2') + depends_on("mvapich2", when='+mvapich2') + depends_on("mvapich", when='+mvapich') + depends_on("mpt", when='+mpt') parallel = False + def adjustBuildTypeParams_cmakeOptions(self, spec, cmakeOptions): + # Sets build type parameters into cmakeOptions the options that will enable the cbtf-krell built type settings + + compile_flags="-O2 -g" + BuildTypeOptions = [] + # Set CMAKE_BUILD_TYPE to what cbtf-krell wants it to be, not the stdcmakeargs + for word in cmakeOptions[:]: + if word.startswith('-DCMAKE_BUILD_TYPE'): + cmakeOptions.remove(word) + if word.startswith('-DCMAKE_CXX_FLAGS'): + cmakeOptions.remove(word) + if word.startswith('-DCMAKE_C_FLAGS'): + cmakeOptions.remove(word) + if word.startswith('-DCMAKE_VERBOSE_MAKEFILE'): + cmakeOptions.remove(word) + BuildTypeOptions.extend([ + '-DCMAKE_VERBOSE_MAKEFILE=ON', + '-DCMAKE_BUILD_TYPE=None', + '-DCMAKE_CXX_FLAGS=%s' % compile_flags, + '-DCMAKE_C_FLAGS=%s' % compile_flags + ]) + + cmakeOptions.extend(BuildTypeOptions) + + + + def set_mpi_cmakeOptions(self, spec, cmakeOptions): + # Appends to cmakeOptions the options that will enable the appropriate MPI implementations + + MPIOptions = [] + + # openmpi + if '+openmpi' in spec: + MPIOptions.extend([ + '-DOPENMPI_DIR=%s' % spec['openmpi'].prefix + ]) + # mpich + if '+mpich' in spec: + MPIOptions.extend([ + '-DMPICH_DIR=%s' % spec['mpich'].prefix + ]) + # mpich2 + if '+mpich2' in spec: + MPIOptions.extend([ + '-DMPICH2_DIR=%s' % spec['mpich2'].prefix + ]) + # mvapich + if '+mvapich' in spec: + MPIOptions.extend([ + '-DMVAPICH_DIR=%s' % spec['mvapich'].prefix + ]) + # mvapich2 + if '+mvapich2' in spec: + MPIOptions.extend([ + '-DMVAPICH2_DIR=%s' % spec['mvapich2'].prefix + ]) + # mpt + if '+mpt' in spec: + MPIOptions.extend([ + '-DMPT_DIR=%s' % spec['mpt'].prefix + ]) + + cmakeOptions.extend(MPIOptions) + def install(self, spec, prefix): # Add in paths for finding package config files that tell us where to find these packages - cmake_prefix_path = join_path(spec['cbtf'].prefix) + ':' + join_path(spec['dyninst'].prefix) - - # FIXME - hard code path until external package support is available - # Need to change this path and/or add additional paths for MPI experiment support on different platforms - #openmpi_prefix_path = "/opt/openmpi-1.8.2" - #mvapich_prefix_path = "/usr/local/tools/mvapich-gnu" - - # Other possibilities, they will need a -DMVAPICH_DIR=, etc clause in the cmake command to be recognized - # mvapich_prefix_path = "" - # mvapich2_prefix_path = "" - # mpich2_prefix_path = "" - # mpich_prefix_path = "" - # mpt_prefix_path = "" - - # Add in paths for cuda if requested via the cuda variant - # FIXME - hard code path until external package support is available - #if '+cuda' in spec: - # cuda_prefix_path = "/usr/local/cuda-6.0" - # cupti_prefix_path = "/usr/local/cuda-6.0/extras/CUPTI" - #else: - # cuda_prefix_path = "" - # cupti_prefix_path = "" - - #'-DMVAPICH2_DIR=%s' % spec['mvapich2'].prefix, - #'-DOPENMPI_DIR=%s' % spec['openmpi'].prefix, - #'-DMPICH_DIR=%s' % spec['mpich'].prefix, - #'-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, - #'-DOPENMPI_DIR=%s' % openmpi_prefix_path, - #'-DMVAPICH_DIR=%s' % mvapich_prefix_path, - #'-DLIB_SUFFIX=64', - #'-DCUDA_DIR=%s' % cuda_prefix_path, - #'-DCUPTI_DIR=%s' % cupti_prefix_path, + #cmake_prefix_path = join_path(spec['cbtf'].prefix) + ':' + join_path(spec['dyninst'].prefix) + #'-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path # Build cbtf-krell with cmake with working_dir('build_cbtf_krell', create=True): - cmake('..', - '-DCMAKE_BUILD_TYPE=Debug', - '-DCMAKE_INSTALL_PREFIX=%s' % prefix, - '-DCBTF_DIR=%s' % spec['cbtf'].prefix, - '-DBINUTILS_DIR=%s' % spec['binutils'].prefix, - '-DLIBMONITOR_DIR=%s' % spec['libmonitor'].prefix, - '-DLIBUNWIND_DIR=%s' % spec['libunwind'].prefix, - '-DPAPI_DIR=%s' % spec['papi'].prefix, - '-DBOOST_DIR=%s' % spec['boost'].prefix, - '-DMRNET_DIR=%s' % spec['mrnet'].prefix, - '-DDYNINST_DIR=%s' % spec['dyninst'].prefix, - '-DXERCESC_DIR=%s' % spec['xerces-c'].prefix, - '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, - *std_cmake_args) + cmakeOptions = [] + cmakeOptions.extend(['-DCMAKE_INSTALL_PREFIX=%s' % prefix, + '-DCBTF_DIR=%s' % spec['cbtf'].prefix, + '-DBINUTILS_DIR=%s' % spec['binutils'].prefix, + '-DLIBMONITOR_DIR=%s' % spec['libmonitor'].prefix, + '-DLIBUNWIND_DIR=%s' % spec['libunwind'].prefix, + '-DPAPI_DIR=%s' % spec['papi'].prefix, + '-DBOOST_DIR=%s' % spec['boost'].prefix, + '-DMRNET_DIR=%s' % spec['mrnet'].prefix, + '-DDYNINST_DIR=%s' % spec['dyninst'].prefix, + '-DXERCESC_DIR=%s' % spec['xerces-c'].prefix + ]) + + + # Add any MPI implementations coming from variant settings + self.set_mpi_cmakeOptions(spec, cmakeOptions) + + # Add in the standard cmake arguments + cmakeOptions.extend(std_cmake_args) + + # Adjust the standard cmake arguments to what we want the build type, etc to be + self.adjustBuildTypeParams_cmakeOptions(spec, cmakeOptions) + + # Invoke cmake + cmake('..', *cmakeOptions) make("clean") make() make("install") + + + #if '+cray' in spec: + #if 'cray' in self.spec.architecture: + # if '+runtime' in spec: + # with working_dir('build_cbtf_cray_runtime', create=True): + # python_vers='%d.%d' % spec['python'].version[:2] + # cmake .. \ + # -DCMAKE_BUILD_TYPE=Debug \ + # -DTARGET_OS="cray" \ + # -DRUNTIME_ONLY="true" \ + # -DCMAKE_INSTALL_PREFIX=${CBTF_KRELL_PREFIX} \ + # -DCMAKE_PREFIX_PATH=${CBTF_ROOT} \ + # -DCBTF_DIR=${CBTF_ROOT} \ + # -DBOOST_ROOT=${BOOST_INSTALL_PREFIX} \ + # -DXERCESC_DIR=${XERCESC_INSTALL_PREFIX} \ + # -DBINUTILS_DIR=${KRELL_ROOT} \ + # -DLIBMONITOR_DIR=${KRELL_ROOT_COMPUTE} \ + # -DLIBUNWIND_DIR=${KRELL_ROOT_COMPUTE} \ + # -DPAPI_DIR=${PAPI_ROOT} \ + # -DDYNINST_DIR=${DYNINST_CN_ROOT} \ + # -DMRNET_DIR=${MRNET_INSTALL_PREFIX} \ + # -DMPICH2_DIR=/opt/cray/mpt/7.0.1/gni/mpich2-gnu/48 + # else: + # with working_dir('build_cbtf_cray_frontend', create=True): + # python_vers='%d.%d' % spec['python'].version[:2] + # cmake .. \ + # -DCMAKE_BUILD_TYPE=Debug \ + # -DCMAKE_INSTALL_PREFIX=${CBTF_KRELL_PREFIX} \ + # -DCMAKE_PREFIX_PATH=${CBTF_ROOT} \ + # -DCBTF_DIR=${CBTF_ROOT} \ + # -DRUNTIME_TARGET_OS="cray" \ + # -DCBTF_KRELL_CN_RUNTIME_DIR=${CBTF_KRELL_CN_RUNTIME_ROOT} \ + # -DCBTF_CN_RUNTIME_DIR=${CBTF_CN_RUNTIME_ROOT} \ + # -DLIBMONITOR_CN_RUNTIME_DIR=${LIBMONITOR_CN_ROOT} \ + # -DLIBUNWIND_CN_RUNTIME_DIR=${LIBUNWIND_CN_ROOT} \ + # -DPAPI_CN_RUNTIME_DIR=${PAPI_CN_ROOT} \ + # -DXERCESC_CN_RUNTIME_DIR=/${XERCESC_CN_ROOT} \ + # -DMRNET_CN_RUNTIME_DIR=${MRNET_CN_ROOT} \ + # -DBOOST_CN_RUNTIME_DIR=${BOOST_CN_ROOT} \ + # -DDYNINST_CN_RUNTIME_DIR=${DYNINST_CN_ROOT} \ + # -DBOOST_ROOT=/${KRELL_ROOT} \ + # -DXERCESC_DIR=/${KRELL_ROOT} \ + # -DBINUTILS_DIR=/${KRELL_ROOT} \ + # -DLIBMONITOR_DIR=${KRELL_ROOT} \ + # -DLIBUNWIND_DIR=${KRELL_ROOT} \ + # -DPAPI_DIR=${PAPI_ROOT} \ + # -DDYNINST_DIR=${KRELL_ROOT} \ + # -DMRNET_DIR=${KRELL_ROOT} \ + # -DMPICH2_DIR=/opt/cray/mpt/7.0.1/gni/mpich2-gnu/48 + # fi +# +# make("clean") +# make() +# make("install") +# +# elif '+mic' in spec: +# if '+runtime' in spec: +# with working_dir('build_cbtf_mic_runtime', create=True): +# python_vers='%d.%d' % spec['python'].version[:2] +# cmake .. \ +# +# else: +# with working_dir('build_cbtf_cray_frontend', create=True): +# python_vers='%d.%d' % spec['python'].version[:2] +# cmake .. \ +# fi +# +# else: +# # Build cbtf-krell with cmake +# with working_dir('build_cbtf_krell', create=True): +# cmake('..', +# '-DCMAKE_BUILD_TYPE=Debug', +# '-DCMAKE_INSTALL_PREFIX=%s' % prefix, +# '-DCBTF_DIR=%s' % spec['cbtf'].prefix, +# '-DBINUTILS_DIR=%s' % spec['binutils'].prefix, +# '-DLIBMONITOR_DIR=%s' % spec['libmonitor'].prefix, +# '-DLIBUNWIND_DIR=%s' % spec['libunwind'].prefix, +# '-DPAPI_DIR=%s' % spec['papi'].prefix, +# '-DBOOST_DIR=%s' % spec['boost'].prefix, +# '-DMRNET_DIR=%s' % spec['mrnet'].prefix, +# '-DDYNINST_DIR=%s' % spec['dyninst'].prefix, +# '-DXERCESC_DIR=%s' % spec['xerces-c'].prefix, +# '-DOPENMPI_DIR=%s' % openmpi_prefix_path, +# '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, +# *std_cmake_args) +# +# make("clean") +# make() +# make("install") +# +# fi +# diff --git a/var/spack/repos/builtin/packages/cbtf-lanl/package.py b/var/spack/repos/builtin/packages/cbtf-lanl/package.py index 2da9e8a1f7..5ca88601f3 100644 --- a/var/spack/repos/builtin/packages/cbtf-lanl/package.py +++ b/var/spack/repos/builtin/packages/cbtf-lanl/package.py @@ -1,5 +1,5 @@ ################################################################################ -# Copyright (c) 2015 Krell Institute. All Rights Reserved. +# Copyright (c) 2015-2016 Krell Institute. All Rights Reserved. # # This program is free software; you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free Software @@ -29,32 +29,65 @@ class CbtfLanl(Package): version('1.6', branch='master', git='http://git.code.sf.net/p/cbtf-lanl/cbtf-lanl') - + depends_on("cmake@3.0.2") # Dependencies for cbtf-krell - depends_on("boost@1.50") - depends_on("mrnet@4.1.0:+lwthreads") + depends_on("mrnet@5.0.1:+lwthreads+krellpatch") depends_on("xerces-c@3.1.1:") depends_on("cbtf") depends_on("cbtf-krell") parallel = False + def adjustBuildTypeParams_cmakeOptions(self, spec, cmakeOptions): + # Sets build type parameters into cmakeOptions the options that will enable the cbtf-krell built type settings + + compile_flags="-O2 -g" + BuildTypeOptions = [] + # Set CMAKE_BUILD_TYPE to what cbtf-krell wants it to be, not the stdcmakeargs + for word in cmakeOptions[:]: + if word.startswith('-DCMAKE_BUILD_TYPE'): + cmakeOptions.remove(word) + if word.startswith('-DCMAKE_CXX_FLAGS'): + cmakeOptions.remove(word) + if word.startswith('-DCMAKE_C_FLAGS'): + cmakeOptions.remove(word) + if word.startswith('-DCMAKE_VERBOSE_MAKEFILE'): + cmakeOptions.remove(word) + BuildTypeOptions.extend([ + '-DCMAKE_VERBOSE_MAKEFILE=ON', + '-DCMAKE_BUILD_TYPE=None', + '-DCMAKE_CXX_FLAGS=%s' % compile_flags, + '-DCMAKE_C_FLAGS=%s' % compile_flags + ]) + + cmakeOptions.extend(BuildTypeOptions) + def install(self, spec, prefix): # Add in paths for finding package config files that tell us where to find these packages cmake_prefix_path = join_path(spec['cbtf'].prefix) + ':' + join_path(spec['cbtf-krell'].prefix) with working_dir('build', create=True): - cmake('..', - '-DCBTF_DIR=%s' % spec['cbtf'].prefix, - '-DCBTF_KRELL_DIR=%s' % spec['cbtf-krell'].prefix, - '-DMRNET_DIR=%s' % spec['mrnet'].prefix, - '-DXERCESC_DIR=%s' % spec['xerces-c'].prefix, - '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, - '-DCMAKE_MODULE_PATH=%s' % join_path(prefix.share,'KrellInstitute','cmake'), - *std_cmake_args) - - make("clean") - make() - make("install") + cmakeOptions = [] + cmakeOptions.extend(['-DCMAKE_INSTALL_PREFIX=%s' % prefix, + '-DCBTF_DIR=%s' % spec['cbtf'].prefix, + '-DCBTF_KRELL_DIR=%s' % spec['cbtf-krell'].prefix, + '-DMRNET_DIR=%s' % spec['mrnet'].prefix, + '-DXERCESC_DIR=%s' % spec['xerces-c'].prefix, + '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, + '-DCMAKE_MODULE_PATH=%s' % join_path(prefix.share,'KrellInstitute','cmake') + ]) + + # Add in the standard cmake arguments + cmakeOptions.extend(std_cmake_args) + + # Adjust the standard cmake arguments to what we want the build type, etc to be + self.adjustBuildTypeParams_cmakeOptions(spec, cmakeOptions) + + # Invoke cmake + cmake('..', *cmakeOptions) + + make("clean") + make() + make("install") diff --git a/var/spack/repos/builtin/packages/cbtf/package.py b/var/spack/repos/builtin/packages/cbtf/package.py index 52e6a07020..7ce1cd382b 100644 --- a/var/spack/repos/builtin/packages/cbtf/package.py +++ b/var/spack/repos/builtin/packages/cbtf/package.py @@ -1,5 +1,5 @@ ################################################################################ -# Copyright (c) 2015 Krell Institute. All Rights Reserved. +# Copyright (c) 2015-2016 Krell Institute. All Rights Reserved. # # This program is free software; you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free Software @@ -25,21 +25,44 @@ class Cbtf(Package): homepage = "http://sourceforge.net/p/cbtf/wiki/Home" # Mirror access template example - #url = "file:/g/g24/jeg/cbtf-1.5.tar.gz" - #version('1.6', '1ca88a8834759c4c74452cb97fe7b70a') + #url = "file:/home/jeg/cbtf-1.6.tar.gz" + #version('1.6', 'c1ef4e5aa4e470dffb042abdba0b9987') # Use when the git repository is available - version('1.6', branch='master', git='http://git.code.sf.net/p/cbtf/cbtf') + version('1.6', branch='master', git='https://github.com/OpenSpeedShop/cbtf.git') - depends_on("cmake") - #depends_on("boost@1.42.0:") - depends_on("boost@1.50.0") - depends_on("mrnet@4.1.0+lwthreads") + variant('runtime', default=False, description="build only the runtime libraries and collectors.") + + depends_on("cmake@3.0.2") + depends_on("boost@1.50.0:") + depends_on("mrnet@5.0.1:+lwthreads+krellpatch") depends_on("xerces-c@3.1.1:") - depends_on("libxml2") + # Work around for spack libxml2 package bug, take off python when fixed + depends_on("libxml2+python") parallel = False + def adjustBuildTypeParams_cmakeOptions(self, spec, cmakeOptions): + # Sets build type parameters into cmakeOptions the options that will enable the cbtf-krell built type settings + + compile_flags="-O2 -g" + BuildTypeOptions = [] + # Set CMAKE_BUILD_TYPE to what cbtf-krell wants it to be, not the stdcmakeargs + for word in cmakeOptions[:]: + if word.startswith('-DCMAKE_BUILD_TYPE'): + cmakeOptions.remove(word) + if word.startswith('-DCMAKE_CXX_FLAGS'): + cmakeOptions.remove(word) + if word.startswith('-DCMAKE_C_FLAGS'): + cmakeOptions.remove(word) + BuildTypeOptions.extend([ + '-DCMAKE_BUILD_TYPE=None', + '-DCMAKE_CXX_FLAGS=%s' % compile_flags, + '-DCMAKE_C_FLAGS=%s' % compile_flags + ]) + + cmakeOptions.extend(BuildTypeOptions) + def install(self, spec, prefix): with working_dir('build', create=True): @@ -48,14 +71,45 @@ class Cbtf(Package): # or BOOST_INCLUDEDIR). Useful when specifying BOOST_ROOT. # Defaults to OFF. - cmake('..', - '--debug-output', - '-DBoost_NO_SYSTEM_PATHS=TRUE', - '-DXERCESC_DIR=%s' % spec['xerces-c'].prefix, - '-DBOOST_ROOT=%s' % spec['boost'].prefix, - '-DMRNET_DIR=%s' % spec['mrnet'].prefix, - '-DCMAKE_MODULE_PATH=%s' % join_path(prefix.share,'KrellInstitute','cmake'), - *std_cmake_args) + if '+runtime' in spec: + # Install message tag include file for use in Intel MIC cbtf-krell build + # FIXME + cmakeOptions = [] + cmakeOptions.extend(['-DCMAKE_INSTALL_PREFIX=%s' % prefix, + '-DBoost_NO_SYSTEM_PATHS=TRUE', + '-DXERCESC_DIR=%s' % spec['xerces-c'].prefix, + '-DBOOST_ROOT=%s' % spec['boost'].prefix, + '-DMRNET_DIR=%s' % spec['mrnet'].prefix, + '-DCMAKE_MODULE_PATH=%s' % join_path(prefix.share,'KrellInstitute','cmake') + ]) + + # Add in the standard cmake arguments + cmakeOptions.extend(std_cmake_args) + + # Adjust the standard cmake arguments to what we want the build type, etc to be + self.adjustBuildTypeParams_cmakeOptions(spec, cmakeOptions) + + # Invoke cmake + cmake('..', *cmakeOptions) + + else: + cmakeOptions = [] + cmakeOptions.extend(['-DCMAKE_INSTALL_PREFIX=%s' % prefix, + '-DBoost_NO_SYSTEM_PATHS=TRUE', + '-DXERCESC_DIR=%s' % spec['xerces-c'].prefix, + '-DBOOST_ROOT=%s' % spec['boost'].prefix, + '-DMRNET_DIR=%s' % spec['mrnet'].prefix, + '-DCMAKE_MODULE_PATH=%s' % join_path(prefix.share,'KrellInstitute','cmake') + ]) + + # Add in the standard cmake arguments + cmakeOptions.extend(std_cmake_args) + + # Adjust the standard cmake arguments to what we want the build type, etc to be + self.adjustBuildTypeParams_cmakeOptions(spec, cmakeOptions) + + # Invoke cmake + cmake('..', *cmakeOptions) make("clean") make() diff --git a/var/spack/repos/builtin/packages/openspeedshop/package.py b/var/spack/repos/builtin/packages/openspeedshop/package.py index 8c71bcb7c3..bcd77351aa 100644 --- a/var/spack/repos/builtin/packages/openspeedshop/package.py +++ b/var/spack/repos/builtin/packages/openspeedshop/package.py @@ -1,5 +1,5 @@ ################################################################################ -# Copyright (c) 2015 Krell Institute. All Rights Reserved. +# Copyright (c) 2015-2016 Krell Institute. All Rights Reserved. # # This program is free software; you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free Software @@ -28,20 +28,15 @@ class Openspeedshop(Package): as open source code primarily under LGPL. """ - homepage = "http://www.openspeedshop.org" - url = "http://sourceforge.net/projects/openss/files/openss/openspeedshop-2.2/openspeedshop-2.2.tar.gz/download" + url = "https://github.com/OpenSpeedShop" version('2.2', '16cb051179c2038de4e8a845edf1d573') + # Use when the git repository is available + version('2.2', branch='master', git='https://github.com/OpenSpeedShop/openspeedshop.git') - #homepage = "http://www.openspeedshop.org" - #url = "http://sourceforge.net/projects/openss/files/openss/openspeedshop-2.1/openspeedshop-2.1.tar.gz/download" - #version('2.1', 'bdaa57c1a0db9d0c3e0303fd8496c507') - - # optional mirror template - #url = "file:/g/g24/jeg/openspeedshop-2.1.tar.gz" - #version('2.1', '64ee17166519838c7b94a1adc138e94f') - - + # Optional mirror template + #url = "file:/home/jeg/OpenSpeedShop_ROOT/SOURCES/openspeedshop-2.2.tar.gz" + #version('2.2', '643337740dc6c2faca60f42d3620b0e1') parallel = False @@ -51,11 +46,17 @@ class Openspeedshop(Package): variant('frontend', default=False, description="build only the front-end tool using the runtime_dir to point to the target build.") variant('cuda', default=False, description="build with cuda packages included.") variant('ptgf', default=False, description="build with the PTGF based gui package enabled.") - variant('intelmic', default=False, description="build for the Intel MIC platform.") - variant('cray', default=False, description="build for Cray platforms.") - variant('bluegene', default=False, description="build for Cray platforms.") variant('rtfe', default=False, description="build for generic cluster platforms that have different processors on the fe and be nodes.") + # MPI variants + variant('openmpi', default=False, description="Build mpi experiment collector for openmpi MPI when this variant is enabled.") + variant('mpt', default=False, description="Build mpi experiment collector for SGI MPT MPI when this variant is enabled.") + variant('mvapich2', default=False, description="Build mpi experiment collector for mvapich2 MPI when this variant is enabled.") + variant('mvapich', default=False, description="Build mpi experiment collector for mvapich MPI when this variant is enabled.") + variant('mpich2', default=False, description="Build mpi experiment collector for mpich2 MPI when this variant is enabled.") + variant('mpich', default=False, description="Build mpi experiment collector for mpich MPI when this variant is enabled.") + + depends_on("cmake@3.0.2") # Dependencies for openspeedshop that are common to all the variants of the OpenSpeedShop build depends_on("bison") depends_on("flex") @@ -63,8 +64,8 @@ class Openspeedshop(Package): depends_on("libelf") depends_on("libdwarf") depends_on("sqlite") - depends_on("boost@1.50.0") - depends_on("dyninst@8.2.1") + depends_on("boost@1.50.0:") + depends_on("dyninst@9.1.0") depends_on("python") depends_on("qt@3.3.8b+krellpatch") @@ -72,15 +73,78 @@ class Openspeedshop(Package): depends_on("libunwind", when='+offline') depends_on("papi", when='+offline') depends_on("libmonitor+krellpatch", when='+offline') - #depends_on("openmpi+krelloptions", when='+offline') - #depends_on("openmpi", when='+offline') - #depends_on("mpich", when='+offline') + depends_on("openmpi", when='+offline+openmpi') + depends_on("mpich", when='+offline+mpich') + depends_on("mpich2", when='+offline+mpich2') + depends_on("mvapich2", when='+offline+mvapich2') + depends_on("mvapich", when='+offline+mvapich') + depends_on("mpt", when='+offline+mpt') # Dependencies only for the openspeedshop cbtf package. depends_on("cbtf", when='+cbtf') depends_on("cbtf-krell", when='+cbtf') - depends_on("cbtf-argonavis", when='+cbtf') - depends_on("mrnet@4.1.0:+lwthreads", when='+cbtf') + depends_on("cbtf-argonavis", when='+cbtf+cuda') + depends_on("mrnet@5.0.1:+lwthreads+krellpatch", when='+cbtf') + + def adjustBuildTypeParams_cmakeOptions(self, spec, cmakeOptions): + # Sets build type parameters into cmakeOptions the options that will enable the cbtf-krell built type settings + + compile_flags="-O2 -g" + BuildTypeOptions = [] + # Set CMAKE_BUILD_TYPE to what cbtf-krell wants it to be, not the stdcmakeargs + for word in cmakeOptions[:]: + if word.startswith('-DCMAKE_BUILD_TYPE'): + cmakeOptions.remove(word) + if word.startswith('-DCMAKE_CXX_FLAGS'): + cmakeOptions.remove(word) + if word.startswith('-DCMAKE_C_FLAGS'): + cmakeOptions.remove(word) + BuildTypeOptions.extend([ + '-DCMAKE_BUILD_TYPE=None', + '-DCMAKE_CXX_FLAGS=%s' % compile_flags, + '-DCMAKE_C_FLAGS=%s' % compile_flags + ]) + + cmakeOptions.extend(BuildTypeOptions) + + def set_mpi_cmakeOptions(self, spec, cmakeOptions): + # Appends to cmakeOptions the options that will enable the appropriate MPI implementations + + MPIOptions = [] + + # openmpi + if '+openmpi' in spec: + MPIOptions.extend([ + '-DOPENMPI_DIR=%s' % spec['openmpi'].prefix + ]) + # mpich + if '+mpich' in spec: + MPIOptions.extend([ + '-DMPICH_DIR=%s' % spec['mpich'].prefix + ]) + # mpich2 + if '+mpich2' in spec: + MPIOptions.extend([ + '-DMPICH2_DIR=%s' % spec['mpich2'].prefix + ]) + # mvapich + if '+mvapich' in spec: + MPIOptions.extend([ + '-DMVAPICH_DIR=%s' % spec['mvapich'].prefix + ]) + # mvapich2 + if '+mvapich2' in spec: + MPIOptions.extend([ + '-DMVAPICH2_DIR=%s' % spec['mvapich2'].prefix + ]) + # mpt + if '+mpt' in spec: + MPIOptions.extend([ + '-DMPT_DIR=%s' % spec['mpt'].prefix + ]) + + cmakeOptions.extend(MPIOptions) + def install(self, spec, prefix): @@ -100,51 +164,118 @@ class Openspeedshop(Package): instrumentor_setting = "offline" if '+runtime' in spec: with working_dir('build_runtime', create=True): - cmake('..', - '-DCMAKE_INSTALL_PREFIX=%s' % prefix, - '-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, - '-DINSTRUMENTOR=%s' % instrumentor_setting, - '-DLIBMONITOR_DIR=%s' % spec['libmonitor'].prefix, - '-DLIBUNWIND_DIR=%s' % spec['libunwind'].prefix, - '-DPAPI_DIR=%s' % spec['papi'].prefix, - *std_cmake_args) + + cmakeOptions = [] + cmakeOptions.extend(['-DCMAKE_INSTALL_PREFIX=%s' % prefix, + '-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, + '-DINSTRUMENTOR=%s' % instrumentor_setting, + '-DLIBMONITOR_DIR=%s' % spec['libmonitor'].prefix, + '-DLIBUNWIND_DIR=%s' % spec['libunwind'].prefix, + '-DPAPI_DIR=%s' % spec['papi'].prefix + ]) + + # Add any MPI implementations coming from variant settings + self.set_mpi_cmakeOptions(spec, cmakeOptions) + cmakeOptions.extend(std_cmake_args) + + # Adjust the build options to the favored ones for this build + self.adjustBuildTypeParams_cmakeOptions(spec, cmakeOptions) + + cmake('..', *cmakeOptions) + make("clean") make() make("install") else: cmake_prefix_path = join_path(spec['dyninst'].prefix) with working_dir('build', create=True): + #python_vers=join_path(spec['python'].version[:2]) #'-DOPENMPI_DIR=%s' % openmpi_prefix_path, #'-DMVAPICH_DIR=%s' % mvapich_prefix_path, + #'-DMPICH_DIR=%s' % spec['mpich'].prefix, + #'-DMPICH2_DIR=%s' % spec['mpich2'].prefix, + #'-DBoost_NO_SYSTEM_PATHS=TRUE', + #'-DBOOST_ROOT=%s' % spec['boost'].prefix, + #'-DOPENMPI_DIR=%s' % spec['openmpi'].prefix, + python_vers='%d.%d' % spec['python'].version[:2] - cmake('..', - '-DCMAKE_INSTALL_PREFIX=%s' % prefix, - '-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, - '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, - '-DINSTRUMENTOR=%s' % instrumentor_setting, - '-DBINUTILS_DIR=%s' % spec['binutils'].prefix, - '-DLIBELF_DIR=%s' % spec['libelf'].prefix, - '-DLIBDWARF_DIR=%s' % spec['libdwarf'].prefix, - '-DLIBMONITOR_DIR=%s' % spec['libmonitor'].prefix, - '-DLIBUNWIND_DIR=%s' % spec['libunwind'].prefix, - '-DPAPI_DIR=%s' % spec['papi'].prefix, - '-DSQLITE3_DIR=%s' % spec['sqlite'].prefix, - '-DQTLIB_DIR=%s' % spec['qt'].prefix, - '-DPYTHON_EXECUTABLE=%s' % join_path(spec['python'].prefix + '/bin/python'), - '-DPYTHON_INCLUDE_DIR=%s' % join_path(spec['python'].prefix.include) + '/python' + python_vers, - '-DPYTHON_LIBRARY=%s' % join_path(spec['python'].prefix.lib) + '/libpython' + python_vers + '.so', - '-DBoost_NO_SYSTEM_PATHS=TRUE', - '-DBOOST_ROOT=%s' % spec['boost'].prefix, - '-DDYNINST_DIR=%s' % spec['dyninst'].prefix, - *std_cmake_args) + + cmakeOptions = [] + cmakeOptions.extend(['-DCMAKE_INSTALL_PREFIX=%s' % prefix, + '-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, + '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, + '-DINSTRUMENTOR=%s' % instrumentor_setting, + '-DBINUTILS_DIR=%s' % spec['binutils'].prefix, + '-DLIBELF_DIR=%s' % spec['libelf'].prefix, + '-DLIBDWARF_DIR=%s' % spec['libdwarf'].prefix, + '-DLIBMONITOR_DIR=%s' % spec['libmonitor'].prefix, + '-DLIBUNWIND_DIR=%s' % spec['libunwind'].prefix, + '-DPAPI_DIR=%s' % spec['papi'].prefix, + '-DSQLITE3_DIR=%s' % spec['sqlite'].prefix, + '-DQTLIB_DIR=%s' % spec['qt'].prefix, + '-DPYTHON_EXECUTABLE=%s' % join_path(spec['python'].prefix + '/bin/python'), + '-DPYTHON_INCLUDE_DIR=%s' % join_path(spec['python'].prefix.include) + '/python' + python_vers, + '-DPYTHON_LIBRARY=%s' % join_path(spec['python'].prefix.lib) + '/libpython' + python_vers + '.so', + '-DBoost_NO_SYSTEM_PATHS=TRUE', + '-DBOOST_ROOT=%s' % spec['boost'].prefix, + '-DDYNINST_DIR=%s' % spec['dyninst'].prefix + ]) + + # Add any MPI implementations coming from variant settings + self.set_mpi_cmakeOptions(spec, cmakeOptions) + cmakeOptions.extend(std_cmake_args) + + # Adjust the build options to the favored ones for this build + self.adjustBuildTypeParams_cmakeOptions(spec, cmakeOptions) + + cmake('..', *cmakeOptions) + make("clean") make() make("install") elif '+cbtf' in spec: instrumentor_setting = "cbtf" + resolve_symbols = "symtabapi" cmake_prefix_path = join_path(spec['cbtf'].prefix) + ':' + join_path(spec['cbtf-krell'].prefix) + ':' + join_path(spec['dyninst'].prefix) + #runtime_platform_cray = "cray" + #if '+cray' in spec: + # if '+runtime' in spec: + # #-DCBTF_KRELL_CN_RUNTIME_DIR=${CBTF_KRELL_CN_INSTALL_DIR} \ + # with working_dir('build_cbtf_cray_runtime', create=True): + # python_vers='%d.%d' % spec['python'].version[:2] + # cmake('..', + # '-DCMAKE_INSTALL_PREFIX=%s' % prefix, + # '-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, + # '-DRUNTIME_PLATFORM=%s' % runtime_platform_cray, + # '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, + # '-DRESOLVE_SYMBOLS=%s' % resolve_symbols, + # '-DINSTRUMENTOR=%s' % instrumentor_setting, + # '-DCBTF_DIR=%s' % spec['cbtf'].prefix, + # '-DCBTF_KRELL_DIR=%s' % spec['cbtf-krell'].prefix, + # '-DCBTF_KRELL_CN_RUNTIME_DIR=%s' % spec['cbtf-krell'].prefix, + # '-DBINUTILS_DIR=%s' % spec['binutils'].prefix, + # '-DLIBELF_DIR=%s' % spec['libelf'].prefix, + # '-DLIBDWARF_DIR=%s' % spec['libdwarf'].prefix, + # '-DLIBUNWIND_DIR=%s' % spec['libunwind'].prefix, + # '-DPAPI_DIR=%s' % spec['papi'].prefix, + # '-DDYNINST_DIR=%s' % spec['dyninst'].prefix, + # '-DXERCESC_DIR=%s' % spec['xerces-c'].prefix, + # '-DMRNET_DIR=%s' % spec['mrnet'].prefix, + # '-DBoost_NO_SYSTEM_PATHS=TRUE', + # '-DBOOST_ROOT=%s' % spec['boost'].prefix, + # *std_cmake_args) + + # make("clean") + # make() + # make("install") + + + #elif '+mic' in spec: + # comment out else and shift over the default case below until arch detection is in + #else: + if '+runtime' in spec: with working_dir('build_cbtf_runtime', create=True): python_vers='%d.%d' % spec['python'].version[:2] @@ -203,14 +334,63 @@ class Openspeedshop(Package): # tbd - #if '+intelmic' in spec: - # with working_dir('build_intelmic_compute', create=True): - # tbd - # with working_dir('build_intelmic_frontend', create=True): - # tbd - #if '+cray' in spec: - # with working_dir('build_cray_compute', create=True): - # tbd - # with working_dir('build_cray_frontend', create=True): - # tbd + #if '+cbtf' in spec: + # if cray build type detected: + # if '+runtime' in spec: + # with working_dir('build_cray_cbtf_compute', create=True): + # tbd + # else: + # with working_dir('build_cray_cbtf_frontend', create=True): + # tbd + # with working_dir('build_cray_osscbtf_frontend', create=True): + # tbd + # fi + # elif '+intelmic' in spec: + # if '+runtime' in spec: + # with working_dir('build_intelmic_cbtf_compute', create=True): + # tbd + # else: + # with working_dir('build_intelmic_cbtf_frontend', create=True): + # tbd + # with working_dir('build_intelmic_osscbtf_frontend', create=True): + # fi + # else + # with working_dir('build_cluster_cbtf', create=True): + # tbd + # with working_dir('build_cluster osscbtf', create=True): + # tbd + # fi + #elif '+offline' in spec: + # if cray build type detected: + # if '+runtime' in spec: + # with working_dir('build_cray_ossoff_compute', create=True): + # tbd + # else: + # with working_dir('build_cray_ossoff_frontend', create=True): + # tbd + # fi + # elif '+intelmic' in spec: + # if '+runtime' in spec: + # with working_dir('build_intelmic_ossoff_compute', create=True): + # tbd + # else: + # with working_dir('build_intelmic_ossoff_frontend', create=True): + # tbd + # fi + # elif bgq build type detected: + # if '+runtime' in spec: + # with working_dir('build_bgq_ossoff_compute', create=True): + # tbd + # else: + # with working_dir('build_bgq_ossoff_frontend', create=True): + # tbd + # fi + # else + # with working_dir('build_cluster ossoff', create=True): + # tbd + # fi + #fi + + + -- cgit v1.2.3-70-g09d2 From 5deaaa278ce011e2a143dd017a494d97378959c4 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 19 Apr 2016 14:25:12 +0200 Subject: modules : added a few unit tests --- lib/spack/spack/modules.py | 6 +- lib/spack/spack/test/__init__.py | 1 + lib/spack/spack/test/modules.py | 126 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 lib/spack/spack/test/modules.py diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 5bf003f9ad..bcccf2f9b8 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -132,17 +132,15 @@ def dependencies(spec, request='all'): if request == 'none': return [] - l = [xx for xx in - sorted(spec.traverse(order='post', depth=True, cover='nodes', root=False), reverse=True)] - if request == 'direct': - return [xx for ii, xx in l if ii == 1] + return [xx for _, xx in spec.dependencies.items()] # FIXME : during module file creation nodes seem to be visited multiple times even if cover='nodes' # FIXME : is given. This work around permits to get a unique list of spec anyhow. # FIXME : Possibly we miss a merge step among nodes that refer to the same package. seen = set() seen_add = seen.add + l = [xx for xx in sorted(spec.traverse(order='post', depth=True, cover='nodes', root=False), reverse=True)] return [xx for ii, xx in l if not (xx in seen or seen_add(xx))] diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 175a49428c..b1c91c7903 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -54,6 +54,7 @@ test_names = ['versions', 'svn_fetch', 'hg_fetch', 'mirror', + 'modules', 'url_extrapolate', 'cc', 'link_tree', diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py new file mode 100644 index 0000000000..21afa355e7 --- /dev/null +++ b/lib/spack/spack/test/modules.py @@ -0,0 +1,126 @@ +import collections +from contextlib import contextmanager + +import StringIO + +FILE_REGISTRY = collections.defaultdict(StringIO.StringIO) + +# Monkey-patch open to write module files to a StringIO instance +@contextmanager +def mock_open(filename, mode): + if not mode == 'w': + raise RuntimeError('test.modules : unexpected opening mode for monkey-patched open') + + FILE_REGISTRY[filename] = StringIO.StringIO() + + try: + yield FILE_REGISTRY[filename] + finally: + handle = FILE_REGISTRY[filename] + FILE_REGISTRY[filename] = handle.getvalue() + handle.close() + +import spack.modules + +configuration_autoload_direct = { + 'enable': ['tcl'], + 'tcl': { + 'all': { + 'autoload': 'direct' + } + } +} + +configuration_autoload_all = { + 'enable': ['tcl'], + 'tcl': { + 'all': { + 'autoload': 'all' + } + } +} + +configuration_alter_environment = { + 'enable': ['tcl'], + 'tcl': { + 'all': { + 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']} + }, + '=x86-linux': { + 'environment': {'set': ['FOO,foo'], 'unset': ['BAR']} + } + } +} + +configuration_blacklist = { + 'enable': ['tcl'], + 'tcl': { + 'blacklist': ['callpath'], + 'all': { + 'autoload': 'direct' + } + } +} + +from spack.test.mock_packages_test import MockPackagesTest + + +class TclTests(MockPackagesTest): + def setUp(self): + super(TclTests, self).setUp() + self.configuration_obj = spack.modules.CONFIGURATION + spack.modules.open = mock_open + spack.modules.CONFIGURATION = None # Make sure that a non-mocked configuration will trigger an error + + def tearDown(self): + del spack.modules.open + spack.modules.CONFIGURATION = self.configuration_obj + super(TclTests, self).tearDown() + + def get_modulefile_content(self, spec): + spec.concretize() + generator = spack.modules.TclModule(spec) + generator.write() + content = FILE_REGISTRY[generator.file_name].split('\n') + return content + + def test_simple_case(self): + spack.modules.CONFIGURATION = configuration_autoload_direct + spec = spack.spec.Spec('mpich@3.0.4=x86-linux') + content = self.get_modulefile_content(spec) + self.assertTrue('module-whatis "mpich @3.0.4"' in content ) + self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 1) + + def test_autoload(self): + spack.modules.CONFIGURATION = configuration_autoload_direct + spec = spack.spec.Spec('mpileaks=x86-linux') + content = self.get_modulefile_content(spec) + self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2) + self.assertEqual(len([x for x in content if 'module load ' in x]), 2) + + spack.modules.CONFIGURATION = configuration_autoload_all + spec = spack.spec.Spec('mpileaks=x86-linux') + content = self.get_modulefile_content(spec) + self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5) + self.assertEqual(len([x for x in content if 'module load ' in x]), 5) + + def test_alter_environment(self): + spack.modules.CONFIGURATION = configuration_alter_environment + spec = spack.spec.Spec('mpileaks=x86-linux') + content = self.get_modulefile_content(spec) + self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) + self.assertEqual(len([x for x in content if 'setenv FOO "foo"' in x]), 1) + self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 1) + + spec = spack.spec.Spec('libdwarf=x64-linux') + content = self.get_modulefile_content(spec) + self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) + self.assertEqual(len([x for x in content if 'setenv FOO "foo"' in x]), 0) + self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 0) + + def test_blacklist(self): + spack.modules.CONFIGURATION = configuration_blacklist + spec = spack.spec.Spec('mpileaks=x86-linux') + content = self.get_modulefile_content(spec) + self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) + self.assertEqual(len([x for x in content if 'module load ' in x]), 1) -- cgit v1.2.3-70-g09d2 From 7c155f74567a9e3d949369260e732daf13310269 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 20 Apr 2016 16:33:59 -0400 Subject: Check the installed HDF5 library for consistency --- var/spack/repos/builtin/packages/hdf5/package.py | 63 ++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index cce609eb29..eb58eeb637 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -24,6 +24,8 @@ ############################################################################## from spack import * +import shutil +import subprocess class Hdf5(Package): @@ -114,14 +116,16 @@ class Hdf5(Package): # this is not actually a problem. extra_args.extend([ "--enable-parallel", - "CC=%s" % spec['mpi'].prefix.bin + "/mpicc", + "CC=%s" % join_path(spec['mpi'].prefix.bin, "mpicc"), ]) if '+cxx' in spec: - extra_args.append("CXX=%s" % spec['mpi'].prefix.bin + "/mpic++") + extra_args.append("CXX=%s" % join_path(spec['mpi'].prefix.bin, + "mpic++")) if '+fortran' in spec: - extra_args.append("FC=%s" % spec['mpi'].prefix.bin + "/mpifort") + extra_args.append("FC=%s" % join_path(spec['mpi'].prefix.bin, + "mpifort")) if '+szip' in spec: extra_args.append("--with-szlib=%s" % spec['szip'].prefix) @@ -138,6 +142,59 @@ class Hdf5(Package): *extra_args) make() make("install") + self.check_install(spec) + + def check_install(self, spec): + "Build and run a small program to test the installed HDF5 library" + print "Checking HDF5 installation..." + checkdir = "spack-check" + with working_dir(checkdir, create=True): + source = r""" +#include +#include +#include +int main(int argc, char **argv) { + unsigned majnum, minnum, relnum; + herr_t herr = H5get_libversion(&majnum, &minnum, &relnum); + assert(!herr); + printf("HDF5 version %d.%d.%d %u.%u.%u\n", H5_VERS_MAJOR, H5_VERS_MINOR, + H5_VERS_RELEASE, majnum, minnum, relnum); + return 0; +} +""" + expected = """\ +HDF5 version {version} {version} +""".format(version=str(spec.version)) + with open("check.c", 'w') as f: + f.write(source) + if '+mpi' in spec: + cc = which(join_path(spec['mpi'].prefix.bin, "mpicc")) + else: + cc = which('cc') + # TODO: Automate these path settings + cc('-c', "-I%s" % join_path(spec.prefix, "include"), "check.c") + cc('-o', "check", "check.o", + # I couldn't make libraries work on Darwin + "-L%s" % join_path(spec.prefix, "lib"), "-lhdf5", + # join_path(spec.prefix, "lib", "libhdf5.a"), + "-lz") + try: + output = subprocess.check_output("./check") + except: + output = "" + success = output == expected + if not success: + print "Produced output does not match expected output." + print "Expected output:" + print '-'*80 + print expected + print '-'*80 + print "Produced output:" + print '-'*80 + print output + print '-'*80 + raise RuntimeError("HDF5 install check failed") + shutil.rmtree(checkdir) def url_for_version(self, version): v = str(version) -- cgit v1.2.3-70-g09d2 From 867e1333d0c5973f51944418ad3509ff1cc4e131 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 20 Apr 2016 17:01:26 -0400 Subject: Remove outdated comment --- var/spack/repos/builtin/packages/hdf5/package.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index eb58eeb637..bef34ff5bd 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -171,12 +171,10 @@ HDF5 version {version} {version} cc = which(join_path(spec['mpi'].prefix.bin, "mpicc")) else: cc = which('cc') - # TODO: Automate these path settings + # TODO: Automate these path and library settings cc('-c', "-I%s" % join_path(spec.prefix, "include"), "check.c") cc('-o', "check", "check.o", - # I couldn't make libraries work on Darwin "-L%s" % join_path(spec.prefix, "lib"), "-lhdf5", - # join_path(spec.prefix, "lib", "libhdf5.a"), "-lz") try: output = subprocess.check_output("./check") -- cgit v1.2.3-70-g09d2 From 67a01ef2ee182697df49f808654cf17f0e164b4a Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 22 Apr 2016 10:50:24 +0200 Subject: tcl : extended conflict to be an array of strings --- lib/spack/spack/config.py | 2 +- lib/spack/spack/modules.py | 30 ++++++++++++++++++------------ lib/spack/spack/test/modules.py | 18 ++++++++++++++++++ 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 1ac8a01619..71cdff09ea 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -276,7 +276,7 @@ section_schemas = { }, 'autoload': {'$ref': '#/definitions/dependency_selection'}, 'prerequisites': {'$ref': '#/definitions/dependency_selection'}, - 'conflict': {'type': 'string'}, + 'conflict': {'$ref': '#/definitions/array_of_strings'}, 'environment': { 'type': 'object', 'default': {}, diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index bcccf2f9b8..57a4a2c754 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -46,6 +46,7 @@ import os import os.path import re import textwrap +import string import llnl.util.tty as tty import spack @@ -499,15 +500,20 @@ class TclModule(EnvModule): def module_specific_content(self, configuration): naming_tokens = self.tokens # Conflict - conflict_format = configuration.get('conflict', '') - if conflict_format: - for naming_dir, conflict_dir in zip(self.naming_scheme.split('/'), conflict_format.split('/')): - if naming_dir != conflict_dir: - message = 'Conflict scheme does not match naming scheme [{spec}]\n\n' - message += 'naming scheme : "{nformat}"\n' - message += 'conflict scheme : "{cformat}"\n' - raise tty.error( - message.format(spec=self.spec, nformat=self.naming_scheme, cformat=conflict_format) - ) - conflict_format = 'conflict ' + conflict_format - yield conflict_format.format(**naming_tokens) + conflict_format = configuration.get('conflict', []) + f = string.Formatter() + for item in conflict_format: + line = 'conflict ' + item + '\n' + if len([x for x in f.parse(line)]) > 1: # We do have placeholder to substitute + for naming_dir, conflict_dir in zip(self.naming_scheme.split('/'), item.split('/')): + if naming_dir != conflict_dir: + message = 'conflict scheme does not match naming scheme [{spec}]\n\n' + message += 'naming scheme : "{nformat}"\n' + message += 'conflict scheme : "{cformat}"\n\n' + message += '** You may want to check your `modules.yaml` configuration file **\n' + tty.error( + message.format(spec=self.spec, nformat=self.naming_scheme, cformat=item) + ) + raise SystemExit('Module generation aborted.') + line = line.format(**naming_tokens) + yield line diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 21afa355e7..704700417b 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -62,6 +62,16 @@ configuration_blacklist = { } } +configuration_conflicts = { + 'enable': ['tcl'], + 'tcl': { + 'naming_scheme': '{name}/{version}-{compiler.name}', + 'all': { + 'conflict': ['{name}', 'intel/14.0.1'] + } + } +} + from spack.test.mock_packages_test import MockPackagesTest @@ -124,3 +134,11 @@ class TclTests(MockPackagesTest): content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) self.assertEqual(len([x for x in content if 'module load ' in x]), 1) + + def test_conflicts(self): + spack.modules.CONFIGURATION = configuration_conflicts + spec = spack.spec.Spec('mpileaks=x86-linux') + content = self.get_modulefile_content(spec) + self.assertEqual(len([x for x in content if x.startswith('conflict')]), 2) + self.assertEqual(len([x for x in content if x == 'conflict mpileaks']), 1) + self.assertEqual(len([x for x in content if x == 'conflict intel/14.0.1']), 1) -- cgit v1.2.3-70-g09d2 From 0cbaecca60d5852ef0fe0d1378c685ff1dc65fd1 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 22 Apr 2016 21:52:11 -0400 Subject: Don't use subprocess module --- var/spack/repos/builtin/packages/hdf5/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index bef34ff5bd..333c63c288 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -25,7 +25,6 @@ from spack import * import shutil -import subprocess class Hdf5(Package): @@ -177,7 +176,8 @@ HDF5 version {version} {version} "-L%s" % join_path(spec.prefix, "lib"), "-lhdf5", "-lz") try: - output = subprocess.check_output("./check") + check = Executable('./check') + output = check(return_output=True) except: output = "" success = output == expected -- cgit v1.2.3-70-g09d2 From 62d175c984c5413fd95185d7bb3280bad69e1e33 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 23 Apr 2016 12:30:34 -0500 Subject: This commit explicitly sets `libdir`. This is necessary because different systems use different defaults. --- var/spack/repos/builtin/packages/R/package.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/R/package.py b/var/spack/repos/builtin/packages/R/package.py index 3a76416f27..7c4aa3520c 100644 --- a/var/spack/repos/builtin/packages/R/package.py +++ b/var/spack/repos/builtin/packages/R/package.py @@ -50,9 +50,12 @@ class R(Package): depends_on('tk') def install(self, spec, prefix): + rlibdir = join_path(prefix, 'rlib') options = ['--prefix=%s' % prefix, + '--libdir=%s' % rlibdir, '--enable-R-shlib', - '--enable-BLAS-shlib'] + '--enable-BLAS-shlib', + '--enable-R-framework=no'] if '+external-lapack' in spec: options.extend(['--with-blas', '--with-lapack']) @@ -66,7 +69,7 @@ class R(Package): @property def r_lib_dir(self): - return os.path.join('lib64', 'R', 'library') + return os.path.join('rlib', 'R', 'library') def setup_dependent_environment(self, spack_env, run_env, extension_spec): # Set R_LIBS to include the library dir for the -- cgit v1.2.3-70-g09d2 From 2b7b7f6d9758d47bb59b59e92cb2c7e3b785ac51 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 24 Apr 2016 18:11:18 -0500 Subject: Reformat description text. --- var/spack/repos/builtin/packages/r-BiocGenerics/package.py | 1 + var/spack/repos/builtin/packages/r-abind/package.py | 5 ++++- var/spack/repos/builtin/packages/r-filehash/package.py | 10 +++++++++- var/spack/repos/builtin/packages/r-magic/package.py | 6 +++++- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py index e2d9bb9594..2d92c1c4d8 100644 --- a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py +++ b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py @@ -2,6 +2,7 @@ from spack import * class RBiocgenerics(Package): """S4 generic functions needed by many Bioconductor packages.""" + homepage = 'https://www.bioconductor.org/packages/release/bioc/html/BiocGenerics.html' url = "https://www.bioconductor.org/packages/release/bioc/src/contrib/BiocGenerics_0.16.1.tar.gz" diff --git a/var/spack/repos/builtin/packages/r-abind/package.py b/var/spack/repos/builtin/packages/r-abind/package.py index 54d399c432..d06c7e9240 100644 --- a/var/spack/repos/builtin/packages/r-abind/package.py +++ b/var/spack/repos/builtin/packages/r-abind/package.py @@ -1,7 +1,10 @@ from spack import * class RAbind(Package): - """Combine multidimensional arrays into a single array. This is a generalization of 'cbind' and 'rbind'. Works with vectors, matrices, and higher-dimensional arrays. Also provides functions 'adrop', 'asub', and 'afill' for manipulating, extracting and replacing data in arrays.""" + """Combine multidimensional arrays into a single array. This is a + generalization of 'cbind' and 'rbind'. Works with vectors, matrices, and + higher-dimensional arrays. Also provides functions 'adrop', 'asub', and + 'afill' for manipulating, extracting and replacing data in arrays.""" homepage = "https://cran.r-project.org/" url = "https://cran.r-project.org/src/contrib/abind_1.4-3.tar.gz" diff --git a/var/spack/repos/builtin/packages/r-filehash/package.py b/var/spack/repos/builtin/packages/r-filehash/package.py index a3b688da10..4911c636b4 100644 --- a/var/spack/repos/builtin/packages/r-filehash/package.py +++ b/var/spack/repos/builtin/packages/r-filehash/package.py @@ -1,7 +1,15 @@ from spack import * class RFilehash(Package): - """Implements a simple key-value style database where character string keys are associated with data values that are stored on the disk. A simple interface is provided for inserting, retrieving, and deleting data from the database. Utilities are provided that allow 'filehash' databases to be treated much like environments and lists are already used in R. These utilities are provided to encourage interactive and exploratory analysis on large datasets. Three different file formats for representing the database are currently available and new formats can easily be incorporated by third parties for use in the 'filehash' framework.""" + """Implements a simple key-value style database where character string keys + are associated with data values that are stored on the disk. A simple + interface is provided for inserting, retrieving, and deleting data from the + database. Utilities are provided that allow 'filehash' databases to be + treated much like environments and lists are already used in R. These + utilities are provided to encourage interactive and exploratory analysis on + large datasets. Three different file formats for representing the database + are currently available and new formats can easily be incorporated by third + parties for use in the 'filehash' framework.""" homepage = 'https://cran.r-project.org/' url = "https://cran.r-project.org/src/contrib/filehash_2.3.tar.gz" diff --git a/var/spack/repos/builtin/packages/r-magic/package.py b/var/spack/repos/builtin/packages/r-magic/package.py index 5f25b2a162..e900cdb216 100644 --- a/var/spack/repos/builtin/packages/r-magic/package.py +++ b/var/spack/repos/builtin/packages/r-magic/package.py @@ -1,7 +1,11 @@ from spack import * class RMagic(Package): - """A collection of efficient, vectorized algorithms for the creation and investigation of magic squares and hypercubes, including a variety of functions for the manipulation and analysis of arbitrarily dimensioned arrays.""" + """A collection of efficient, vectorized algorithms for the creation and + investigation of magic squares and hypercubes, including a variety of + functions for the manipulation and analysis of arbitrarily dimensioned + arrays.""" + homepage = "https://cran.r-project.org/" url = "https://cran.r-project.org/src/contrib/magic_1.5-6.tar.gz" -- cgit v1.2.3-70-g09d2 From ff9145f8a5a95808d43db1dbc40414e644448419 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Wed, 30 Mar 2016 16:08:09 -0400 Subject: executable: quote arguments This allows command line arguments with spaces to be shown. The quoting madness is because a single quote cannot appear within a single quoted argument on the command line. To do so, you have to stop the single quote argument, double quote the single quote, then open the single quote again: $ echo 'before'"'"'after' before'after Fixes #174 --- lib/spack/spack/util/executable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index fc27b789d0..25819b6fc7 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -144,7 +144,7 @@ class Executable(object): cmd = self.exe + list(args) - cmd_line = ' '.join(cmd) + cmd_line = "'%s'" % "' '".join(map(lambda arg: arg.replace("'", "'\"'\"'"), cmd)) tty.debug(cmd_line) try: -- cgit v1.2.3-70-g09d2 From 1a585a6c748d9445ff691e766efba350259364b5 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Mon, 25 Apr 2016 22:18:38 -0400 Subject: Added nccmp package --- var/spack/repos/builtin/packages/nccmp/package.py | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 var/spack/repos/builtin/packages/nccmp/package.py diff --git a/var/spack/repos/builtin/packages/nccmp/package.py b/var/spack/repos/builtin/packages/nccmp/package.py new file mode 100644 index 0000000000..458afbb1e8 --- /dev/null +++ b/var/spack/repos/builtin/packages/nccmp/package.py @@ -0,0 +1,30 @@ +from spack import * +import os + +class Nccmp(Package): + """Compare NetCDF Files""" + homepage = "http://nccmp.sourceforge.net/" + url = "http://downloads.sourceforge.net/project/nccmp/nccmp-1.8.2.0.tar.gz" + + version('1.8.2.0', '81e6286d4413825aec4327e61a28a580') + + depends_on('netcdf') + + def install(self, spec, prefix): + # Configure says: F90 and F90FLAGS are replaced by FC and + # FCFLAGS respectively in this configure, please unset + # F90/F90FLAGS and set FC/FCFLAGS instead and rerun configure + # again. + os.environ['FC'] = os.environ['F90'] + del os.environ['F90'] + try: + os.environ['FCFLAGS'] = os.environ['F90FLAGS'] + del os.environ['F90FLAGS'] + except KeyError: # There are no flags + pass + + configure('--prefix=%s' % prefix) + + make() + make("check") + make("install") -- cgit v1.2.3-70-g09d2 From 55d339194d59f78b001ad939aae2e67f6ebae5ad Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 26 Apr 2016 15:29:16 -0400 Subject: Update hwloc to 1.11.3 --- var/spack/repos/builtin/packages/hwloc/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/hwloc/package.py b/var/spack/repos/builtin/packages/hwloc/package.py index ab7205646e..a461a7482c 100644 --- a/var/spack/repos/builtin/packages/hwloc/package.py +++ b/var/spack/repos/builtin/packages/hwloc/package.py @@ -17,6 +17,7 @@ class Hwloc(Package): list_url = "http://www.open-mpi.org/software/hwloc/" list_depth = 3 + version('1.11.3', 'c1d36a9de6028eac1d18ea4782ef958f') version('1.11.2', 'e4ca55c2a5c5656da4a4e37c8fc51b23') version('1.11.1', 'feb4e416a1b25963ed565d8b42252fdc') version('1.9', '1f9f9155682fe8946a97c08896109508') -- cgit v1.2.3-70-g09d2 From b7c064142e6e4beb960aa979815471c3889ec925 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Wed, 27 Apr 2016 01:23:53 +0200 Subject: preserve lookup order in PATH when invoking 'spack compiler add' --- lib/spack/spack/compiler.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index d38c0b00b1..2f89ec6144 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -202,6 +202,10 @@ class Compiler(object): return None successful = [key for key in parmap(check, checks) if key is not None] + # The 'successful' list is ordered like the input paths. + # Reverse it here so that the dict creation (last insert wins) + # does not spoil the intented precedence. + successful.reverse() return dict(((v, p, s), path) for v, p, s, path in successful) @classmethod -- cgit v1.2.3-70-g09d2 From b5ebd12fe2708997b63fea1a0f1cac43c3e8e847 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Wed, 27 Apr 2016 13:42:59 -0400 Subject: Add to sub dirs checked for pkg-config files. This lets me build against the pure-include package Eigen. --- lib/spack/spack/build_environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index eb72f2a6b4..cd9f647ddf 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -175,8 +175,8 @@ def set_build_environment_variables(pkg, env): # Add any pkgconfig directories to PKG_CONFIG_PATH pkg_config_dirs = [] for p in dep_prefixes: - for libdir in ('lib', 'lib64'): - pcdir = join_path(p, libdir, 'pkgconfig') + for maybe in ('lib', 'lib64', 'share'): + pcdir = join_path(p, maybe, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) env.set_path('PKG_CONFIG_PATH', pkg_config_dirs) -- cgit v1.2.3-70-g09d2 From 17f696d3b3ee83bcb185f583a465be3aef6c2d5f Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 29 Apr 2016 09:05:54 +0200 Subject: glib: add missing dependencies --- var/spack/repos/builtin/packages/glib/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/glib/package.py b/var/spack/repos/builtin/packages/glib/package.py index 67ead5f941..a3fc3f79eb 100644 --- a/var/spack/repos/builtin/packages/glib/package.py +++ b/var/spack/repos/builtin/packages/glib/package.py @@ -1,4 +1,5 @@ from spack import * +import sys class Glib(Package): """The GLib package contains a low-level libraries useful for @@ -12,6 +13,8 @@ class Glib(Package): depends_on("libffi") depends_on("zlib") + depends_on("pkg-config") + depends_on('gettext', sys.platform=='darwin') def install(self, spec, prefix): configure("--prefix=%s" % prefix) -- cgit v1.2.3-70-g09d2 From 176b9febb4ae0c653645cf5253b90019e9fc73f5 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 29 Apr 2016 09:06:15 +0200 Subject: libxcb: add missing dependencies --- var/spack/repos/builtin/packages/libxcb/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/libxcb/package.py b/var/spack/repos/builtin/packages/libxcb/package.py index d7d94c4546..b2543be5da 100644 --- a/var/spack/repos/builtin/packages/libxcb/package.py +++ b/var/spack/repos/builtin/packages/libxcb/package.py @@ -13,6 +13,7 @@ class Libxcb(Package): version('1.11.1', '118623c15a96b08622603a71d8789bf3') depends_on("python") depends_on("xcb-proto") + depends_on("pkg-config") # depends_on('pthread') # Ubuntu: apt-get install libpthread-stubs0-dev # depends_on('xau') # Ubuntu: apt-get install libxau-dev -- cgit v1.2.3-70-g09d2 From 3ad71700dd61d41841a581dc4f59f390dfd112b6 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 29 Apr 2016 09:52:49 +0200 Subject: the_silver_searcher: add missing dependency --- var/spack/repos/builtin/packages/the_silver_searcher/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/the_silver_searcher/package.py b/var/spack/repos/builtin/packages/the_silver_searcher/package.py index e4020b6766..30f06354bf 100644 --- a/var/spack/repos/builtin/packages/the_silver_searcher/package.py +++ b/var/spack/repos/builtin/packages/the_silver_searcher/package.py @@ -9,6 +9,7 @@ class TheSilverSearcher(Package): depends_on('pcre') depends_on('xz') + depends_on('pkg-config') def install(self, spec, prefix): configure("--prefix=%s" % prefix) -- cgit v1.2.3-70-g09d2 From a588a1fd84869a593903747372a7a2484b80fd52 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Tue, 19 Apr 2016 13:51:46 -0500 Subject: openmp needs to be an option for clang build --- var/spack/repos/builtin/packages/fftw/package.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/fftw/package.py b/var/spack/repos/builtin/packages/fftw/package.py index bc129aaf1a..5ca6547c9f 100644 --- a/var/spack/repos/builtin/packages/fftw/package.py +++ b/var/spack/repos/builtin/packages/fftw/package.py @@ -42,7 +42,7 @@ class Fftw(Package): variant('float', default=True, description='Produces a single precision version of the library') variant('long_double', default=True, description='Produces a long double precision version of the library') variant('quad', default=False, description='Produces a quad precision version of the library (works only with GCC and libquadmath)') - + variant('openmp', default=True, description="Enable OpenMP support.") variant('mpi', default=False, description='Activate MPI support') depends_on('mpi', when='+mpi') @@ -52,8 +52,9 @@ class Fftw(Package): def install(self, spec, prefix): options = ['--prefix=%s' % prefix, '--enable-shared', - '--enable-threads', - '--enable-openmp'] + '--enable-threads'] + if '+openmp' in spec: + options.append('--enable-openmp') if not self.compiler.f77 or not self.compiler.fc: options.append("--disable-fortran") if '+mpi' in spec: -- cgit v1.2.3-70-g09d2 From f8c14e1d98fd7d0d3df0bf8693b93081bf4b70fc Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Wed, 20 Apr 2016 09:41:19 -0500 Subject: Make openmp variant false by default. --- var/spack/repos/builtin/packages/fftw/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/fftw/package.py b/var/spack/repos/builtin/packages/fftw/package.py index 5ca6547c9f..e48e0dc46a 100644 --- a/var/spack/repos/builtin/packages/fftw/package.py +++ b/var/spack/repos/builtin/packages/fftw/package.py @@ -42,7 +42,7 @@ class Fftw(Package): variant('float', default=True, description='Produces a single precision version of the library') variant('long_double', default=True, description='Produces a long double precision version of the library') variant('quad', default=False, description='Produces a quad precision version of the library (works only with GCC and libquadmath)') - variant('openmp', default=True, description="Enable OpenMP support.") + variant('openmp', default=False, description="Enable OpenMP support.") variant('mpi', default=False, description='Activate MPI support') depends_on('mpi', when='+mpi') -- cgit v1.2.3-70-g09d2 From 83108f815ce09fa95e99e0ef747cf3cee50328b2 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 21 Apr 2016 09:59:37 -0500 Subject: Error out if +openmp used with OS X clang --- var/spack/repos/builtin/packages/fftw/package.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/fftw/package.py b/var/spack/repos/builtin/packages/fftw/package.py index e48e0dc46a..eb8f96cbac 100644 --- a/var/spack/repos/builtin/packages/fftw/package.py +++ b/var/spack/repos/builtin/packages/fftw/package.py @@ -53,7 +53,11 @@ class Fftw(Package): options = ['--prefix=%s' % prefix, '--enable-shared', '--enable-threads'] - if '+openmp' in spec: + # Add support for OpenMP + if '+openmp' in spec: + # Note: Apple's Clang does not support OpenMP. + if spec.satisfies('%clang'): + raise InstallError("Apple's clang does not support OpenMP") options.append('--enable-openmp') if not self.compiler.f77 or not self.compiler.fc: options.append("--disable-fortran") -- cgit v1.2.3-70-g09d2 From 4e2154e58fef151f22c6e77a923c7cc76b18517b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 18 Apr 2016 16:27:07 -0500 Subject: Add argcomplete python package --- var/spack/repos/builtin/packages/py-argcomplete/package.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-argcomplete/package.py diff --git a/var/spack/repos/builtin/packages/py-argcomplete/package.py b/var/spack/repos/builtin/packages/py-argcomplete/package.py new file mode 100644 index 0000000000..c94ef7238b --- /dev/null +++ b/var/spack/repos/builtin/packages/py-argcomplete/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyArgcomplete(Package): + """Bash tab completion for argparse.""" + + homepage = "https://pypi.python.org/pypi/argcomplete" + url = "https://pypi.python.org/packages/source/a/argcomplete/argcomplete-1.1.1.tar.gz" + + version('1.1.1', '89a3839096c9f991ad33828e72d21abf') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) -- cgit v1.2.3-70-g09d2 From 9c3d8dae574560f805f430b042b16821fabcf4ed Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 18 Apr 2016 14:54:38 -0500 Subject: Add latest jpeg version --- var/spack/repos/builtin/packages/jpeg/package.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/jpeg/package.py b/var/spack/repos/builtin/packages/jpeg/package.py index 87820467db..2f15e59ad4 100644 --- a/var/spack/repos/builtin/packages/jpeg/package.py +++ b/var/spack/repos/builtin/packages/jpeg/package.py @@ -1,14 +1,19 @@ from spack import * class Jpeg(Package): - """jpeg library""" + """libjpeg is a widely used free library with functions for handling the + JPEG image data format. It implements a JPEG codec (encoding and decoding) + alongside various utilities for handling JPEG data.""" + homepage = "http://www.ijg.org" - url = "http://www.ijg.org/files/jpegsrc.v9a.tar.gz" + url = "http://www.ijg.org/files/jpegsrc.v9b.tar.gz" + version('9b', '6a9996ce116ec5c52b4870dbcd6d3ddb') version('9a', '3353992aecaee1805ef4109aadd433e7') def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() + make("test") make("install") -- cgit v1.2.3-70-g09d2 From 7eb463a66e600e7a4d9692610763bfffbe9ac64a Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Wed, 4 May 2016 15:05:56 +0200 Subject: only fail when it is apples clang --- var/spack/repos/builtin/packages/fftw/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/fftw/package.py b/var/spack/repos/builtin/packages/fftw/package.py index eb8f96cbac..008939ece8 100644 --- a/var/spack/repos/builtin/packages/fftw/package.py +++ b/var/spack/repos/builtin/packages/fftw/package.py @@ -57,6 +57,8 @@ class Fftw(Package): if '+openmp' in spec: # Note: Apple's Clang does not support OpenMP. if spec.satisfies('%clang'): + ver = '%s' % self.compiler.version + if ver.endswith('-apple'): raise InstallError("Apple's clang does not support OpenMP") options.append('--enable-openmp') if not self.compiler.f77 or not self.compiler.fc: -- cgit v1.2.3-70-g09d2 From 67d64b804f5b08bc9e4493efd25ab549333e550e Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Wed, 4 May 2016 23:53:10 +0200 Subject: more succinctly --- var/spack/repos/builtin/packages/fftw/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/fftw/package.py b/var/spack/repos/builtin/packages/fftw/package.py index 008939ece8..4ffc787594 100644 --- a/var/spack/repos/builtin/packages/fftw/package.py +++ b/var/spack/repos/builtin/packages/fftw/package.py @@ -57,7 +57,7 @@ class Fftw(Package): if '+openmp' in spec: # Note: Apple's Clang does not support OpenMP. if spec.satisfies('%clang'): - ver = '%s' % self.compiler.version + ver = str(self.compiler.version) if ver.endswith('-apple'): raise InstallError("Apple's clang does not support OpenMP") options.append('--enable-openmp') -- cgit v1.2.3-70-g09d2 From 32f7b06a36e99e2fbfc56147b3c4803fa2946838 Mon Sep 17 00:00:00 2001 From: Dhanannjay 'Djay' Deo Date: Wed, 4 May 2016 18:14:38 -0400 Subject: Add version 2.10.2 --- var/spack/repos/builtin/packages/visit/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/visit/package.py b/var/spack/repos/builtin/packages/visit/package.py index 9b21370fa3..716cd2c101 100644 --- a/var/spack/repos/builtin/packages/visit/package.py +++ b/var/spack/repos/builtin/packages/visit/package.py @@ -7,6 +7,7 @@ class Visit(Package): url = "http://portal.nersc.gov/project/visit/releases/2.10.1/visit2.10.1.tar.gz" version('2.10.1', '3cbca162fdb0249f17c4456605c4211e') + version('2.10.2', '253de0837a9d69fb689befc98ea4d068') depends_on("vtk@6.1.0~opengl2") depends_on("qt@4.8.6") -- cgit v1.2.3-70-g09d2 From 1785de0f3152a7d853425f07c236faf60eb91e0e Mon Sep 17 00:00:00 2001 From: Dhanannjay 'Djay' Deo Date: Wed, 4 May 2016 22:08:01 -0400 Subject: remove hdf5 which is silo actually a silo dependency --- var/spack/repos/builtin/packages/visit/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/visit/package.py b/var/spack/repos/builtin/packages/visit/package.py index 716cd2c101..91ffd4c045 100644 --- a/var/spack/repos/builtin/packages/visit/package.py +++ b/var/spack/repos/builtin/packages/visit/package.py @@ -12,7 +12,6 @@ class Visit(Package): depends_on("vtk@6.1.0~opengl2") depends_on("qt@4.8.6") depends_on("python") - depends_on("hdf5") # silo seems to need it depends_on("silo+shared") def install(self, spec, prefix): -- cgit v1.2.3-70-g09d2 From 45e77e7739e3e96b776eb0691daa5cfaaa9e26ce Mon Sep 17 00:00:00 2001 From: Dhanannjay 'Djay' Deo Date: Wed, 4 May 2016 22:49:47 -0400 Subject: Correctly extend std_cmake_args --- var/spack/repos/builtin/packages/visit/package.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/visit/package.py b/var/spack/repos/builtin/packages/visit/package.py index 91ffd4c045..14e3b6a0c1 100644 --- a/var/spack/repos/builtin/packages/visit/package.py +++ b/var/spack/repos/builtin/packages/visit/package.py @@ -18,13 +18,12 @@ class Visit(Package): with working_dir('spack-build', create=True): feature_args = std_cmake_args[:] - feature_args = ["-DVTK_MAJOR_VERSION=6", - "-DVTK_MINOR_VERSION=1", - "-DCMAKE_INSTALL_PREFIX:PATH=%s" % spec.prefix, - "-DVISIT_LOC_QMAKE_EXE:FILEPATH=%s/qmake-qt4" % spec['qt'].prefix.bin, - "-DPYTHON_EXECUTABLE:FILEPATH=%s/python" % spec['python'].prefix.bin, - "-DVISIT_SILO_DIR:PATH=%s" % spec['silo'].prefix, - "-DVISIT_HDF5_DIR:PATH=%s" % spec['hdf5'].prefix] + feature_args.extend(["-DVTK_MAJOR_VERSION=6", + "-DVTK_MINOR_VERSION=1", + "-DVISIT_LOC_QMAKE_EXE:FILEPATH=%s/qmake-qt4" % spec['qt'].prefix.bin, + "-DPYTHON_EXECUTABLE:FILEPATH=%s/python" % spec['python'].prefix.bin, + "-DVISIT_SILO_DIR:PATH=%s" % spec['silo'].prefix, + "-DVISIT_HDF5_DIR:PATH=%s" % spec['hdf5'].prefix]) cmake('../src', *feature_args) -- cgit v1.2.3-70-g09d2 From ef202fbe0ca1f23ca7f0ea271d45cee8a0623e4e Mon Sep 17 00:00:00 2001 From: Dhanannjay 'Djay' Deo Date: Wed, 4 May 2016 22:50:15 -0400 Subject: Build static and shared libraries for silo --- var/spack/repos/builtin/packages/silo/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/silo/package.py b/var/spack/repos/builtin/packages/silo/package.py index 638a894b7b..7e68663c6e 100644 --- a/var/spack/repos/builtin/packages/silo/package.py +++ b/var/spack/repos/builtin/packages/silo/package.py @@ -24,7 +24,6 @@ class Silo(Package): '--enable-fortran' if '+fortran' in spec else '--disable-fortran', '--enable-silex' if '+silex' in spec else '--disable-silex', '--enable-shared' if '+shared' in spec else '--disable-shared', - '--disable-static' if '+shared' in spec else '--enable-static', ] if '+silex' in spec: -- cgit v1.2.3-70-g09d2 From 2cdfe14e5a8cbdefd3533d3bb0b0ac09fa9e4fa6 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 3 May 2016 10:47:39 +0200 Subject: compilers: make sure cxx11_flag() is defined for all compilers --- lib/spack/spack/compilers/clang.py | 11 +++++++++++ lib/spack/spack/compilers/nag.py | 6 ++++++ lib/spack/spack/compilers/pgi.py | 7 ++++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index e406d86a24..8a60ebcaed 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -47,6 +47,17 @@ class Clang(Compiler): 'f77' : 'f77', 'fc' : 'f90' } + @property + def cxx11_flag(self): + if ver.endswith('-apple'): + # FIXME: figure out from which version Apple's clang supports c++11 + return "-std=c++11" + else: + if self.version < ver('3.3'): + tty.die("Only Clang 3.3 and above support c++11.") + else: + return "-std=c++11" + @classmethod def default_version(self, comp): """The '--version' option works for clang compilers. diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index 527a05a090..1df6b1c591 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -20,6 +20,12 @@ class Nag(Compiler): 'f77' : 'nag/nagfor', 'fc' : 'nag/nagfor' } + @property + def cxx11_flag(self): + tty.die("cxx11_flag() is not implemented for nag. Consider creating a pull-request.") + return "-std=c++11" + + @classmethod def default_version(self, comp): """The '-V' option works for nag compilers. diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index c6a1078bd9..ebf644404b 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -43,6 +43,12 @@ class Pgi(Compiler): 'f77' : 'pgi/pgfortran', 'fc' : 'pgi/pgfortran' } + @property + def cxx11_flag(self): + tty.die("cxx11_flag() is not implemented for pgi. Consider creating a pull-request.") + return "-std=c++11" + + @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. @@ -54,4 +60,3 @@ class Pgi(Compiler): """ return get_compiler_version( comp, '-V', r'pg[^ ]* ([^ ]+) \d\d\d?-bit target') - -- cgit v1.2.3-70-g09d2 From 9776dc0433fee264ae1fd2cbff9b16fd499fdb1a Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 3 May 2016 10:49:04 +0200 Subject: compilers: add openmp_flag() method --- lib/spack/spack/compilers/clang.py | 8 ++++++++ lib/spack/spack/compilers/gcc.py | 4 ++++ lib/spack/spack/compilers/intel.py | 6 ++++-- lib/spack/spack/compilers/nag.py | 4 ++++ lib/spack/spack/compilers/pgi.py | 4 ++++ lib/spack/spack/compilers/xl.py | 4 ++++ 6 files changed, 28 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 8a60ebcaed..44de77af4f 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -47,6 +47,14 @@ class Clang(Compiler): 'f77' : 'f77', 'fc' : 'f90' } + @property + def openmp_flag(self): + ver = '%s' % self.version + if ver.endswith('-apple'): + tty.die("Clang from Apple does not support Openmp yet.") + else: + return "-fopenmp" + @property def cxx11_flag(self): if ver.endswith('-apple'): diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index 2e57e44856..91c498ac82 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -49,6 +49,10 @@ class Gcc(Compiler): 'f77' : 'gcc/gfortran', 'fc' : 'gcc/gfortran' } + @property + def openmp_flag(self): + return "-fopenmp" + @property def cxx11_flag(self): if self.version < ver('4.3'): diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 69e9764790..f04a6aa899 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -43,6 +43,10 @@ class Intel(Compiler): 'f77' : 'intel/ifort', 'fc' : 'intel/ifort' } + @property + def openmp_flag(self): + return "-openmp" + @property def cxx11_flag(self): if self.version < ver('11.1'): @@ -68,5 +72,3 @@ class Intel(Compiler): """ return get_compiler_version( comp, '--version', r'\((?:IFORT|ICC)\) ([^ ]+)') - - diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index 1df6b1c591..61486f22bd 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -20,6 +20,10 @@ class Nag(Compiler): 'f77' : 'nag/nagfor', 'fc' : 'nag/nagfor' } + @property + def openmp_flag(self): + return "-openmp" + @property def cxx11_flag(self): tty.die("cxx11_flag() is not implemented for nag. Consider creating a pull-request.") diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index ebf644404b..299b9a7016 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -43,6 +43,10 @@ class Pgi(Compiler): 'f77' : 'pgi/pgfortran', 'fc' : 'pgi/pgfortran' } + @property + def openmp_flag(self): + return "-mp" + @property def cxx11_flag(self): tty.die("cxx11_flag() is not implemented for pgi. Consider creating a pull-request.") diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index c1d55109a3..657309fe06 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -44,6 +44,10 @@ class Xl(Compiler): 'f77' : 'xl/xlf', 'fc' : 'xl/xlf90' } + @property + def openmp_flag(self): + return "-qsmp=omp" + @property def cxx11_flag(self): if self.version < ver('13.1'): -- cgit v1.2.3-70-g09d2 From c078deaab1d62d881bbed8efece779ca01c504c7 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 3 May 2016 11:08:44 +0200 Subject: compilers: add missing import statements --- lib/spack/spack/compilers/clang.py | 2 ++ lib/spack/spack/compilers/intel.py | 2 ++ lib/spack/spack/compilers/nag.py | 1 + lib/spack/spack/compilers/pgi.py | 1 + lib/spack/spack/compilers/xl.py | 1 + 5 files changed, 7 insertions(+) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 44de77af4f..799b92b20d 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -26,6 +26,8 @@ import re import spack.compiler as cpr from spack.compiler import * from spack.util.executable import * +import llnl.util.tty as tty +from spack.version import ver class Clang(Compiler): # Subclasses use possible names of C compiler diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index f04a6aa899..bc13db5dc7 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -23,6 +23,8 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack.compiler import * +import llnl.util.tty as tty +from spack.version import ver class Intel(Compiler): # Subclasses use possible names of C compiler diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index 61486f22bd..729aed0caf 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -1,4 +1,5 @@ from spack.compiler import * +import llnl.util.tty as tty class Nag(Compiler): # Subclasses use possible names of C compiler diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 299b9a7016..5ab4a9d109 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack.compiler import * +import llnl.util.tty as tty class Pgi(Compiler): # Subclasses use possible names of C compiler diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index 657309fe06..fd78abd091 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -24,6 +24,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack.compiler import * +import llnl.util.tty as tty class Xl(Compiler): # Subclasses use possible names of C compiler -- cgit v1.2.3-70-g09d2 From 3cd3052c564451f375977522e23a458edd25611a Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 3 May 2016 11:26:03 +0200 Subject: compilers: minor fixes to Clang::cxx11_flag() and Clang::openmp_flag() --- lib/spack/spack/compilers/clang.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 799b92b20d..1f0eda3220 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -51,15 +51,16 @@ class Clang(Compiler): @property def openmp_flag(self): - ver = '%s' % self.version - if ver.endswith('-apple'): + ver_string = '%s' % self.version + if ver_string.endswith('-apple'): tty.die("Clang from Apple does not support Openmp yet.") else: return "-fopenmp" @property def cxx11_flag(self): - if ver.endswith('-apple'): + ver_string = '%s' % self.version + if ver_string.endswith('-apple'): # FIXME: figure out from which version Apple's clang supports c++11 return "-std=c++11" else: -- cgit v1.2.3-70-g09d2 From 592045cd5453f3214737de04ce489126287f52dc Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 3 May 2016 11:28:49 +0200 Subject: compilers: make Intel::openmp_flag() return -openmp and -qopenmp based on the compiler version --- lib/spack/spack/compilers/intel.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index bc13db5dc7..9b1cf07c36 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -47,7 +47,10 @@ class Intel(Compiler): @property def openmp_flag(self): - return "-openmp" + if self.version < ver('16.0'): + return "-openmp" + else: + return "-qopenmp" @property def cxx11_flag(self): -- cgit v1.2.3-70-g09d2 From 07fd0ccc9aeb1fb47ce6bbb1353a5c5fe7cf7e9a Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 3 May 2016 12:13:31 +0200 Subject: compiler: add Clang.is_apple property which checks if Clang is from Apple or not using version --- lib/spack/spack/compilers/clang.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 1f0eda3220..a6c9a69505 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -50,17 +50,23 @@ class Clang(Compiler): 'fc' : 'f90' } @property - def openmp_flag(self): + def is_apple(self): ver_string = '%s' % self.version if ver_string.endswith('-apple'): + return True + else: + return False + + @property + def openmp_flag(self): + if self.is_apple: tty.die("Clang from Apple does not support Openmp yet.") else: return "-fopenmp" @property def cxx11_flag(self): - ver_string = '%s' % self.version - if ver_string.endswith('-apple'): + if self.is_apple: # FIXME: figure out from which version Apple's clang supports c++11 return "-std=c++11" else: -- cgit v1.2.3-70-g09d2 From f2f1c49c90de5c1b620d7bdb7cd5ff85173fcb22 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 3 May 2016 15:21:33 +0200 Subject: compilers: one more missing import statement --- lib/spack/spack/compilers/xl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index fd78abd091..61a2e730dc 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -25,6 +25,7 @@ ############################################################################## from spack.compiler import * import llnl.util.tty as tty +from spack.version import ver class Xl(Compiler): # Subclasses use possible names of C compiler -- cgit v1.2.3-70-g09d2 From 6a418cfb8d60a26d1195aeb74b2d54ae9cb38616 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 3 May 2016 17:09:49 +0200 Subject: compiler: simplify Clang.is_apple --- lib/spack/spack/compilers/clang.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index a6c9a69505..8c646905c7 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -51,11 +51,8 @@ class Clang(Compiler): @property def is_apple(self): - ver_string = '%s' % self.version - if ver_string.endswith('-apple'): - return True - else: - return False + ver_string = str(self.version) + return ver_string.endswith('-apple') @property def openmp_flag(self): -- cgit v1.2.3-70-g09d2 From d5a760776a47552aab7b8575e7ad9ac9eaba9384 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 3 May 2016 17:26:43 +0200 Subject: compiler: add default implementation of openmp_flag() and css11_flag() --- lib/spack/spack/compiler.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 20896f9eec..a707b2e3aa 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -120,6 +120,20 @@ class Compiler(object): def version(self): return self.spec.version + # default implementation of OpenMP linking flag. + # Override in derived classes if needed + @property + def openmp_flag(self): + return "-fopenmp" + + + # default implementation of c++11 linking flag. + # raise an error to force derived classes implement it when used + @property + def cxx11_flag(self): + return "-std=c++11" + + # # Compiler classes have methods for querying the version of # specific compiler executables. This is used when discovering compilers. -- cgit v1.2.3-70-g09d2 From e28ca3922feaba84f5bc2e1b8bf6ababe964ace3 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 3 May 2016 17:27:46 +0200 Subject: compiler: cleanup Nag.cxx11_flag and Pgi.cxx11_flag --- lib/spack/spack/compilers/nag.py | 5 ----- lib/spack/spack/compilers/pgi.py | 1 - 2 files changed, 6 deletions(-) diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index 729aed0caf..bbc291d7b6 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -25,11 +25,6 @@ class Nag(Compiler): def openmp_flag(self): return "-openmp" - @property - def cxx11_flag(self): - tty.die("cxx11_flag() is not implemented for nag. Consider creating a pull-request.") - return "-std=c++11" - @classmethod def default_version(self, comp): diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 5ab4a9d109..94c6b8365c 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -50,7 +50,6 @@ class Pgi(Compiler): @property def cxx11_flag(self): - tty.die("cxx11_flag() is not implemented for pgi. Consider creating a pull-request.") return "-std=c++11" -- cgit v1.2.3-70-g09d2 From 30b65d3114e20506fb2d36c0aa7f34babf3c4f72 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 3 May 2016 17:29:37 +0200 Subject: fix comment in Compiler class --- lib/spack/spack/compiler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index a707b2e3aa..1f1cf97ce9 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -120,15 +120,15 @@ class Compiler(object): def version(self): return self.spec.version - # default implementation of OpenMP linking flag. + # Default implementation of OpenMP linking flag. # Override in derived classes if needed @property def openmp_flag(self): return "-fopenmp" - # default implementation of c++11 linking flag. - # raise an error to force derived classes implement it when used + # Default implementation of c++11 linking flag. + # Override in derived classes if needed @property def cxx11_flag(self): return "-std=c++11" -- cgit v1.2.3-70-g09d2 From 473a5542bed01e2f70370ddd5ceb70ac41fe178e Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 3 May 2016 18:04:08 +0200 Subject: compiler: make default openmp_flag() and cxx11_flag() die when these properties are not implemented in a derived class --- lib/spack/spack/compiler.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 1f1cf97ce9..622eed6c10 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -120,19 +120,20 @@ class Compiler(object): def version(self): return self.spec.version - # Default implementation of OpenMP linking flag. - # Override in derived classes if needed + # This property should be overridden in the compiler subclass if + # OpenMP is supported by that compiler @property def openmp_flag(self): - return "-fopenmp" + # If it is not overridden, assume it is not supported and warn the user + tty.die("The compiler you have chosen does not currently support OpenMP. If you think it should, please edit the compiler subclass and submit a pull request or issue.") - # Default implementation of c++11 linking flag. - # Override in derived classes if needed + # This property should be overridden in the compiler subclass if + # C++11 is supported by that compiler @property def cxx11_flag(self): - return "-std=c++11" - + # If it is not overridden, assume it is not supported and warn the user + tty.die("The compiler you have chosen does not currently support C++11. If you think it should, please edit the compiler subclass and submit a pull request or issue.") # # Compiler classes have methods for querying the version of -- cgit v1.2.3-70-g09d2 From ddcb97f9531c65bfc370177dfd2090c9e82a4cb3 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 3 May 2016 19:04:42 +0200 Subject: add a temporary Nag.cxx11_flag property --- lib/spack/spack/compilers/nag.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index bbc291d7b6..e9038c1039 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -25,6 +25,11 @@ class Nag(Compiler): def openmp_flag(self): return "-openmp" + @property + def cxx11_flag(self): + # NAG does not have a C++ compiler + # However, it can be mixed with a compiler that does support it + return "-std=c++11" @classmethod def default_version(self, comp): -- cgit v1.2.3-70-g09d2 From c6fb6bde40798903dcdd5d503c32368068a0f8e4 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 3 May 2016 19:07:16 +0200 Subject: remove cxx11_flag from Compiler as it is now substituted by a property with the same name --- lib/spack/spack/compiler.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 622eed6c10..42529777bc 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -94,9 +94,6 @@ class Compiler(object): # Names of generic arguments used by this compiler arg_rpath = '-Wl,-rpath,%s' - # argument used to get C++11 options - cxx11_flag = "-std=c++11" - # argument used to get C++14 options cxx14_flag = "-std=c++1y" -- cgit v1.2.3-70-g09d2 From f84f04591be44ba6c1aa5bef50d5efa872cfb1c9 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 5 May 2016 10:48:31 +0200 Subject: substitute cxx14_flag by a property with the same name to be overridden in derived classes --- lib/spack/spack/compiler.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 42529777bc..a28d7302aa 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -94,9 +94,6 @@ class Compiler(object): # Names of generic arguments used by this compiler arg_rpath = '-Wl,-rpath,%s' - # argument used to get C++14 options - cxx14_flag = "-std=c++1y" - def __init__(self, cspec, cc, cxx, f77, fc): def check(exe): @@ -132,6 +129,14 @@ class Compiler(object): # If it is not overridden, assume it is not supported and warn the user tty.die("The compiler you have chosen does not currently support C++11. If you think it should, please edit the compiler subclass and submit a pull request or issue.") + # This property should be overridden in the compiler subclass if + # C++14 is supported by that compiler + @property + def cxx14_flag(self): + # If it is not overridden, assume it is not supported and warn the user + tty.die("The compiler you have chosen does not currently support C++14. If you think it should, please edit the compiler subclass and submit a pull request or issue.") + + # # Compiler classes have methods for querying the version of # specific compiler executables. This is used when discovering compilers. -- cgit v1.2.3-70-g09d2 From c37ea9aff548bfdf106aa141b8d6e6adec2ffd01 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 5 May 2016 11:56:58 +0200 Subject: document usage of compiler flags properties --- lib/spack/docs/packaging_guide.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 34d11308f5..31c676d4f5 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1831,6 +1831,25 @@ successfully find ``libdwarf.h`` and ``libdwarf.so``, without the packager having to provide ``--with-libdwarf=/path/to/libdwarf`` on the command line. +Compiler flags +~~~~~~~~~~~~~~ +In rare circumstances such as compiling and running small unit tests, a package +developer may need to know what are the appropriate compiler flags to enable +features like ``OpenMP``, ``c++11``, ``c++14`` and alike. To that end the +compiler classes in ``spack`` implement the following _properties_ : +``openmp_flag``, ``cxx11_flag``, ``cxx14_flag``, which can be accessed in a +package by ``self.compiler.cxx11_flag`` and alike. Note that the implementation +is such that if a given compiler version does not support this feature, an +error will be produced. Therefore package developers can also use these properties +to assert that a compiler supports the requested feature. This is handy when a +package supports additional variants like + +.. code-block:: python + + variant('openmp', default=True, description="Enable OpenMP support.") + + + Message Parsing Interface (MPI) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ It is common for high performance computing software/packages to use ``MPI``. -- cgit v1.2.3-70-g09d2 From 1203a14563ff8589a65def3bd99dc6a9cc399e92 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 5 May 2016 14:01:21 -0500 Subject: Remove tutorial comments --- var/spack/repos/builtin/packages/xerces-c/package.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/var/spack/repos/builtin/packages/xerces-c/package.py b/var/spack/repos/builtin/packages/xerces-c/package.py index e36fb936e0..bd02ddcd4b 100644 --- a/var/spack/repos/builtin/packages/xerces-c/package.py +++ b/var/spack/repos/builtin/packages/xerces-c/package.py @@ -1,19 +1,3 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install xerces-c -# -# You can always get back here to change things with: -# -# spack edit xerces-c -# -# See the spack documentation for more information on building -# packages. -# from spack import * class XercesC(Package): -- cgit v1.2.3-70-g09d2 From db80c5e97e7fc95b08e4a38cb88ec598c2939e2c Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 5 May 2016 21:11:54 -0400 Subject: Disable -Werror This leads to problems if new compiler versions report new kinds of warnings. --- var/spack/repos/builtin/packages/binutils/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/binutils/package.py b/var/spack/repos/builtin/packages/binutils/package.py index b8064093d2..158d722046 100644 --- a/var/spack/repos/builtin/packages/binutils/package.py +++ b/var/spack/repos/builtin/packages/binutils/package.py @@ -29,6 +29,7 @@ class Binutils(Package): configure_args = [ '--prefix=%s' % prefix, '--disable-dependency-tracking', + '--disable-werror', '--enable-interwork', '--enable-multilib', '--enable-shared', -- cgit v1.2.3-70-g09d2 From 9f212e72014e6397215bf7695695c59c710b516d Mon Sep 17 00:00:00 2001 From: "Tanzima Z. Islam" Date: Fri, 6 May 2016 13:50:34 -0700 Subject: Adding a new package file for Kripke --- var/spack/repos/builtin/packages/kripke/package.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 var/spack/repos/builtin/packages/kripke/package.py diff --git a/var/spack/repos/builtin/packages/kripke/package.py b/var/spack/repos/builtin/packages/kripke/package.py new file mode 100644 index 0000000000..68ccc4cb6c --- /dev/null +++ b/var/spack/repos/builtin/packages/kripke/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Kripke(Package): + """Kripke is a simple, scalable, 3D Sn deterministic particle transport code.""" + + homepage = "https://codesign.llnl.gov/kripke.php" + url = "" + #version('master', git='https://lc.llnl.gov/stash/scm/kripke/kripke.git') + version('master', git='https://lc.llnl.gov/stash/scm/~islam3/kripke.git') + + def install(self, spec, prefix): + with working_dir('build', create=True): + cmake('-DCMAKE_INSTALL_PREFIX:PATH=.', '-DCMAKE_TOOLCHAIN_FILE=../cmake/Toolchain/chaos_5_x86_64_ib-ic15.cmake', '-DENABLE_OPENMP=1', '..', *std_cmake_args) + make() + make("install") + -- cgit v1.2.3-70-g09d2 From 7bf724b1d3c92073543e8e3e72cfbf788531d401 Mon Sep 17 00:00:00 2001 From: "Cecilia W. Castillo" Date: Fri, 6 May 2016 14:14:13 -0700 Subject: add support for crypto version 5.6.1 --- var/spack/repos/builtin/packages/cryptopp/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/cryptopp/package.py b/var/spack/repos/builtin/packages/cryptopp/package.py index bc83cb2b65..c2778e14da 100644 --- a/var/spack/repos/builtin/packages/cryptopp/package.py +++ b/var/spack/repos/builtin/packages/cryptopp/package.py @@ -13,6 +13,7 @@ class Cryptopp(Package): version('5.6.3', '3c5b70e2ec98b7a24988734446242d07') version('5.6.2', '7ed022585698df48e65ce9218f6c6a67') + version('5.6.1', '96cbeba0907562b077e26bcffb483828') def install(self, spec, prefix): make() -- cgit v1.2.3-70-g09d2 From c82db2116b6a1945f21d636d2981427a3bb957fe Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Fri, 6 May 2016 17:13:09 -0400 Subject: nco: Added new package --- var/spack/repos/builtin/packages/antlr/package.py | 47 +++++++++++++++++++++++ var/spack/repos/builtin/packages/nco/package.py | 30 +++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 var/spack/repos/builtin/packages/antlr/package.py create mode 100644 var/spack/repos/builtin/packages/nco/package.py diff --git a/var/spack/repos/builtin/packages/antlr/package.py b/var/spack/repos/builtin/packages/antlr/package.py new file mode 100644 index 0000000000..c7c7e3e850 --- /dev/null +++ b/var/spack/repos/builtin/packages/antlr/package.py @@ -0,0 +1,47 @@ +from spack import * + +class Antlr(Package): + + homepage = "http://www.antlr.org" + url = "https://github.com/antlr/antlr/tarball/v2.7.7" + + # NOTE: This requires that a system Java be available. + # Spack does not yet know how to install Java compilers + + # Notes from http://nco.sourceforge.net/#bld + # The first steps to build (i.e., compile, for the most part) NCO from + # source code are to install the pre-requisites: ANTLR version 2.7.7 + # (like this one not version 3.x or 4.x!) (required for ncap2)... ANTLR + # binaries from major distributions are pre-built with the source patch + # necessary to allow NCO to link to ANTLR... The ANTLR source file + # CharScanner.hpp must include this line: #include or else + # ncap2 will not compile (this tarball is already patched). + version('2.7.7', '914865e853fe8e1e61a9f23d045cb4ab', + # Patched version as described above + url='http://dust.ess.uci.edu/tmp/antlr-2.7.7.tar.gz') + # Unpatched version + # url='http://dust.ess.uci.edu/nco/antlr-2.7.7.tar.gz') + + variant('cxx', default=False, description='Enable ANTLR for C++') + variant('java', default=False, description='Enable ANTLR for Java') + variant('python', default=False, description='Enable ANTLR for Python') + variant('csharp', default=False, description='Enable ANTLR for Csharp') + + + def install(self, spec, prefix): + # Check for future enabling of variants + for v in ('+java', '+python', '+csharp'): + if v in spec: + raise Error('Illegal variant %s; for now, Spack only knows how to build antlr or antlr+cxx') + + config_args = [ + '--prefix=%s' % prefix, + '--%s-cxx' % ('enable' if '+cxx' in spec else 'disable'), + '--%s-java' % ('enable' if '+java' in spec else 'disable'), + '--%s-python' % ('enable' if '+python' in spec else 'disable'), + '--%s-csharp' % ('enable' if '+csharp' in spec else 'disable')] + + # which('autoreconf')('-iv') + configure(*config_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/nco/package.py b/var/spack/repos/builtin/packages/nco/package.py new file mode 100644 index 0000000000..3a9aeaa656 --- /dev/null +++ b/var/spack/repos/builtin/packages/nco/package.py @@ -0,0 +1,30 @@ +from spack import * +import os + +class Nco(Package): + """The NCO toolkit manipulates and analyzes data stored in + netCDF-accessible formats""" + + homepage = "https://sourceforge.net/projects/nco" + url = "https://github.com/nco/nco/archive/4.5.5.tar.gz" + + version('4.5.5', '9f1f1cb149ad6407c5a03c20122223ce') + + # See "Compilation Requirements" at: + # http://nco.sourceforge.net/#bld + + depends_on('netcdf') + depends_on('antlr@2.7.7+cxx') # (required for ncap2) + depends_on('gsl') # (desirable for ncap2) + depends_on('udunits2') # (allows dimensional unit transformations) + # depends_on('opendap') # (enables network transparency), + + def install(self, spec, prefix): + opts = [ + '--prefix=%s' % prefix, + '--disable-openmp', # TODO: Make this a variant + '--disable-dap', # TODO: Make this a variant + '--disable-esmf'] + configure(*opts) + make() + make("install") -- cgit v1.2.3-70-g09d2 From 86449790fe66f07239cd552f1ce579bf341bd1f4 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Fri, 6 May 2016 15:15:40 -0700 Subject: add pmi support and process managers to the MPIs --- var/spack/repos/builtin/packages/mpich/package.py | 4 ++++ var/spack/repos/builtin/packages/openmpi/package.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 5d68f20351..2179086fe5 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -43,6 +43,8 @@ class Mpich(Package): version('3.0.4', '9c5d5d4fe1e17dd12153f40bc5b6dbc0') variant('verbs', default=False, description='Build support for OpenFabrics verbs.') + variant('pmi', default=True, description='Build with PMI support') + variant('hydra', default=True, description='Build the hydra process manager') provides('mpi@:3.0', when='@3:') provides('mpi@:1.3', when='@1:') @@ -62,6 +64,8 @@ class Mpich(Package): def install(self, spec, prefix): config_args = ["--prefix=" + prefix, + "--with-pmi=" + ("yes" if '+pmi' in spec else 'no'), + "--with-pm=" + ('hydra' if '+hydra' in spec else 'no'), "--enable-shared"] # Variants diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 776fb6eeaa..d0dd2d657f 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -26,6 +26,7 @@ class Openmpi(Package): patch('configure.patch', when="@1.10.0:1.10.1") variant('psm', default=False, description='Build support for the PSM library.') + variant('pmi', default=True, description='Build support for PMI-based launchers') variant('verbs', default=False, description='Build support for OpenFabrics verbs.') # TODO : variant support for other schedulers is missing @@ -67,6 +68,9 @@ class Openmpi(Package): if '+psm' in spec: config_args.append("--with-psm") + if '+pmi' in spec: + config_args.append("--with-pmi") #TODO: let user specify directory when possible + if '+verbs' in spec: # Up through version 1.6, this option was previously named --with-openib if spec.satisfies('@:1.6'): -- cgit v1.2.3-70-g09d2 From 22afc6dadd81db12262571bddad6256c2900db64 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Fri, 6 May 2016 15:33:26 -0700 Subject: pile of dependencies for neovim, including luajit and libuv --- var/spack/repos/builtin/packages/LuaJIT/package.py | 15 +++++++++++++++ .../repos/builtin/packages/libtermkey/package.py | 17 +++++++++++++++++ var/spack/repos/builtin/packages/libuv/package.py | 21 +++++++++++++++++++++ .../repos/builtin/packages/libvterm/package.py | 12 ++++++++++++ .../repos/builtin/packages/msgpack-c/package.py | 14 ++++++++++++++ .../repos/builtin/packages/unibilium/package.py | 12 ++++++++++++ 6 files changed, 91 insertions(+) create mode 100644 var/spack/repos/builtin/packages/LuaJIT/package.py create mode 100644 var/spack/repos/builtin/packages/libtermkey/package.py create mode 100644 var/spack/repos/builtin/packages/libuv/package.py create mode 100644 var/spack/repos/builtin/packages/libvterm/package.py create mode 100644 var/spack/repos/builtin/packages/msgpack-c/package.py create mode 100644 var/spack/repos/builtin/packages/unibilium/package.py diff --git a/var/spack/repos/builtin/packages/LuaJIT/package.py b/var/spack/repos/builtin/packages/LuaJIT/package.py new file mode 100644 index 0000000000..7b2a269212 --- /dev/null +++ b/var/spack/repos/builtin/packages/LuaJIT/package.py @@ -0,0 +1,15 @@ +import os +from spack import * + +class Luajit(Package): + """Flast flexible JITed lua""" + homepage = "http://www.luajit.org" + url = "http://luajit.org/download/LuaJIT-2.0.4.tar.gz" + + version('2.0.4', 'dd9c38307f2223a504cbfb96e477eca0') + + def install(self, spec, prefix): + # Linking with the C++ compiler is a dirty hack to deal with the fact + # that unwinding symbols are not included by libc, this is necessary + # on some platforms for the final link stage to work + make("install", "PREFIX=" + prefix, "TARGET_LD=" + os.environ['CXX']) diff --git a/var/spack/repos/builtin/packages/libtermkey/package.py b/var/spack/repos/builtin/packages/libtermkey/package.py new file mode 100644 index 0000000000..7f25edaf76 --- /dev/null +++ b/var/spack/repos/builtin/packages/libtermkey/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Libtermkey(Package): + """Easy keyboard entry processing for terminal programs""" + homepage = "http://www.leonerd.org.uk/code/libtermkey/" + url = "http://www.leonerd.org.uk/code/libtermkey/libtermkey-0.18.tar.gz" + + version('0.18' , '3be2e3e5a851a49cc5e8567ac108b520') + version('0.17' , '20edb99e0d95ec1690fe90e6a555ae6d') + version('0.16' , '7a24b675aaeb142d30db28e7554987d4') + version('0.15b', '27689756e6c86c56ae454f2ac259bc3d') + version('0.14' , 'e08ce30f440f9715c459060e0e048978') + + + def install(self, spec, prefix): + make() + make("install", "PREFIX=" + prefix) diff --git a/var/spack/repos/builtin/packages/libuv/package.py b/var/spack/repos/builtin/packages/libuv/package.py new file mode 100644 index 0000000000..eace94d1a6 --- /dev/null +++ b/var/spack/repos/builtin/packages/libuv/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Libuv(Package): + """Multi-platform library with a focus on asynchronous IO""" + homepage = "http://libuv.org" + url = "https://github.com/libuv/libuv/archive/v1.9.0.tar.gz" + + version('1.9.0', '14737f9c76123a19a290dabb7d1cd04c') + + depends_on('automake') + depends_on('autoconf') + depends_on('libtool') + + def install(self, spec, prefix): + bash = which("bash") + bash('autogen.sh') + configure('--prefix=%s' % prefix) + + make() + make("check") + make("install") diff --git a/var/spack/repos/builtin/packages/libvterm/package.py b/var/spack/repos/builtin/packages/libvterm/package.py new file mode 100644 index 0000000000..3212f6550d --- /dev/null +++ b/var/spack/repos/builtin/packages/libvterm/package.py @@ -0,0 +1,12 @@ +from spack import * + +class Libvterm(Package): + """An abstract library implementation of a terminal emulator""" + homepage = "http://www.leonerd.org.uk/code/libvterm/" + url = "http://www.leonerd.org.uk/code/libvterm/libvterm-0+bzr681.tar.gz" + + version('681', '7a4325a7350b7092245c04e8ee185ac3') + + def install(self, spec, prefix): + make() + make("install", "PREFIX=" + prefix) diff --git a/var/spack/repos/builtin/packages/msgpack-c/package.py b/var/spack/repos/builtin/packages/msgpack-c/package.py new file mode 100644 index 0000000000..a363bc89be --- /dev/null +++ b/var/spack/repos/builtin/packages/msgpack-c/package.py @@ -0,0 +1,14 @@ +from spack import * + +class MsgpackC(Package): + """A small, fast binary interchange format convertible to/from JSON""" + homepage = "http://www.msgpack.org" + url = "https://github.com/msgpack/msgpack-c/archive/cpp-1.4.1.tar.gz" + + version('1.4.1', 'e2fd3a7419b9bc49e5017fdbefab87e0') + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/unibilium/package.py b/var/spack/repos/builtin/packages/unibilium/package.py new file mode 100644 index 0000000000..ef5de56f79 --- /dev/null +++ b/var/spack/repos/builtin/packages/unibilium/package.py @@ -0,0 +1,12 @@ +from spack import * + +class Unibilium(Package): + """A terminfo parsing library""" + homepage = "https://github.com/mauke/unibilium" + url = "https://github.com/mauke/unibilium/archive/v1.2.0.tar.gz" + + version('1.2.0', '9b1c97839a880a373da6c097443b43c4') + + def install(self, spec, prefix): + make("PREFIX="+prefix) + make("install", "PREFIX="+prefix) -- cgit v1.2.3-70-g09d2 From 72b91758c9537009de967cfe560c2a510b9c2795 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 20 Apr 2016 10:06:49 +0200 Subject: openblas: raise an error when using +openmp with clang; set +openmp to false by default --- var/spack/repos/builtin/packages/openblas/package.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 99649da9ca..14f0395c4b 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -8,13 +8,14 @@ class Openblas(Package): homepage = "http://www.openblas.net" url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + version('0.2.18', '805e7f660877d588ea7e3792cda2ee65') version('0.2.17', '664a12807f2a2a7cda4781e3ab2ae0e1') version('0.2.16', 'fef46ab92463bdbb1479dcec594ef6dc') version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') - variant('shared', default=True, description="Build shared libraries as well as static libs.") - variant('openmp', default=True, description="Enable OpenMP support.") - variant('fpic', default=True, description="Build position independent code") + variant('shared', default=True, description="Build shared libraries as well as static libs.") + variant('openmp', default=False, description="Enable OpenMP support.") + variant('fpic', default=True, description="Build position independent code") # virtual dependency provides('blas') @@ -45,8 +46,13 @@ class Openblas(Package): make_defs += ['BUILD_LAPACK_DEPRECATED=1'] # Add support for OpenMP - # Note: Make sure your compiler supports OpenMP if '+openmp' in spec: + # Note: Apple's most recent Clang 7.3.0 still does not support OpenMP. + # What is worse, Openblas (as of 0.2.18) hardcoded that OpenMP cannot + # be used with any (!) compiler named clang, bummer. + if spec.satisfies('%clang'): + raise InstallError('OpenBLAS does not support OpenMP with clang!') + make_defs += ['USE_OPENMP=1'] make_args = make_defs + make_targets -- cgit v1.2.3-70-g09d2 From 95c7f4fba3477240c03019414923537b1da1a364 Mon Sep 17 00:00:00 2001 From: Jean-Paul Pelteret Date: Sun, 8 May 2016 16:53:31 +0200 Subject: Fixes #915 --- var/spack/repos/builtin/packages/gmsh/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/gmsh/package.py b/var/spack/repos/builtin/packages/gmsh/package.py index 9d759303cb..eb2981bba2 100644 --- a/var/spack/repos/builtin/packages/gmsh/package.py +++ b/var/spack/repos/builtin/packages/gmsh/package.py @@ -62,6 +62,9 @@ class Gmsh(Package): build_directory = join_path(self.stage.path, 'spack-build') source_directory = self.stage.source_path + + # Prevent GMsh from using its own strange directory structure on OSX + options.append('-DENABLE_OS_SPECIFIC_INSTALL=OFF') if '+shared' in spec: options.extend(['-DENABLE_BUILD_SHARED:BOOL=ON', -- cgit v1.2.3-70-g09d2 From 6e07f46df89ef272357a2aa42bfaf4c4d60e0dfe Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Mon, 9 May 2016 14:02:41 +0200 Subject: add missing m4 dependency to bison --- var/spack/repos/builtin/packages/bison/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/bison/package.py b/var/spack/repos/builtin/packages/bison/package.py index 7c526fb958..9a2ddcbf96 100644 --- a/var/spack/repos/builtin/packages/bison/package.py +++ b/var/spack/repos/builtin/packages/bison/package.py @@ -10,6 +10,8 @@ class Bison(Package): version('3.0.4', 'a586e11cd4aff49c3ff6d3b6a4c9ccf8') + depends_on("m4") + def install(self, spec, prefix): configure("--prefix=%s" % prefix) -- cgit v1.2.3-70-g09d2 From 970196d825237b96fdf27ecdbeb07bcf49580fff Mon Sep 17 00:00:00 2001 From: Jean-Paul Pelteret Date: Mon, 9 May 2016 14:23:07 +0200 Subject: GMsh: Fix binary linking against its own libraries GMsh binary now links against full path name of libraries. This fixes problems, such as `dyld: Library not loaded: libGmsh.2.11.dylib`, when running the executable. --- var/spack/repos/builtin/packages/gmsh/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/gmsh/package.py b/var/spack/repos/builtin/packages/gmsh/package.py index eb2981bba2..5f659c56df 100644 --- a/var/spack/repos/builtin/packages/gmsh/package.py +++ b/var/spack/repos/builtin/packages/gmsh/package.py @@ -62,7 +62,9 @@ class Gmsh(Package): build_directory = join_path(self.stage.path, 'spack-build') source_directory = self.stage.source_path - + + options.append('-DCMAKE_INSTALL_NAME_DIR:PATH=%s/lib' % prefix) + # Prevent GMsh from using its own strange directory structure on OSX options.append('-DENABLE_OS_SPECIFIC_INSTALL=OFF') -- cgit v1.2.3-70-g09d2 From 7a2d65967ce60efef1a5cce4969f3607915427f5 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 9 May 2016 22:04:34 +0200 Subject: wrap tty.die to 80 chars --- lib/spack/spack/compiler.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index a28d7302aa..b53c17494c 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -119,7 +119,9 @@ class Compiler(object): @property def openmp_flag(self): # If it is not overridden, assume it is not supported and warn the user - tty.die("The compiler you have chosen does not currently support OpenMP. If you think it should, please edit the compiler subclass and submit a pull request or issue.") + tty.die("The compiler you have chosen does not currently support OpenMP.", + "If you think it should, please edit the compiler subclass and", + "submit a pull request or issue.") # This property should be overridden in the compiler subclass if @@ -127,14 +129,20 @@ class Compiler(object): @property def cxx11_flag(self): # If it is not overridden, assume it is not supported and warn the user - tty.die("The compiler you have chosen does not currently support C++11. If you think it should, please edit the compiler subclass and submit a pull request or issue.") + tty.die("The compiler you have chosen does not currently support C++11.", + "If you think it should, please edit the compiler subclass and", + "submit a pull request or issue.") + # This property should be overridden in the compiler subclass if # C++14 is supported by that compiler @property def cxx14_flag(self): # If it is not overridden, assume it is not supported and warn the user - tty.die("The compiler you have chosen does not currently support C++14. If you think it should, please edit the compiler subclass and submit a pull request or issue.") + tty.die("The compiler you have chosen does not currently support C++14.", + "If you think it should, please edit the compiler subclass and", + "submit a pull request or issue.") + # -- cgit v1.2.3-70-g09d2 From 3948b082ad7f5eecc3b1078a2944f226a9a7ae87 Mon Sep 17 00:00:00 2001 From: Ben Couturier Date: Mon, 9 May 2016 22:32:46 +0200 Subject: Added gdb 7.11 --- var/spack/repos/builtin/packages/gdb/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/gdb/package.py b/var/spack/repos/builtin/packages/gdb/package.py index b346fe80c2..0e9e8fc099 100644 --- a/var/spack/repos/builtin/packages/gdb/package.py +++ b/var/spack/repos/builtin/packages/gdb/package.py @@ -34,6 +34,7 @@ class Gdb(Package): homepage = "https://www.gnu.org/software/gdb" url = "http://ftp.gnu.org/gnu/gdb/gdb-7.10.tar.gz" + version('7.11', 'f585059252836a981ea5db9a5f8ce97f') version('7.10.1', 'b93a2721393e5fa226375b42d567d90b') version('7.10', 'fa6827ad0fd2be1daa418abb11a54d86') version('7.9.1', 'f3b97de919a9dba84490b2e076ec4cb0') -- cgit v1.2.3-70-g09d2 From 7e6be184bc51060a3df79e35991a6018149e439f Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Fri, 22 Apr 2016 14:50:19 -0700 Subject: Updated and fixed the Scotch package. - Fixed a bug that was causing shared library usage to fail when linking with another application. - Updated the repository URL to allow for more general version downloading. - Added installation support for version 5.1.10b. - Cleaned up the installation file to make it a bit easier to follow and modify. --- var/spack/repos/builtin/packages/scotch/package.py | 118 +++++++++++---------- 1 file changed, 61 insertions(+), 57 deletions(-) diff --git a/var/spack/repos/builtin/packages/scotch/package.py b/var/spack/repos/builtin/packages/scotch/package.py index 8229ed8686..6f8c8d2706 100644 --- a/var/spack/repos/builtin/packages/scotch/package.py +++ b/var/spack/repos/builtin/packages/scotch/package.py @@ -4,85 +4,90 @@ import os class Scotch(Package): """Scotch is a software package for graph and mesh/hypergraph partitioning, graph clustering, and sparse matrix ordering.""" + homepage = "http://www.labri.fr/perso/pelegrin/scotch/" - url = "http://gforge.inria.fr/frs/download.php/file/34099/scotch_6.0.3.tar.gz" + url = "http://gforge.inria.fr/frs/download.php/latestfile/298/scotch_6.0.3.tar.gz" list_url = "http://gforge.inria.fr/frs/?group_id=248" version('6.0.3', '10b0cc0f184de2de99859eafaca83cfc') + version('5.1.10b', '9b8622b39c141ecaca4a46298486fd99') variant('mpi', default=False, description='Activate the compilation of PT-Scotch') variant('compression', default=True, description='Activate the posibility to use compressed files') variant('esmumps', default=False, description='Activate the compilation of the lib esmumps needed by mumps') variant('shared', default=True, description='Build shared libraries') - depends_on('mpi', when='+mpi') - depends_on('zlib', when='+compression') depends_on('flex') depends_on('bison') + depends_on('mpi', when='+mpi') + depends_on('zlib', when='+compression') - def compiler_specifics(self, makefile_inc, defines): - if self.compiler.name == 'gcc': - defines.append('-Drestrict=__restrict') - elif self.compiler.name == 'intel': - defines.append('-restrict') + def validate(self, spec): + # NOTE : Scotch v6.0.0 and older have separate tar files for their esmumps- + # compatible versions. In any normal circumstance, it would be better just + # to use these tar files since they're more comprehensive, but they + # unfortunately have very strange URLs that are non-uniform. For the time + # being, I'm going to just use the '~esmumps' URLs that are uniform for + # the sake of simplicity. + if spec.satisfies('@:6.0.0') and '+esmumps' in spec: + raise RuntimeError('The "+esmumps" variant is only supported for Scotch v6.0.1+.') - makefile_inc.append('CCS = $(CC)') + def patch(self): + makefile_inc = [] + cflags = [ + '-O3', + '-DCOMMON_RANDOM_FIXED_SEED', + '-DSCOTCH_DETERMINISTIC', + '-DSCOTCH_RENAME', + '-DIDXSIZE64' + ] - if '+mpi' in self.spec: + ## Library Build Type ## + + if '+shared' in self.spec: makefile_inc.extend([ - 'CCP = %s' % os.path.join(self.spec['mpi'].prefix.bin, 'mpicc'), - 'CCD = $(CCP)' - ]) + 'LIB = .so', + 'CLIBFLAGS = -shared -fPIC', + 'RANLIB = echo', + 'AR = $(CC)', + 'ARFLAGS = -shared $(LDFLAGS) -o' + ]) + cflags.append('-fPIC') else: makefile_inc.extend([ - 'CCP = mpicc', # It is set but not used - 'CCD = $(CCS)' - ]) + 'LIB = .a', + 'CLIBFLAGS = ', + 'RANLIB = ranlib', + 'AR = ar', + 'ARFLAGS = -ruv ' + ]) + ## Compiler-Specific Options ## + if self.compiler.name == 'gcc': + cflags.append('-Drestrict=__restrict') + elif self.compiler.name == 'intel': + cflags.append('-restrict') - def library_build_type(self, makefile_inc, defines): - makefile_inc.extend([ - 'LIB = .a', - 'CLIBFLAGS = ', - 'RANLIB = ranlib', - 'AR = ar', - 'ARFLAGS = -ruv ' - ]) + makefile_inc.append('CCS = $(CC)') + makefile_inc.append('CCP = %s' % + (os.path.join(self.spec['mpi'].prefix.bin, 'mpicc') if '+mpi' in self.spec else 'mpicc')) + makefile_inc.append('CCD = $(CCS)') - @when('+shared') - def library_build_type(self, makefile_inc, defines): - makefile_inc.extend([ - 'LIB = .so', - 'CLIBFLAGS = -shared -fPIC', - 'RANLIB = echo', - 'AR = $(CC)', - 'ARFLAGS = -shared $(LDFLAGS) -o' - ]) + ## Extra Features ## - def extra_features(self, makefile_inc, defines): ldflags = [] - + if '+compression' in self.spec: - defines.append('-DCOMMON_FILE_COMPRESS_GZ') + cflags.append('-DCOMMON_FILE_COMPRESS_GZ') ldflags.append('-L%s -lz' % (self.spec['zlib'].prefix.lib)) - defines.append('-DCOMMON_PTHREAD') + cflags.append('-DCOMMON_PTHREAD') ldflags.append('-lm -lrt -pthread') - - makefile_inc.append('LDFLAGS = %s' % ' '.join(ldflags)) - def patch(self): - makefile_inc = [] - defines = [ - '-DCOMMON_RANDOM_FIXED_SEED', - '-DSCOTCH_DETERMINISTIC', - '-DSCOTCH_RENAME', - '-DIDXSIZE64' ] + makefile_inc.append('LDFLAGS = %s' % ' '.join(ldflags)) - self.library_build_type(makefile_inc, defines) - self.compiler_specifics(makefile_inc, defines) - self.extra_features(makefile_inc, defines) + ## General Features ## makefile_inc.extend([ 'EXE =', @@ -93,18 +98,19 @@ class Scotch(Package): 'MKDIR = mkdir', 'MV = mv', 'CP = cp', - 'CFLAGS = -O3 %s' % (' '.join(defines)), + 'CFLAGS = %s' % ' '.join(cflags), 'LEX = %s -Pscotchyy -olex.yy.c' % os.path.join(self.spec['flex'].prefix.bin , 'flex'), 'YACC = %s -pscotchyy -y -b y' % os.path.join(self.spec['bison'].prefix.bin, 'bison'), - 'prefix = %s' % self.prefix, - '' + 'prefix = %s' % self.prefix ]) with working_dir('src'): with open('Makefile.inc', 'w') as fh: fh.write('\n'.join(makefile_inc)) - + def install(self, spec, prefix): + self.validate(spec) + targets = ['scotch'] if '+mpi' in self.spec: targets.append('ptscotch') @@ -115,12 +121,10 @@ class Scotch(Package): targets.append('ptesmumps') with working_dir('src'): - for app in targets: - make(app, parallel=(not app=='ptesmumps')) + for target in targets: + make(target, parallel=(target!='ptesmumps')) - install_tree('bin', prefix.bin) install_tree('lib', prefix.lib) install_tree('include', prefix.include) install_tree('man/man1', prefix.share_man1) - -- cgit v1.2.3-70-g09d2 From 88d2f6b83a46edb407397193fe5294d1303070a0 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Fri, 29 Apr 2016 11:44:38 -0700 Subject: Enabled the '+esmumps' variant for 'scotch@:6.0.0'. Added support for 'scotch@6.0.0'. --- .../repos/builtin/packages/scotch/Makefile.esmumps | 5 ++ var/spack/repos/builtin/packages/scotch/package.py | 58 +++++++++++++++++----- 2 files changed, 50 insertions(+), 13 deletions(-) create mode 100644 var/spack/repos/builtin/packages/scotch/Makefile.esmumps diff --git a/var/spack/repos/builtin/packages/scotch/Makefile.esmumps b/var/spack/repos/builtin/packages/scotch/Makefile.esmumps new file mode 100644 index 0000000000..4bfc760197 --- /dev/null +++ b/var/spack/repos/builtin/packages/scotch/Makefile.esmumps @@ -0,0 +1,5 @@ +esmumps : scotch + (cd esmumps ; $(MAKE) scotch && $(MAKE) install) + +ptesmumps : ptscotch + (cd esmumps ; $(MAKE) ptscotch && $(MAKE) ptinstall) diff --git a/var/spack/repos/builtin/packages/scotch/package.py b/var/spack/repos/builtin/packages/scotch/package.py index 6f8c8d2706..5820e826de 100644 --- a/var/spack/repos/builtin/packages/scotch/package.py +++ b/var/spack/repos/builtin/packages/scotch/package.py @@ -1,16 +1,18 @@ from spack import * -import os +import os, re class Scotch(Package): """Scotch is a software package for graph and mesh/hypergraph partitioning, graph clustering, and sparse matrix ordering.""" homepage = "http://www.labri.fr/perso/pelegrin/scotch/" - url = "http://gforge.inria.fr/frs/download.php/latestfile/298/scotch_6.0.3.tar.gz" + url = "http://gforge.inria.fr/frs/download.php/latestfile/298/scotch_6.0.3.tar.gz" + base_url = "http://gforge.inria.fr/frs/download.php/latestfile/298" list_url = "http://gforge.inria.fr/frs/?group_id=248" version('6.0.3', '10b0cc0f184de2de99859eafaca83cfc') - version('5.1.10b', '9b8622b39c141ecaca4a46298486fd99') + version('6.0.0', 'c50d6187462ba801f9a82133ee666e8e') + version('5.1.10b', 'f587201d6cf5cf63527182fbfba70753') variant('mpi', default=False, description='Activate the compilation of PT-Scotch') variant('compression', default=True, description='Activate the posibility to use compressed files') @@ -22,17 +24,47 @@ class Scotch(Package): depends_on('mpi', when='+mpi') depends_on('zlib', when='+compression') - def validate(self, spec): - # NOTE : Scotch v6.0.0 and older have separate tar files for their esmumps- - # compatible versions. In any normal circumstance, it would be better just - # to use these tar files since they're more comprehensive, but they - # unfortunately have very strange URLs that are non-uniform. For the time - # being, I'm going to just use the '~esmumps' URLs that are uniform for - # the sake of simplicity. - if spec.satisfies('@:6.0.0') and '+esmumps' in spec: - raise RuntimeError('The "+esmumps" variant is only supported for Scotch v6.0.1+.') + # NOTE: Versions of Scotch up to version 6.0.0 don't include support for + # building with 'esmumps' in their default packages. In order to enable + # support for this feature, we must grab the 'esmumps' enabled archives + # from the Scotch hosting site. These alternative archives include a strict + # superset of the behavior in their default counterparts, so we choose to + # always grab these versions for older Scotch versions for simplicity. + @when('@:6.0.0') + def url_for_version(self, version): + return '%s/scotch_%s_esmumps.tar.gz' % (Scotch.base_url, version) + + @when('@6.0.1:') + def url_for_version(self, version): + return super(Scotch, self).url_for_version(version) + + # NOTE: Several of the 'esmumps' enabled Scotch releases up to version 6.0.0 + # have broken build scripts that don't properly build 'esmumps' as a separate + # target, so we need a patch procedure to remove 'esmumps' from existing targets + # and to add it as a standalone target. + @when('@:6.0.0') + def patch(self): + makefile_path = os.path.join('src', 'Makefile') + with open(makefile_path, 'r') as makefile: + esmumps_enabled = any(re.search(r'^esmumps(\s*):(.*)$', line) for line in makefile.readlines()) + + if not esmumps_enabled: + mff = FileFilter(makefile_path) + mff.filter(r'^.*((esmumps)|(ptesmumps)).*(install).*$', '') + makefile_esmumps_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'Makefile.esmumps') + with open(makefile_path, 'a') as makefile: + makefile.write('\ninclude %s\n' % makefile_esmumps_path) + + @when('@6.0.1:') def patch(self): + pass + + # NOTE: Configuration of Scotch is achieved by writing a 'Makefile.inc' file + # that contains all of the configuration variables and their desired values + # for the installation. This function writes this file based on the given + # installation variants. + def configure(self): makefile_inc = [] cflags = [ '-O3', @@ -109,7 +141,7 @@ class Scotch(Package): fh.write('\n'.join(makefile_inc)) def install(self, spec, prefix): - self.validate(spec) + self.configure() targets = ['scotch'] if '+mpi' in self.spec: -- cgit v1.2.3-70-g09d2 From ceab445b9fe371d50703c599ec2cf742f6199f41 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Mon, 9 May 2016 14:39:42 -0700 Subject: Integrated improvements from PR #893 to remove hardcoded MPI path. --- var/spack/repos/builtin/packages/scotch/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/scotch/package.py b/var/spack/repos/builtin/packages/scotch/package.py index 5820e826de..8fad74b24f 100644 --- a/var/spack/repos/builtin/packages/scotch/package.py +++ b/var/spack/repos/builtin/packages/scotch/package.py @@ -103,7 +103,7 @@ class Scotch(Package): makefile_inc.append('CCS = $(CC)') makefile_inc.append('CCP = %s' % - (os.path.join(self.spec['mpi'].prefix.bin, 'mpicc') if '+mpi' in self.spec else 'mpicc')) + (self.spec['mpi'].mpicc if '+mpi' in self.spec else 'mpicc')) makefile_inc.append('CCD = $(CCS)') ## Extra Features ## -- cgit v1.2.3-70-g09d2 From 4473311bdbddb6abe2f1ad9748536db89f79081d Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 5 May 2016 10:50:07 -0500 Subject: Allow compilers to specify their own rpath linking flags --- lib/spack/env/cc | 39 +++++++++++++++++++++++++----------- lib/spack/spack/build_environment.py | 14 +++++++++---- lib/spack/spack/compiler.py | 18 +++++++++++++++-- lib/spack/spack/compilers/nag.py | 11 ++++++++++ lib/spack/spack/test/cc.py | 5 +++++ 5 files changed, 69 insertions(+), 18 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index cb07a2ffea..4564e84bd2 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -38,15 +38,20 @@ # -Wl,-rpath arguments for dependency /lib directories. # -# This is the list of environment variables that need to be set before +# This is an array of environment variables that need to be set before # the script runs. They are set by routines in spack.build_environment # as part of spack.package.Package.do_install(). -parameters=" -SPACK_PREFIX -SPACK_ENV_PATH -SPACK_DEBUG_LOG_DIR -SPACK_COMPILER_SPEC -SPACK_SHORT_SPEC" +parameters=( + SPACK_PREFIX + SPACK_ENV_PATH + SPACK_DEBUG_LOG_DIR + SPACK_COMPILER_SPEC + SPACK_CC_RPATH_ARG + SPACK_CXX_RPATH_ARG + SPACK_F77_RPATH_ARG + SPACK_FC_RPATH_ARG + SPACK_SHORT_SPEC +) # The compiler input variables are checked for sanity later: # SPACK_CC, SPACK_CXX, SPACK_F77, SPACK_FC @@ -64,7 +69,7 @@ function die { exit 1 } -for param in $parameters; do +for param in ${parameters[@]}; do if [[ -z ${!param} ]]; then die "Spack compiler must be run from Spack! Input '$param' is missing." fi @@ -85,6 +90,7 @@ done # ccld compile & link command=$(basename "$0") +comp="CC" case "$command" in cpp) mode=cpp @@ -92,18 +98,22 @@ case "$command" in cc|c89|c99|gcc|clang|icc|pgcc|xlc) command="$SPACK_CC" language="C" + comp="CC" ;; c++|CC|g++|clang++|icpc|pgc++|xlc++) command="$SPACK_CXX" language="C++" + comp="CXX" ;; f90|fc|f95|gfortran|ifort|pgfortran|xlf90|nagfor) command="$SPACK_FC" language="Fortran 90" + comp="FC" ;; f77|gfortran|ifort|pgfortran|xlf|nagfor) command="$SPACK_F77" language="Fortran 77" + comp="F77" ;; ld) mode=ld @@ -142,6 +152,9 @@ if [[ -z $mode ]]; then done fi +# Set up rpath variable according to language. +eval rpath=\$SPACK_${comp}_RPATH_ARG + # Dump the version and exit if we're in testing mode. if [[ $SPACK_TEST_COMMAND == dump-mode ]]; then echo "$mode" @@ -188,7 +201,7 @@ for dep in "${deps[@]}"; do # Prepend lib and RPATH directories if [[ -d $dep/lib ]]; then if [[ $mode == ccld ]]; then - $add_rpaths && args=("-Wl,-rpath,$dep/lib" "${args[@]}") + $add_rpaths && args=("$rpath$dep/lib" "${args[@]}") args=("-L$dep/lib" "${args[@]}") elif [[ $mode == ld ]]; then $add_rpaths && args=("-rpath" "$dep/lib" "${args[@]}") @@ -199,7 +212,7 @@ for dep in "${deps[@]}"; do # Prepend lib64 and RPATH directories if [[ -d $dep/lib64 ]]; then if [[ $mode == ccld ]]; then - $add_rpaths && args=("-Wl,-rpath,$dep/lib64" "${args[@]}") + $add_rpaths && args=("$rpath$dep/lib64" "${args[@]}") args=("-L$dep/lib64" "${args[@]}") elif [[ $mode == ld ]]; then $add_rpaths && args=("-rpath" "$dep/lib64" "${args[@]}") @@ -210,9 +223,11 @@ done # Include all -L's and prefix/whatever dirs in rpath if [[ $mode == ccld ]]; then - $add_rpaths && args=("-Wl,-rpath,$SPACK_PREFIX/lib" "-Wl,-rpath,$SPACK_PREFIX/lib64" "${args[@]}") + $add_rpaths && args=("$rpath$SPACK_PREFIX/lib64" "${args[@]}") + $add_rpaths && args=("$rpath$SPACK_PREFIX/lib" "${args[@]}") elif [[ $mode == ld ]]; then - $add_rpaths && args=("-rpath" "$SPACK_PREFIX/lib" "-rpath" "$SPACK_PREFIX/lib64" "${args[@]}") + $add_rpaths && args=("-rpath" "$SPACK_PREFIX/lib64" "${args[@]}") + $add_rpaths && args=("-rpath" "$SPACK_PREFIX/lib" "${args[@]}") fi # diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index eb72f2a6b4..411425549a 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -98,21 +98,27 @@ def set_compiler_environment_variables(pkg, env): # and return it # TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc. link_dir = spack.build_env_path - env.set('CC', join_path(link_dir, pkg.compiler.link_paths['cc'])) + env.set('CC', join_path(link_dir, pkg.compiler.link_paths['cc'])) env.set('CXX', join_path(link_dir, pkg.compiler.link_paths['cxx'])) env.set('F77', join_path(link_dir, pkg.compiler.link_paths['f77'])) - env.set('FC', join_path(link_dir, pkg.compiler.link_paths['fc'])) + env.set('FC', join_path(link_dir, pkg.compiler.link_paths['fc'])) # Set SPACK compiler variables so that our wrapper knows what to call compiler = pkg.compiler if compiler.cc: - env.set('SPACK_CC', compiler.cc) + env.set('SPACK_CC', compiler.cc) if compiler.cxx: env.set('SPACK_CXX', compiler.cxx) if compiler.f77: env.set('SPACK_F77', compiler.f77) if compiler.fc: - env.set('SPACK_FC', compiler.fc) + env.set('SPACK_FC', compiler.fc) + + # Set SPACK compiler rpath flags so that our wrapper knows what to use + env.set('SPACK_CC_RPATH_ARG', compiler.cc_rpath_arg) + env.set('SPACK_CXX_RPATH_ARG', compiler.cxx_rpath_arg) + env.set('SPACK_F77_RPATH_ARG', compiler.f77_rpath_arg) + env.set('SPACK_FC_RPATH_ARG', compiler.fc_rpath_arg) env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler)) return env diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index b53c17494c..8c197520b5 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -91,8 +91,22 @@ class Compiler(object): # version suffix for gcc. suffixes = [r'-.*'] - # Names of generic arguments used by this compiler - arg_rpath = '-Wl,-rpath,%s' + # Flags used by this compiler to set an rpath + @property + def cc_rpath_arg(self): + return '-Wl,-rpath,' + + @property + def cxx_rpath_arg(self): + return '-Wl,-rpath,' + + @property + def f77_rpath_arg(self): + return '-Wl,-rpath,' + + @property + def fc_rpath_arg(self): + return '-Wl,-rpath,' def __init__(self, cspec, cc, cxx, f77, fc): diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index e9038c1039..49b77eae6b 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -31,6 +31,17 @@ class Nag(Compiler): # However, it can be mixed with a compiler that does support it return "-std=c++11" + # Unlike other compilers, the NAG compiler passes options to GCC, which + # then passes them to the linker. Therefore, we need to doubly wrap the + # options with '-Wl,-Wl,,' + @property + def f77_rpath_arg(self): + return '-Wl,-Wl,,-rpath,' + + @property + def fc_rpath_arg(self): + return '-Wl,-Wl,,-rpath,' + @classmethod def default_version(self, comp): """The '-V' option works for nag compilers. diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 594cd6efe9..cdac319088 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -67,6 +67,11 @@ class CompilerTest(unittest.TestCase): os.environ['SPACK_COMPILER_SPEC'] = "gcc@4.4.7" os.environ['SPACK_SHORT_SPEC'] = "foo@1.2" + os.environ['SPACK_CC_RPATH_ARG'] = "-Wl,-rpath" + os.environ['SPACK_CXX_RPATH_ARG'] = "-Wl,-rpath" + os.environ['SPACK_F77_RPATH_ARG'] = "-Wl,-rpath" + os.environ['SPACK_FC_RPATH_ARG'] = "-Wl,-rpath" + # Make some fake dependencies self.tmp_deps = tempfile.mkdtemp() self.dep1 = join_path(self.tmp_deps, 'dep1') -- cgit v1.2.3-70-g09d2 From b211829fb137994fbd7467c713fd026bc29cef3f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 5 May 2016 13:54:10 -0500 Subject: Testing typo --- lib/spack/spack/test/cc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index cdac319088..35392d9d6d 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -67,10 +67,10 @@ class CompilerTest(unittest.TestCase): os.environ['SPACK_COMPILER_SPEC'] = "gcc@4.4.7" os.environ['SPACK_SHORT_SPEC'] = "foo@1.2" - os.environ['SPACK_CC_RPATH_ARG'] = "-Wl,-rpath" - os.environ['SPACK_CXX_RPATH_ARG'] = "-Wl,-rpath" - os.environ['SPACK_F77_RPATH_ARG'] = "-Wl,-rpath" - os.environ['SPACK_FC_RPATH_ARG'] = "-Wl,-rpath" + os.environ['SPACK_CC_RPATH_ARG'] = "-Wl,-rpath," + os.environ['SPACK_CXX_RPATH_ARG'] = "-Wl,-rpath," + os.environ['SPACK_F77_RPATH_ARG'] = "-Wl,-rpath," + os.environ['SPACK_FC_RPATH_ARG'] = "-Wl,-rpath," # Make some fake dependencies self.tmp_deps = tempfile.mkdtemp() -- cgit v1.2.3-70-g09d2 From 58733eb26a734fd00cac40ca2ff7422ccf7669fa Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 6 May 2016 09:57:59 -0500 Subject: Comment change --- lib/spack/spack/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 8c197520b5..976d96825c 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -91,7 +91,7 @@ class Compiler(object): # version suffix for gcc. suffixes = [r'-.*'] - # Flags used by this compiler to set an rpath + # Default flags used by a compiler to set an rpath @property def cc_rpath_arg(self): return '-Wl,-rpath,' -- cgit v1.2.3-70-g09d2 From 6665a996e62e2097b68392afaa45d01305bcc399 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 9 May 2016 14:08:11 -0500 Subject: Add documentation for rpath_flag handling --- lib/spack/docs/packaging_guide.rst | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 31c676d4f5..1b7941ab24 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1803,15 +1803,15 @@ Compile-time library search paths * ``-L$dep_prefix/lib`` * ``-L$dep_prefix/lib64`` Runtime library search paths (RPATHs) - * ``-Wl,-rpath,$dep_prefix/lib`` - * ``-Wl,-rpath,$dep_prefix/lib64`` + * ``$rpath_flag$dep_prefix/lib`` + * ``$rpath_flag$dep_prefix/lib64`` Include search paths * ``-I$dep_prefix/include`` An example of this would be the ``libdwarf`` build, which has one dependency: ``libelf``. Every call to ``cc`` in the ``libdwarf`` build will have ``-I$LIBELF_PREFIX/include``, -``-L$LIBELF_PREFIX/lib``, and ``-Wl,-rpath,$LIBELF_PREFIX/lib`` +``-L$LIBELF_PREFIX/lib``, and ``$rpath_flag$LIBELF_PREFIX/lib`` inserted on the command line. This is done transparently to the project's build system, which will just think it's using a system where ``libelf`` is readily available. Because of this, you **do @@ -1831,6 +1831,14 @@ successfully find ``libdwarf.h`` and ``libdwarf.so``, without the packager having to provide ``--with-libdwarf=/path/to/libdwarf`` on the command line. +.. note:: + + For most compilers, ``$rpath_flag`` is ``-Wl,-rpath,``. However, NAG + passes its flags to GCC instead of passing them directly to the linker. + Therefore, its ``$rpath_flag`` is doubly wrapped: ``-Wl,-Wl,,-rpath,``. + ``$rpath_flag`` can be overriden on a compiler specific basis in + ``lib/spack/spack/compilers/$compiler.py``. + Compiler flags ~~~~~~~~~~~~~~ In rare circumstances such as compiling and running small unit tests, a package @@ -1848,8 +1856,6 @@ package supports additional variants like variant('openmp', default=True, description="Enable OpenMP support.") - - Message Parsing Interface (MPI) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ It is common for high performance computing software/packages to use ``MPI``. -- cgit v1.2.3-70-g09d2 From 045e5bd4585fdf556651b9c71a0ca7428a27563c Mon Sep 17 00:00:00 2001 From: "Tanzima Z. Islam" Date: Mon, 9 May 2016 16:22:07 -0700 Subject: Adding a new package: Kripke from the public tar ball --- var/spack/repos/builtin/packages/kripke/package.py | 38 ++++++++++++++++------ 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/kripke/package.py b/var/spack/repos/builtin/packages/kripke/package.py index 68ccc4cb6c..b6f4e4fd73 100644 --- a/var/spack/repos/builtin/packages/kripke/package.py +++ b/var/spack/repos/builtin/packages/kripke/package.py @@ -1,16 +1,34 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install kripke +# +# You can always get back here to change things with: +# +# spack edit kripke +# +# See the spack documentation for more information on building +# packages. +# from spack import * class Kripke(Package): - """Kripke is a simple, scalable, 3D Sn deterministic particle transport code.""" - + """Kripke is a simple, scalable, 3D Sn deterministic particle transport proxy/mini app.""" homepage = "https://codesign.llnl.gov/kripke.php" - url = "" - #version('master', git='https://lc.llnl.gov/stash/scm/kripke/kripke.git') - version('master', git='https://lc.llnl.gov/stash/scm/~islam3/kripke.git') + url = "https://codesign.llnl.gov/downloads/kripke-openmp-1.1.tar.gz" - def install(self, spec, prefix): - with working_dir('build', create=True): - cmake('-DCMAKE_INSTALL_PREFIX:PATH=.', '-DCMAKE_TOOLCHAIN_FILE=../cmake/Toolchain/chaos_5_x86_64_ib-ic15.cmake', '-DENABLE_OPENMP=1', '..', *std_cmake_args) - make() - make("install") + version('1.1', '7fe6f2b26ed983a6ce5495ab701f85bf') + #depends_on("mvapich2@1.9:") + + def install(self, spec, prefix): + with working_dir('build', create=True): + cmake('-DCMAKE_INSTALL_PREFIX:PATH=.', '-DENABLE_OPENMP=1', '-DENABLE_MPI=1', '..', *std_cmake_args) + make() + #Kripke does not provide an install, so creating one here. + mkdirp(prefix.bin) + install('kripke', prefix.bin) -- cgit v1.2.3-70-g09d2 From 23ec6c6bb031d48e64be07acd199b957ada774e8 Mon Sep 17 00:00:00 2001 From: "Tanzima Z. Islam" Date: Mon, 9 May 2016 16:34:27 -0700 Subject: Removed FIXME comments --- var/spack/repos/builtin/packages/kripke/package.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/var/spack/repos/builtin/packages/kripke/package.py b/var/spack/repos/builtin/packages/kripke/package.py index b6f4e4fd73..f7cc96053a 100644 --- a/var/spack/repos/builtin/packages/kripke/package.py +++ b/var/spack/repos/builtin/packages/kripke/package.py @@ -1,19 +1,3 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install kripke -# -# You can always get back here to change things with: -# -# spack edit kripke -# -# See the spack documentation for more information on building -# packages. -# from spack import * class Kripke(Package): -- cgit v1.2.3-70-g09d2 From 2e0ee5404d839a69c626407436c6cb059a51d9fd Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 9 May 2016 17:14:25 -0700 Subject: clean up Kripke package and dependencies. --- var/spack/repos/builtin/packages/kripke/package.py | 24 +++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/kripke/package.py b/var/spack/repos/builtin/packages/kripke/package.py index f7cc96053a..345b8af4d0 100644 --- a/var/spack/repos/builtin/packages/kripke/package.py +++ b/var/spack/repos/builtin/packages/kripke/package.py @@ -1,18 +1,28 @@ from spack import * class Kripke(Package): - """Kripke is a simple, scalable, 3D Sn deterministic particle transport proxy/mini app.""" + """Kripke is a simple, scalable, 3D Sn deterministic particle + transport proxy/mini app. + """ homepage = "https://codesign.llnl.gov/kripke.php" url = "https://codesign.llnl.gov/downloads/kripke-openmp-1.1.tar.gz" version('1.1', '7fe6f2b26ed983a6ce5495ab701f85bf') - #depends_on("mvapich2@1.9:") + variant('mpi', default=True, description='Enable MPI support') + + depends_on('mpi', when="+mpi") def install(self, spec, prefix): with working_dir('build', create=True): - cmake('-DCMAKE_INSTALL_PREFIX:PATH=.', '-DENABLE_OPENMP=1', '-DENABLE_MPI=1', '..', *std_cmake_args) - make() - #Kripke does not provide an install, so creating one here. - mkdirp(prefix.bin) - install('kripke', prefix.bin) + cmake('-DCMAKE_INSTALL_PREFIX:PATH=.', + '-DENABLE_OPENMP=1', + '-DENABLE_MPI=1', + '..', + *std_cmake_args) + make() + + # Kripke does not provide install target, so we have to copy + # things into place. + mkdirp(prefix.bin) + install('kripke', prefix.bin) -- cgit v1.2.3-70-g09d2 From b063ab42bff9a3382a6c79663d41d13a0c028c50 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 27 Apr 2016 11:57:48 +0200 Subject: openblas: fix and cleanup the unit test --- .../repos/builtin/packages/openblas/package.py | 68 +++++++++------------- .../builtin/packages/openblas/test_cblas_dgemm.c | 54 ++++++++++++++--- .../packages/openblas/test_cblas_dgemm.output | 3 + 3 files changed, 74 insertions(+), 51 deletions(-) diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 99649da9ca..72b0b65a2f 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -88,48 +88,33 @@ class Openblas(Package): self.spec.lapack_shared_lib = self.spec.blas_shared_lib def check_install(self, spec): - "Build and run a small program to test that we have Lapack symbols" + # TODO: Pull this out to the framework function which recieves a pair of xyz.c and xyz.output print "Checking Openblas installation..." - checkdir = "spack-check" - with working_dir(checkdir, create=True): - source = r""" -#include -#include -int main(void) { -int i=0; -double A[6] = {1.0, 2.0, 1.0, -3.0, 4.0, -1.0}; -double B[6] = {1.0, 2.0, 1.0, -3.0, 4.0, -1.0}; -double C[9] = {.5, .5, .5, .5, .5, .5, .5, .5, .5}; -cblas_dgemm(CblasColMajor, CblasNoTrans, CblasTrans, - 3, 3, 2, 1, A, 3, B, 3, 2, C, 3); -for (i = 0; i < 9; i++) - printf("%f\n", C[i]); -return 0; -} -""" - expected = """\ -11.000000 --9.000000 -5.000000 --9.000000 -21.000000 --1.000000 -5.000000 --1.000000 -3.000000 -""" - with open("check.c", 'w') as f: - f.write(source) - cc = which('cc') - # TODO: Automate these path and library settings - cc('-c', "-I%s" % join_path(spec.prefix, "include"), "check.c") - cc('-o', "check", "check.o", - "-L%s" % join_path(spec.prefix, "lib"), "-llapack", "-lblas", "-lpthread") - try: - check = Executable('./check') - output = check(return_output=True) - except: - output = "" + source_file = join_path(os.path.dirname(self.module.__file__), + 'test_cblas_dgemm.c') + output_file = join_path(os.path.dirname(self.module.__file__), + 'test_cblas_dgemm.output') + + with open(output_file, 'r') as f: + expected = f.read() + + cc = which('cc') + cc('-c', "-I%s" % join_path(spec.prefix, "include"), source_file) + link_flags = ["-L%s" % join_path(spec.prefix, "lib"), + "-llapack", + "-lblas", + "-lpthread" + ] + if '+openmp' in spec: + link_flags.extend([self.compiler.openmp_flag]) + cc('-o', "check", "test_cblas_dgemm.o", + *link_flags) + + try: + check = Executable('./check') + output = check(return_output=True) + except: + output = "" success = output == expected if not success: print "Produced output does not match expected output." @@ -142,4 +127,3 @@ return 0; print output print '-'*80 raise RuntimeError("Openblas install check failed") - shutil.rmtree(checkdir) diff --git a/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.c b/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.c index 634e99d20b..3813a23b69 100644 --- a/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.c +++ b/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.c @@ -1,13 +1,49 @@ #include #include + +double m[] = { + 3, 1, 3, + 1, 5, 9, + 2, 6, 5 +}; + +double x[] = { + -1, 3, -3 +}; + +#ifdef __cplusplus +extern "C" { +#endif + + void dgesv_(int *n, int *nrhs, double *a, int *lda, + int *ipivot, double *b, int *ldb, int *info); + +#ifdef __cplusplus +} +#endif + int main(void) { -int i=0; -double A[6] = {1.0, 2.0, 1.0, -3.0, 4.0, -1.0}; -double B[6] = {1.0, 2.0, 1.0, -3.0, 4.0, -1.0}; -double C[9] = {.5, .5, .5, .5, .5, .5, .5, .5, .5}; -cblas_dgemm(CblasColMajor, CblasNoTrans, CblasTrans, - 3, 3, 2, 1, A, 3, B, 3, 2, C, 3); -for (i = 0; i < 9; i++) - printf("%f\n", C[i]); -return 0; + int i; + // blas: + double A[6] = {1.0, 2.0, 1.0, -3.0, 4.0, -1.0}; + double B[6] = {1.0, 2.0, 1.0, -3.0, 4.0, -1.0}; + double C[9] = {.5, .5, .5, .5, .5, .5, .5, .5, .5}; + cblas_dgemm(CblasColMajor, CblasNoTrans, CblasTrans, + 3, 3, 2, 1, A, 3, B, 3, 2, C, 3); + for (i = 0; i < 9; i++) + printf("%f\n", C[i]); + + // lapack: + int ipiv[3]; + int j; + int info; + int n = 1; + int nrhs = 1; + int lda = 3; + int ldb = 3; + dgesv_(&n,&nrhs, &m[0], &lda, ipiv, &x[0], &ldb, &info); + for (i=0; i<3; ++i) + printf("%5.1f %3d\n", x[i], ipiv[i]); + + return 0; } diff --git a/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.output b/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.output index b8316d7477..9c235e314f 100644 --- a/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.output +++ b/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.output @@ -7,3 +7,6 @@ 5.000000 -1.000000 3.000000 + -0.3 1 + 3.0 1499101120 + -3.0 32767 -- cgit v1.2.3-70-g09d2 From 474048ae8ba5118ad995d271523241c80ab7c3e5 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 10 May 2016 09:02:16 +0200 Subject: openmpi : fixes #920, added sqlite as a dependency The default for the variant `pmi` has been set to `False` `sqlite` is now a dependency if `+sqlite3` --- var/spack/repos/builtin/packages/openmpi/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 042f072226..c656f78dab 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -29,7 +29,7 @@ class Openmpi(Package): variant('psm', default=False, description='Build support for the PSM library.') variant('psm2', default=False, description='Build support for the Intel PSM2 library.') - variant('pmi', default=True, description='Build support for PMI-based launchers') + variant('pmi', default=False, description='Build support for PMI-based launchers') variant('verbs', default=False, description='Build support for OpenFabrics verbs.') variant('mxm', default=False, description='Build Mellanox Messaging support') @@ -47,6 +47,7 @@ class Openmpi(Package): provides('mpi@:3.0', when='@1.7.5:') depends_on('hwloc') + depends_on('sqlite', when='+sqlite3') def url_for_version(self, version): return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) -- cgit v1.2.3-70-g09d2 From e3115aa505b8fc68893e4ebeaf5e5f152507e693 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 10 May 2016 00:31:06 -0700 Subject: Kripke variants. --- var/spack/repos/builtin/packages/kripke/package.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/kripke/package.py b/var/spack/repos/builtin/packages/kripke/package.py index 345b8af4d0..7d067ea44d 100644 --- a/var/spack/repos/builtin/packages/kripke/package.py +++ b/var/spack/repos/builtin/packages/kripke/package.py @@ -9,15 +9,19 @@ class Kripke(Package): version('1.1', '7fe6f2b26ed983a6ce5495ab701f85bf') - variant('mpi', default=True, description='Enable MPI support') + variant('mpi', default=True, description='Build with MPI.') + variant('openmp', default=True, description='Build with OpenMP enabled.') depends_on('mpi', when="+mpi") def install(self, spec, prefix): with working_dir('build', create=True): + def enabled(variant): + return (1 if variant in spec else 0) + cmake('-DCMAKE_INSTALL_PREFIX:PATH=.', - '-DENABLE_OPENMP=1', - '-DENABLE_MPI=1', + '-DENABLE_OPENMP=%d' % enabled('+openmp'), + '-DENABLE_MPI=%d' % enabled('+mpi'), '..', *std_cmake_args) make() -- cgit v1.2.3-70-g09d2 From 81cb520b470511509446246c72be4ae6220e088e Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 9 May 2016 15:18:57 -0700 Subject: Make Travis badge use SVG. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7797da299c..1f7bcc5c61 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ![image](share/spack/logo/spack-logo-text-64.png "Spack") ============ -[![Build Status](https://travis-ci.org/LLNL/spack.png?branch=develop)](https://travis-ci.org/LLNL/spack) +[![Build Status](https://travis-ci.org/LLNL/spack.svg?branch=develop)](https://travis-ci.org/LLNL/spack) [![Coverage Status](https://coveralls.io/repos/github/LLNL/spack/badge.svg?branch=develop)](https://coveralls.io/github/LLNL/spack?branch=develop) Spack is a package management tool designed to support multiple -- cgit v1.2.3-70-g09d2 From d3ade02c161822589e4647056293da2b8313d70c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 9 May 2016 15:19:17 -0700 Subject: Make .style.yapf use 80 chars. --- .style.yapf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.style.yapf b/.style.yapf index b4a1f688b8..4741fb4f3b 100644 --- a/.style.yapf +++ b/.style.yapf @@ -1,3 +1,3 @@ [style] based_on_style = pep8 -column_limit = 120 \ No newline at end of file +column_limit = 80 -- cgit v1.2.3-70-g09d2 From e60f25f20944517ce064e2091f9e680d9c46eda4 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 10 May 2016 00:31:43 -0700 Subject: Move args to .coveragerc --- .coveragerc | 7 +++++++ .travis.yml | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index 37410a3677..a1271a94fc 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,13 @@ +# -*- conf -*- # .coveragerc to control coverage.py [run] branch = True +source = lib +omit = + lib/spack/spack/test/* + lib/spack/env/* + lib/spack/docs/* + lib/spack/external/* [report] # Regexes for lines to exclude from consideration diff --git a/.travis.yml b/.travis.yml index 30dfebf783..4ff4d5f483 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,9 +24,9 @@ script: - spack config get compilers - spack install -v libdwarf # Run unit tests with code coverage - - coverage run --source=lib --omit=lib/spack/spack/test/*,lib/spack/env/*,lib/spack/docs/*,lib/spack/external/* bin/spack test + - coverage run bin/spack test # Checks if the file that have been changed are flake8 conformant - - CHANGED_PYTHON_FILES=`git diff develop... --name-only | perl -ne 'print if /\.py/g'` + - CHANGED_PYTHON_FILES=`git diff develop... --name-only | perl -ne 'print if /\.py$/'` - if [[ ${CHANGED_PYTHON_FILES} ]] ; then flake8 --format pylint --config flake8.ini ${CHANGED_PYTHON_FILES} ; fi -- cgit v1.2.3-70-g09d2 From 0f427ed334f8a58e888872d60419709cfd6f41c3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 10 May 2016 01:07:17 -0700 Subject: Tweak nccmp to be more spack-compatible. - Spack doesn't set F90, but it confuses the nccmp build. Just remove it from the environment. - TODO: should build environment unset this variable? --- var/spack/repos/builtin/packages/nccmp/package.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/nccmp/package.py b/var/spack/repos/builtin/packages/nccmp/package.py index 458afbb1e8..72e86831c6 100644 --- a/var/spack/repos/builtin/packages/nccmp/package.py +++ b/var/spack/repos/builtin/packages/nccmp/package.py @@ -1,5 +1,4 @@ from spack import * -import os class Nccmp(Package): """Compare NetCDF Files""" @@ -15,16 +14,10 @@ class Nccmp(Package): # FCFLAGS respectively in this configure, please unset # F90/F90FLAGS and set FC/FCFLAGS instead and rerun configure # again. - os.environ['FC'] = os.environ['F90'] - del os.environ['F90'] - try: - os.environ['FCFLAGS'] = os.environ['F90FLAGS'] - del os.environ['F90FLAGS'] - except KeyError: # There are no flags - pass + env.pop('F90', None) + env.pop('F90FLAGS', None) configure('--prefix=%s' % prefix) - make() make("check") make("install") -- cgit v1.2.3-70-g09d2 From 8f4efeea973a7d2190b660acc27df882341c95dc Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 10 May 2016 02:33:22 -0700 Subject: Max line length of 79 chars. --- .style.yapf | 2 +- flake8.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.style.yapf b/.style.yapf index 4741fb4f3b..a4b3f65252 100644 --- a/.style.yapf +++ b/.style.yapf @@ -1,3 +1,3 @@ [style] based_on_style = pep8 -column_limit = 80 +column_limit = 79 diff --git a/flake8.ini b/flake8.ini index 757c71705e..15fc1b87b7 100644 --- a/flake8.ini +++ b/flake8.ini @@ -1,3 +1,3 @@ [flake8] ignore = W391,F403 -max-line-length = 120 \ No newline at end of file +max-line-length = 79 -- cgit v1.2.3-70-g09d2 From aca5941a0e670e6bcc5873ecf08ce33bfe569955 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Tue, 10 May 2016 11:48:18 +0200 Subject: Add a few more python packages py-Genshi py-astroid py-jinja2 py-logilab-common py-markupsafe py-mistune py-prettytable py-py2neo py-storm --- .../repos/builtin/packages/py-Genshi/package.py | 16 ++++++++++++++ .../repos/builtin/packages/py-astroid/package.py | 20 +++++++++++++++++ .../repos/builtin/packages/py-jinja2/package.py | 25 ++++++++++++++++++++++ .../builtin/packages/py-logilab-common/package.py | 16 ++++++++++++++ .../builtin/packages/py-markupsafe/package.py | 25 ++++++++++++++++++++++ .../repos/builtin/packages/py-mistune/package.py | 20 +++++++++++++++++ .../builtin/packages/py-prettytable/package.py | 18 ++++++++++++++++ .../repos/builtin/packages/py-py2neo/package.py | 20 +++++++++++++++++ .../repos/builtin/packages/py-storm/package.py | 15 +++++++++++++ 9 files changed, 175 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-Genshi/package.py create mode 100644 var/spack/repos/builtin/packages/py-astroid/package.py create mode 100644 var/spack/repos/builtin/packages/py-jinja2/package.py create mode 100644 var/spack/repos/builtin/packages/py-logilab-common/package.py create mode 100644 var/spack/repos/builtin/packages/py-markupsafe/package.py create mode 100644 var/spack/repos/builtin/packages/py-mistune/package.py create mode 100644 var/spack/repos/builtin/packages/py-prettytable/package.py create mode 100644 var/spack/repos/builtin/packages/py-py2neo/package.py create mode 100644 var/spack/repos/builtin/packages/py-storm/package.py diff --git a/var/spack/repos/builtin/packages/py-Genshi/package.py b/var/spack/repos/builtin/packages/py-Genshi/package.py new file mode 100644 index 0000000000..6feb4a5bcf --- /dev/null +++ b/var/spack/repos/builtin/packages/py-Genshi/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyGenshi(Package): + """Python toolkit for generation of output for the web""" + homepage = "https://genshi.edgewall.org/" + url = "http://ftp.edgewall.com/pub/genshi/Genshi-0.7.tar.gz" + + version('0.7' , '54e64dd69da3ec961f86e686e0848a82') + version('0.6.1', '372c368c8931110b0a521fa6091742d7') + version('0.6' , '604e8b23b4697655d36a69c2d8ef7187') + + extends("python") + depends_on("py-setuptools") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-astroid/package.py b/var/spack/repos/builtin/packages/py-astroid/package.py new file mode 100644 index 0000000000..0c7971b27c --- /dev/null +++ b/var/spack/repos/builtin/packages/py-astroid/package.py @@ -0,0 +1,20 @@ +from spack import * + +class PyAstroid(Package): + homepage = "https://www.astroid.org/" + url = "https://github.com/PyCQA/astroid/archive/astroid-1.4.5.tar.gz" + + version('1.4.5', '7adfc55809908297ef430efe4ea20ac3') + version('1.4.4', '8ae6f63f6a2b260bb7f647dafccbc796') + version('1.4.3', '4647159de7d4d0c4b1de23ecbfb8e246') + version('1.4.2', '677f7965840f375af51b0e86403bee6a') + version('1.4.1', 'ed70bfed5e4b25be4292e7fe72da2c02') + + extends('python') + depends_on('py-logilab-common') + depends_on('py-setuptools') + depends_on('py-six') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + diff --git a/var/spack/repos/builtin/packages/py-jinja2/package.py b/var/spack/repos/builtin/packages/py-jinja2/package.py new file mode 100644 index 0000000000..bf1fcb74d2 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-jinja2/package.py @@ -0,0 +1,25 @@ +from spack import * + +class PyJinja2(Package): + """ + Jinja2 is a template engine written in pure Python. It provides + a Django inspired non-XML syntax but supports inline expressions + and an optional sandboxed environment. + """ + + homepage = "http://jinja.pocoo.org/" + url = "https://github.com/pallets/jinja/archive/2.8.tar.gz" + + version('2.8' , '4114200650d7630594e3bc70af23f59e') + version('2.7.3', '55b87bdc8e585b8b5b86734eefce2621') + version('2.7.2', '8e8f226809ae6363009b9296e30adf30') + version('2.7.1', '69b6675553c81b1087f95cae7f2179bb') + version('2.7' , 'ec70433f325051dcedacbb2465028a35') + + extends("python") + depends_on("py-setuptools") + depends_on("py-markupsafe") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + diff --git a/var/spack/repos/builtin/packages/py-logilab-common/package.py b/var/spack/repos/builtin/packages/py-logilab-common/package.py new file mode 100644 index 0000000000..2cbe7abcef --- /dev/null +++ b/var/spack/repos/builtin/packages/py-logilab-common/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyLogilabCommon(Package): + """Common modules used by Logilab projects""" + homepage = "https://www.logilab.org/project/logilab-common" + url = "https://pypi.python.org/packages/a7/31/1650d23e44794d46935d82b86e73454cc83b814cbe1365260ccce8a2f4c6/logilab-common-1.2.0.tar.gz" + + version('1.2.0', 'f7b51351b7bfe052746fa04c03253c0b') + + extends("python") + depends_on("py-setuptools") + depends_on("py-six") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + diff --git a/var/spack/repos/builtin/packages/py-markupsafe/package.py b/var/spack/repos/builtin/packages/py-markupsafe/package.py new file mode 100644 index 0000000000..78cbdd7a99 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-markupsafe/package.py @@ -0,0 +1,25 @@ +from spack import * + +class PyMarkupsafe(Package): + """ + MarkupSafe is a library for Python that implements a unicode + string that is aware of HTML escaping rules and can be used + to implement automatic string escaping. It is used by Jinja 2, + the Mako templating engine, the Pylons web framework and many more. + """ + + homepage = "http://www.pocoo.org/projects/markupsafe/" + url = "https://github.com/pallets/markupsafe/archive/0.23.tar.gz" + + version('0.23', '1a0dadc95169832367c9dcf142155cde') + version('0.22', '7a2ac7427b58def567628d06dc328396') + version('0.21', 'aebcd93ee05269773c8b80bb6c86fc2f') + version('0.20', '0c1fef97c8fd6a986d708f08d7f84a02') + version('0.19', '64b05361adb92c11839fc470e308c593') + + extends("python") + depends_on("py-setuptools") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + diff --git a/var/spack/repos/builtin/packages/py-mistune/package.py b/var/spack/repos/builtin/packages/py-mistune/package.py new file mode 100644 index 0000000000..4a9d5b5eb4 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-mistune/package.py @@ -0,0 +1,20 @@ +from spack import * + +class PyMistune(Package): + """ + Python markdown parser + """ + homepage = "http://mistune.readthedocs.org/en/latest/" + url = "https://github.com/lepture/mistune/archive/v0.7.1.tar.gz" + + version('0.7.1', '0d9c29700c670790c5b2471070d32ec2') + version('0.7' , '77750ae8b8d0d584894224a7e0c0523a') + version('0.6' , 'd4f3d4f28a69e715f82b591d5dacf9a6') + version('0.5.1', '1c6cfce28a4aa90cf125217cd6c6fe6c') + version('0.5' , '997736554f1f95eea78c66ae339b5722') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-prettytable/package.py b/var/spack/repos/builtin/packages/py-prettytable/package.py new file mode 100644 index 0000000000..2f7577d8c4 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-prettytable/package.py @@ -0,0 +1,18 @@ +from spack import * + +class PyPrettytable(Package): + """ + PrettyTable is a simple Python library designed to make + it quick and easy to represent tabular data in visually + appealing ASCII tables + """ + homepage = "https://code.google.com/archive/p/prettytable/" + url = "https://pypi.python.org/packages/e0/a1/36203205f77ccf98f3c6cf17cf068c972e6458d7e58509ca66da949ca347/prettytable-0.7.2.tar.gz" + + version('0.7.2', 'a6b80afeef286ce66733d54a0296b13b') + + extends("python") + depends_on("py-setuptools") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-py2neo/package.py b/var/spack/repos/builtin/packages/py-py2neo/package.py new file mode 100644 index 0000000000..c9cfdade2c --- /dev/null +++ b/var/spack/repos/builtin/packages/py-py2neo/package.py @@ -0,0 +1,20 @@ +from spack import * + +class PyPy2neo(Package): + """FIXME: put a proper description of your package here.""" + # FIXME: add a proper url for your package's homepage here. + homepage = "http://www.example.com" + url = "https://github.com/nigelsmall/py2neo/archive/py2neo-2.0.8.tar.gz" + + version('2.0.8', 'e3ec5172a9e006515ef4155688a05a55') + version('2.0.7', '4cfbc5b7dfd7757f3d2e324805faa639') + version('2.0.6', '53e4cdb1a95fbae501c66e541d5f4929') + version('2.0.5', '143b1f9c0aa22faf170c1b9f84c7343b') + version('2.0.4', 'b3f7efd3344dc3f66db4eda11e5899f7') + + depends_on("py-setuptools") + extends("python") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + diff --git a/var/spack/repos/builtin/packages/py-storm/package.py b/var/spack/repos/builtin/packages/py-storm/package.py new file mode 100644 index 0000000000..d504b231d4 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-storm/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyStorm(Package): + """Storm is an object-relational mapper (ORM) for Python""" + homepage = "https://storm.canonical.com/" + url = "https://launchpad.net/storm/trunk/0.20/+download/storm-0.20.tar.gz" + + version('0.20', '8628503141f0f06c0749d607ac09b9c7') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + -- cgit v1.2.3-70-g09d2 From 1a563c2b31315e741fc050c46d203c11e079ec5f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 10 May 2016 03:06:55 -0700 Subject: Ignore "multiple spaces before operator" error. --- flake8.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake8.ini b/flake8.ini index 15fc1b87b7..f8bff04355 100644 --- a/flake8.ini +++ b/flake8.ini @@ -1,3 +1,3 @@ [flake8] -ignore = W391,F403 +ignore = W391,F403,E221 max-line-length = 79 -- cgit v1.2.3-70-g09d2 From ad42579f0896a295cbbfb8daf7f02917ef35bc0e Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Tue, 10 May 2016 13:17:36 +0200 Subject: addressing coding style rules --- var/spack/repos/builtin/packages/py-Genshi/package.py | 8 +++++--- var/spack/repos/builtin/packages/py-astroid/package.py | 4 +++- var/spack/repos/builtin/packages/py-jinja2/package.py | 12 +++++++----- .../repos/builtin/packages/py-logilab-common/package.py | 4 +++- var/spack/repos/builtin/packages/py-markupsafe/package.py | 10 ++++++---- var/spack/repos/builtin/packages/py-mistune/package.py | 10 ++++++---- var/spack/repos/builtin/packages/py-prettytable/package.py | 8 +++++--- var/spack/repos/builtin/packages/py-py2neo/package.py | 4 +++- var/spack/repos/builtin/packages/py-storm/package.py | 5 +++-- 9 files changed, 41 insertions(+), 24 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-Genshi/package.py b/var/spack/repos/builtin/packages/py-Genshi/package.py index 6feb4a5bcf..7e0629200e 100644 --- a/var/spack/repos/builtin/packages/py-Genshi/package.py +++ b/var/spack/repos/builtin/packages/py-Genshi/package.py @@ -1,13 +1,15 @@ -from spack import * +from spack import version, extends, depends_on, python +from spack import Package + class PyGenshi(Package): """Python toolkit for generation of output for the web""" homepage = "https://genshi.edgewall.org/" url = "http://ftp.edgewall.com/pub/genshi/Genshi-0.7.tar.gz" - version('0.7' , '54e64dd69da3ec961f86e686e0848a82') + version('0.7', '54e64dd69da3ec961f86e686e0848a82') version('0.6.1', '372c368c8931110b0a521fa6091742d7') - version('0.6' , '604e8b23b4697655d36a69c2d8ef7187') + version('0.6', '604e8b23b4697655d36a69c2d8ef7187') extends("python") depends_on("py-setuptools") diff --git a/var/spack/repos/builtin/packages/py-astroid/package.py b/var/spack/repos/builtin/packages/py-astroid/package.py index 0c7971b27c..a8420d86c4 100644 --- a/var/spack/repos/builtin/packages/py-astroid/package.py +++ b/var/spack/repos/builtin/packages/py-astroid/package.py @@ -1,4 +1,6 @@ -from spack import * +from spack import depends_on, extends, python, version +from spack import Package + class PyAstroid(Package): homepage = "https://www.astroid.org/" diff --git a/var/spack/repos/builtin/packages/py-jinja2/package.py b/var/spack/repos/builtin/packages/py-jinja2/package.py index bf1fcb74d2..a745877786 100644 --- a/var/spack/repos/builtin/packages/py-jinja2/package.py +++ b/var/spack/repos/builtin/packages/py-jinja2/package.py @@ -1,25 +1,27 @@ -from spack import * +from spack import depends_on, extends, python, version +from spack import Package + class PyJinja2(Package): """ Jinja2 is a template engine written in pure Python. It provides - a Django inspired non-XML syntax but supports inline expressions + a Django inspired non-XML syntax but supports inline expressions and an optional sandboxed environment. """ homepage = "http://jinja.pocoo.org/" url = "https://github.com/pallets/jinja/archive/2.8.tar.gz" - version('2.8' , '4114200650d7630594e3bc70af23f59e') + version('2.8', '4114200650d7630594e3bc70af23f59e') version('2.7.3', '55b87bdc8e585b8b5b86734eefce2621') version('2.7.2', '8e8f226809ae6363009b9296e30adf30') version('2.7.1', '69b6675553c81b1087f95cae7f2179bb') - version('2.7' , 'ec70433f325051dcedacbb2465028a35') + version('2.7', 'ec70433f325051dcedacbb2465028a35') extends("python") depends_on("py-setuptools") depends_on("py-markupsafe") - + def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-logilab-common/package.py b/var/spack/repos/builtin/packages/py-logilab-common/package.py index 2cbe7abcef..5b3c6e26cc 100644 --- a/var/spack/repos/builtin/packages/py-logilab-common/package.py +++ b/var/spack/repos/builtin/packages/py-logilab-common/package.py @@ -1,4 +1,6 @@ -from spack import * +from spack import depends_on, extends, python, version +from spack import Package + class PyLogilabCommon(Package): """Common modules used by Logilab projects""" diff --git a/var/spack/repos/builtin/packages/py-markupsafe/package.py b/var/spack/repos/builtin/packages/py-markupsafe/package.py index 78cbdd7a99..ac55e5ae28 100644 --- a/var/spack/repos/builtin/packages/py-markupsafe/package.py +++ b/var/spack/repos/builtin/packages/py-markupsafe/package.py @@ -1,13 +1,15 @@ -from spack import * +from spack import depends_on, extends, python, version +from spack import Package + class PyMarkupsafe(Package): """ - MarkupSafe is a library for Python that implements a unicode - string that is aware of HTML escaping rules and can be used + MarkupSafe is a library for Python that implements a unicode + string that is aware of HTML escaping rules and can be used to implement automatic string escaping. It is used by Jinja 2, the Mako templating engine, the Pylons web framework and many more. """ - + homepage = "http://www.pocoo.org/projects/markupsafe/" url = "https://github.com/pallets/markupsafe/archive/0.23.tar.gz" diff --git a/var/spack/repos/builtin/packages/py-mistune/package.py b/var/spack/repos/builtin/packages/py-mistune/package.py index 4a9d5b5eb4..aacd921d7b 100644 --- a/var/spack/repos/builtin/packages/py-mistune/package.py +++ b/var/spack/repos/builtin/packages/py-mistune/package.py @@ -1,4 +1,6 @@ -from spack import * +from spack import depends_on, extends, python, version +from spack import Package + class PyMistune(Package): """ @@ -8,10 +10,10 @@ class PyMistune(Package): url = "https://github.com/lepture/mistune/archive/v0.7.1.tar.gz" version('0.7.1', '0d9c29700c670790c5b2471070d32ec2') - version('0.7' , '77750ae8b8d0d584894224a7e0c0523a') - version('0.6' , 'd4f3d4f28a69e715f82b591d5dacf9a6') + version('0.7', '77750ae8b8d0d584894224a7e0c0523a') + version('0.6', 'd4f3d4f28a69e715f82b591d5dacf9a6') version('0.5.1', '1c6cfce28a4aa90cf125217cd6c6fe6c') - version('0.5' , '997736554f1f95eea78c66ae339b5722') + version('0.5', '997736554f1f95eea78c66ae339b5722') extends('python') depends_on('py-setuptools') diff --git a/var/spack/repos/builtin/packages/py-prettytable/package.py b/var/spack/repos/builtin/packages/py-prettytable/package.py index 2f7577d8c4..1adfba3d1b 100644 --- a/var/spack/repos/builtin/packages/py-prettytable/package.py +++ b/var/spack/repos/builtin/packages/py-prettytable/package.py @@ -1,9 +1,11 @@ -from spack import * +from spack import depends_on, extends, python, version +from Spack import Package + class PyPrettytable(Package): """ PrettyTable is a simple Python library designed to make - it quick and easy to represent tabular data in visually + it quick and easy to represent tabular data in visually appealing ASCII tables """ homepage = "https://code.google.com/archive/p/prettytable/" @@ -13,6 +15,6 @@ class PyPrettytable(Package): extends("python") depends_on("py-setuptools") - + def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-py2neo/package.py b/var/spack/repos/builtin/packages/py-py2neo/package.py index c9cfdade2c..1697bdb0ee 100644 --- a/var/spack/repos/builtin/packages/py-py2neo/package.py +++ b/var/spack/repos/builtin/packages/py-py2neo/package.py @@ -1,4 +1,6 @@ -from spack import * +from spack import depends_on, extends, python, version +from spack import Package + class PyPy2neo(Package): """FIXME: put a proper description of your package here.""" diff --git a/var/spack/repos/builtin/packages/py-storm/package.py b/var/spack/repos/builtin/packages/py-storm/package.py index d504b231d4..1953f285c4 100644 --- a/var/spack/repos/builtin/packages/py-storm/package.py +++ b/var/spack/repos/builtin/packages/py-storm/package.py @@ -1,4 +1,6 @@ -from spack import * +from spack import depends_on, extends, python, version +from spack import Package + class PyStorm(Package): """Storm is an object-relational mapper (ORM) for Python""" @@ -12,4 +14,3 @@ class PyStorm(Package): def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) - -- cgit v1.2.3-70-g09d2 From f8f71b1c2c456210b64a90eb5b21484a8265cfa6 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 10 May 2016 13:37:03 +0200 Subject: modules : prefix_inspections moved to modules.yaml --- etc/spack/modules.yaml | 10 ++++++++++ lib/spack/spack/config.py | 8 ++++++++ lib/spack/spack/modules.py | 18 +++--------------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/etc/spack/modules.yaml b/etc/spack/modules.yaml index aa2a2c3fe2..8f8f88e908 100644 --- a/etc/spack/modules.yaml +++ b/etc/spack/modules.yaml @@ -5,4 +5,14 @@ # although users can override these settings in their ~/.spack/modules.yaml. # ------------------------------------------------------------------------- modules: + prefix_inspections: { + bin: ['PATH'], + man: ['MANPATH'], + lib: ['LIBRARY_PATH', 'LD_LIBRARY_PATH'], + lib64: ['LIBRARY_PATH', 'LD_LIBRARY_PATH'], + include: ['CPATH'], + lib/pkgconfig: ['PKGCONFIG'], + lib64/pkgconfig: ['PKGCONFIG'], + '': ['CMAKE_PREFIX_PATH'] + } enable: ['tcl', 'dotkit'] diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 64809462f9..d008a513e7 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -315,6 +315,14 @@ section_schemas = { 'default': {}, 'additionalProperties': False, 'properties': { + 'prefix_inspections': { + 'type': 'object', + 'patternProperties': { + r'\w[\w-]*': { # path to be inspected for existence (relative to prefix) + '$ref': '#/definitions/array_of_strings' + } + } + }, 'enable': { 'type': 'array', 'default': [], diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 57a4a2c754..f1c0bd87de 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -94,24 +94,12 @@ def inspect_path(prefix): """ env = EnvironmentModifications() # Inspect the prefix to check for the existence of common directories - prefix_inspections = { - 'bin': ('PATH',), - 'man': ('MANPATH',), - 'lib': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), - 'lib64': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), - 'include': ('CPATH',) - } - for attribute, variables in prefix_inspections.items(): - expected = getattr(prefix, attribute) + prefix_inspections = CONFIGURATION.get('prefix_inspections', {}) + for relative_path, variables in prefix_inspections.items(): + expected = join_path(prefix, relative_path) if os.path.isdir(expected): for variable in variables: env.prepend_path(variable, expected) - # PKGCONFIG - for expected in (join_path(prefix.lib, 'pkgconfig'), join_path(prefix.lib64, 'pkgconfig')): - if os.path.isdir(expected): - env.prepend_path('PKG_CONFIG_PATH', expected) - # CMake related variables - env.prepend_path('CMAKE_PREFIX_PATH', prefix) return env -- cgit v1.2.3-70-g09d2 From cd9cdd641062c5e4254feccd2f358cf8c7bbef01 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 10 May 2016 13:37:57 +0200 Subject: astyle: compile with spack_cxx --- var/spack/repos/builtin/packages/astyle/package.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/astyle/package.py b/var/spack/repos/builtin/packages/astyle/package.py index 5274fc018f..c8e8c3cbec 100644 --- a/var/spack/repos/builtin/packages/astyle/package.py +++ b/var/spack/repos/builtin/packages/astyle/package.py @@ -11,8 +11,13 @@ class Astyle(Package): def install(self, spec, prefix): with working_dir('src'): + # unfortunately we need to edit the makefile in place to set compiler: + make_file = join_path(self.stage.source_path,'build','gcc','Makefile') + filter_file(r'^CXX\s*=.*', 'CXX=%s' % spack_cxx, make_file) + make('-f', - join_path(self.stage.source_path,'build','clang','Makefile'), + make_file, parallel=False) + mkdirp(self.prefix.bin) install(join_path(self.stage.source_path, 'src','bin','astyle'), self.prefix.bin) -- cgit v1.2.3-70-g09d2 From f5fb77157360a29215486fbaa70dd4995017e5ff Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Tue, 10 May 2016 14:52:14 +0200 Subject: fixing imports; configure commands aren't available at import time yet --- var/spack/repos/builtin/packages/py-Genshi/package.py | 2 +- var/spack/repos/builtin/packages/py-astroid/package.py | 2 +- var/spack/repos/builtin/packages/py-jinja2/package.py | 2 +- var/spack/repos/builtin/packages/py-logilab-common/package.py | 2 +- var/spack/repos/builtin/packages/py-markupsafe/package.py | 2 +- var/spack/repos/builtin/packages/py-mistune/package.py | 2 +- var/spack/repos/builtin/packages/py-prettytable/package.py | 4 ++-- var/spack/repos/builtin/packages/py-py2neo/package.py | 2 +- var/spack/repos/builtin/packages/py-storm/package.py | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-Genshi/package.py b/var/spack/repos/builtin/packages/py-Genshi/package.py index 7e0629200e..d485c89429 100644 --- a/var/spack/repos/builtin/packages/py-Genshi/package.py +++ b/var/spack/repos/builtin/packages/py-Genshi/package.py @@ -1,4 +1,4 @@ -from spack import version, extends, depends_on, python +from spack import version, extends, depends_on from spack import Package diff --git a/var/spack/repos/builtin/packages/py-astroid/package.py b/var/spack/repos/builtin/packages/py-astroid/package.py index a8420d86c4..1ecb5eecee 100644 --- a/var/spack/repos/builtin/packages/py-astroid/package.py +++ b/var/spack/repos/builtin/packages/py-astroid/package.py @@ -1,4 +1,4 @@ -from spack import depends_on, extends, python, version +from spack import depends_on, extends, version from spack import Package diff --git a/var/spack/repos/builtin/packages/py-jinja2/package.py b/var/spack/repos/builtin/packages/py-jinja2/package.py index a745877786..5d92cdd49a 100644 --- a/var/spack/repos/builtin/packages/py-jinja2/package.py +++ b/var/spack/repos/builtin/packages/py-jinja2/package.py @@ -1,4 +1,4 @@ -from spack import depends_on, extends, python, version +from spack import depends_on, extends, version from spack import Package diff --git a/var/spack/repos/builtin/packages/py-logilab-common/package.py b/var/spack/repos/builtin/packages/py-logilab-common/package.py index 5b3c6e26cc..a47c4ac0ec 100644 --- a/var/spack/repos/builtin/packages/py-logilab-common/package.py +++ b/var/spack/repos/builtin/packages/py-logilab-common/package.py @@ -1,4 +1,4 @@ -from spack import depends_on, extends, python, version +from spack import depends_on, extends, version from spack import Package diff --git a/var/spack/repos/builtin/packages/py-markupsafe/package.py b/var/spack/repos/builtin/packages/py-markupsafe/package.py index ac55e5ae28..0a0c3a724a 100644 --- a/var/spack/repos/builtin/packages/py-markupsafe/package.py +++ b/var/spack/repos/builtin/packages/py-markupsafe/package.py @@ -1,4 +1,4 @@ -from spack import depends_on, extends, python, version +from spack import depends_on, extends, version from spack import Package diff --git a/var/spack/repos/builtin/packages/py-mistune/package.py b/var/spack/repos/builtin/packages/py-mistune/package.py index aacd921d7b..44a114b173 100644 --- a/var/spack/repos/builtin/packages/py-mistune/package.py +++ b/var/spack/repos/builtin/packages/py-mistune/package.py @@ -1,4 +1,4 @@ -from spack import depends_on, extends, python, version +from spack import depends_on, extends, version from spack import Package diff --git a/var/spack/repos/builtin/packages/py-prettytable/package.py b/var/spack/repos/builtin/packages/py-prettytable/package.py index 1adfba3d1b..27fab7c046 100644 --- a/var/spack/repos/builtin/packages/py-prettytable/package.py +++ b/var/spack/repos/builtin/packages/py-prettytable/package.py @@ -1,5 +1,5 @@ -from spack import depends_on, extends, python, version -from Spack import Package +from spack import depends_on, extends, version +from spack import Package class PyPrettytable(Package): diff --git a/var/spack/repos/builtin/packages/py-py2neo/package.py b/var/spack/repos/builtin/packages/py-py2neo/package.py index 1697bdb0ee..d30b823a87 100644 --- a/var/spack/repos/builtin/packages/py-py2neo/package.py +++ b/var/spack/repos/builtin/packages/py-py2neo/package.py @@ -1,4 +1,4 @@ -from spack import depends_on, extends, python, version +from spack import depends_on, extends, version from spack import Package diff --git a/var/spack/repos/builtin/packages/py-storm/package.py b/var/spack/repos/builtin/packages/py-storm/package.py index 1953f285c4..abc121d30b 100644 --- a/var/spack/repos/builtin/packages/py-storm/package.py +++ b/var/spack/repos/builtin/packages/py-storm/package.py @@ -1,4 +1,4 @@ -from spack import depends_on, extends, python, version +from spack import depends_on, extends, version from spack import Package -- cgit v1.2.3-70-g09d2 From 0b7c673205049565a95707fd55d2c086dd601b35 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 10 May 2016 15:48:37 +0200 Subject: modules : changed syntax for environment modifications --- lib/spack/spack/config.py | 14 +++++++++++--- lib/spack/spack/modules.py | 22 +++++++++++++++++----- lib/spack/spack/test/modules.py | 3 +-- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index d008a513e7..6ddf07776b 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -251,6 +251,14 @@ section_schemas = { 'type': 'string' } }, + 'dictionary_of_strings': { + 'type': 'object', + 'patternProperties': { + r'\w[\w-]*': { # key + 'type': 'string' + } + } + }, 'dependency_selection': { 'type': 'string', 'enum': ['none', 'direct', 'all'] @@ -282,10 +290,10 @@ section_schemas = { 'default': {}, 'additionalProperties': False, 'properties': { - 'set': {'$ref': '#/definitions/array_of_strings'}, + 'set': {'$ref': '#/definitions/dictionary_of_strings'}, 'unset': {'$ref': '#/definitions/array_of_strings'}, - 'prepend_path': {'$ref': '#/definitions/array_of_strings'}, - 'append_path': {'$ref': '#/definitions/array_of_strings'} + 'prepend_path': {'$ref': '#/definitions/dictionary_of_strings'}, + 'append_path': {'$ref': '#/definitions/dictionary_of_strings'} } } } diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index f1c0bd87de..19bd1993a7 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -185,14 +185,26 @@ def parse_config_options(module_generator): # Environment modifications environment_actions = module_file_actions.pop('environment', {}) env = EnvironmentModifications() + + def process_arglist(arglist): + if method == 'unset': + for x in arglist: + yield (x,) + else: + for x in arglist.iteritems(): + yield x + for method, arglist in environment_actions.items(): - for item in arglist: - if method == 'unset': - args = [item] - else: - args = item.split(',') + for args in process_arglist(arglist): getattr(env, method)(*args) + # for item in arglist: + # if method == 'unset': + # args = [item] + # else: + # args = item.split(',') + # getattr(env, method)(*args) + return module_file_actions, env diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 704700417b..b8b0d6fc6a 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -47,7 +47,7 @@ configuration_alter_environment = { 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']} }, '=x86-linux': { - 'environment': {'set': ['FOO,foo'], 'unset': ['BAR']} + 'environment': {'set': {'FOO': 'foo'}, 'unset': ['BAR']} } } } @@ -99,7 +99,6 @@ class TclTests(MockPackagesTest): spec = spack.spec.Spec('mpich@3.0.4=x86-linux') content = self.get_modulefile_content(spec) self.assertTrue('module-whatis "mpich @3.0.4"' in content ) - self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 1) def test_autoload(self): spack.modules.CONFIGURATION = configuration_autoload_direct -- cgit v1.2.3-70-g09d2 From c3f3f26632ff0450683c99bebcce3acfad8a5ebc Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 10 May 2016 16:16:50 +0200 Subject: modules : added warning if a user tries to add prerequisite with dotkit --- lib/spack/spack/modules.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 19bd1993a7..ffed469b20 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -382,11 +382,11 @@ class EnvModule(object): return tuple() def autoload(self, spec): - m = TclModule(spec) + m = type(self)(spec) return self.autoload_format.format(module_file=m.use_name) def prerequisite(self, spec): - m = TclModule(spec) + m = type(self)(spec) return self.prerequisite_format.format(module_file=m.use_name) def process_environment_command(self, env): @@ -449,6 +449,11 @@ class Dotkit(EnvModule): header += '#h %s\n' % line return header + def prerequisite(self, spec): + tty.warn('prerequisites: not supported by dotkit module files') + tty.warn('\tYou may want to check ~/.spack/modules.yaml') + return '' + class TclModule(EnvModule): name = 'tcl' -- cgit v1.2.3-70-g09d2 From 71e49e289a849b8aaa4f0d9a195d07569051ca88 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 10 May 2016 17:19:22 +0200 Subject: flake8 : fixed all issues? --- lib/spack/spack/cmd/module.py | 16 +-- lib/spack/spack/config.py | 204 ++++++++++++++++++++++++--------------- lib/spack/spack/environment.py | 51 ++++++---- lib/spack/spack/modules.py | 202 ++++++++++++++++++++++---------------- lib/spack/spack/test/__init__.py | 69 ++++--------- lib/spack/spack/test/modules.py | 42 +++++--- 6 files changed, 330 insertions(+), 254 deletions(-) diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index f996f4eb84..cfe59c8d98 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -32,18 +32,21 @@ from llnl.util.filesystem import mkdirp from spack.modules import module_types from spack.util.string import * -description ="Manipulate modules and dotkits." +description = "Manipulate modules and dotkits." def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='module_command') - refresh_parser = sp.add_parser('refresh', help='Regenerate all module files.') + sp.add_parser('refresh', help='Regenerate all module files.') find_parser = sp.add_parser('find', help='Find module files for packages.') - find_parser.add_argument( - 'module_type', help="Type of module to find file for. [" + '|'.join(module_types) + "]") - find_parser.add_argument('spec', nargs='+', help='spec to find a module file for.') + find_parser.add_argument('module_type', + help="Type of module to find file for. [" + + '|'.join(module_types) + "]") + find_parser.add_argument('spec', + nargs='+', + help='spec to find a module file for.') def module_find(mtype, spec_array): @@ -53,7 +56,8 @@ def module_find(mtype, spec_array): should type to use that package's module. """ if mtype not in module_types: - tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types))) + tty.die("Invalid module type: '%s'. Options are %s" % + (mtype, comma_or(module_types))) specs = spack.cmd.parse_specs(spec_array) if len(specs) > 1: diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 6ddf07776b..684a420b3b 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -134,8 +134,6 @@ from yaml.error import MarkedYAMLError # Hacked yaml for configuration files preserves line numbers. import spack.util.spack_yaml as syaml - - """Dict from section names -> schema for that section.""" section_schemas = { 'compilers': { @@ -149,25 +147,31 @@ section_schemas = { 'default': {}, 'additionalProperties': False, 'patternProperties': { - r'\w[\w-]*': { # architecture + r'\w[\w-]*': { # architecture 'type': 'object', 'additionalProperties': False, 'patternProperties': { - r'\w[\w-]*@\w[\w-]*': { # compiler spec + r'\w[\w-]*@\w[\w-]*': { # compiler spec 'type': 'object', 'additionalProperties': False, 'required': ['cc', 'cxx', 'f77', 'fc'], 'properties': { - 'cc': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]}, - 'cxx': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]}, - 'f77': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]}, - 'fc': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]}, - },},},},},},},}, - + 'cc': {'anyOf': [{'type': 'string'}, + {'type': 'null'}]}, + 'cxx': {'anyOf': [{'type': 'string'}, + {'type': 'null'}]}, + 'f77': {'anyOf': [{'type': 'string'}, + {'type': 'null'}]}, + 'fc': {'anyOf': [{'type': 'string'}, + {'type': 'null'}]}, + }, + }, + }, + }, + }, + }, + }, + }, 'mirrors': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack mirror configuration file schema', @@ -180,8 +184,12 @@ section_schemas = { 'additionalProperties': False, 'patternProperties': { r'\w[\w-]*': { - 'type': 'string'},},},},}, - + 'type': 'string' + }, + }, + }, + }, + }, 'repos': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack repository configuration file schema', @@ -192,8 +200,11 @@ section_schemas = { 'type': 'array', 'default': [], 'items': { - 'type': 'string'},},},}, - + 'type': 'string' + }, + }, + }, + }, 'packages': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack package configuration file schema', @@ -205,39 +216,48 @@ section_schemas = { 'default': {}, 'additionalProperties': False, 'patternProperties': { - r'\w[\w-]*': { # package name + r'\w[\w-]*': { # package name 'type': 'object', 'default': {}, 'additionalProperties': False, 'properties': { 'version': { - 'type' : 'array', - 'default' : [], - 'items' : { 'anyOf' : [ { 'type' : 'string' }, - { 'type' : 'number'}]}}, #version strings + 'type': 'array', + 'default': [], + 'items': {'anyOf': [{'type': 'string'}, + {'type': 'number'}]} + }, # version strings 'compiler': { - 'type' : 'array', - 'default' : [], - 'items' : { 'type' : 'string' } }, #compiler specs + 'type': 'array', + 'default': [], + 'items': {'type': 'string'} + }, # compiler specs 'buildable': { - 'type': 'boolean', + 'type': 'boolean', 'default': True, - }, + }, 'providers': { - 'type': 'object', + 'type': 'object', 'default': {}, 'additionalProperties': False, 'patternProperties': { r'\w[\w-]*': { - 'type' : 'array', - 'default' : [], - 'items' : { 'type' : 'string' },},},}, + 'type': 'array', + 'default': [], + 'items': {'type': 'string'}, + }, + }, + }, 'paths': { - 'type' : 'object', - 'default' : {}, + 'type': 'object', + 'default': {}, } - },},},},},}, - + }, + }, + }, + }, + }, + }, 'modules': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack module file configuration file schema', @@ -283,17 +303,22 @@ section_schemas = { } }, 'autoload': {'$ref': '#/definitions/dependency_selection'}, - 'prerequisites': {'$ref': '#/definitions/dependency_selection'}, + 'prerequisites': + {'$ref': '#/definitions/dependency_selection'}, 'conflict': {'$ref': '#/definitions/array_of_strings'}, 'environment': { 'type': 'object', 'default': {}, 'additionalProperties': False, 'properties': { - 'set': {'$ref': '#/definitions/dictionary_of_strings'}, - 'unset': {'$ref': '#/definitions/array_of_strings'}, - 'prepend_path': {'$ref': '#/definitions/dictionary_of_strings'}, - 'append_path': {'$ref': '#/definitions/dictionary_of_strings'} + 'set': + {'$ref': '#/definitions/dictionary_of_strings'}, + 'unset': + {'$ref': '#/definitions/array_of_strings'}, + 'prepend_path': + {'$ref': '#/definitions/dictionary_of_strings'}, + 'append_path': + {'$ref': '#/definitions/dictionary_of_strings'} } } } @@ -304,15 +329,20 @@ section_schemas = { 'anyOf': [ { 'properties': { - 'whitelist': {'$ref': '#/definitions/array_of_strings'}, - 'blacklist': {'$ref': '#/definitions/array_of_strings'}, + 'whitelist': + {'$ref': '#/definitions/array_of_strings'}, + 'blacklist': + {'$ref': '#/definitions/array_of_strings'}, 'naming_scheme': { - 'type': 'string' # Can we be more specific here? + 'type': + 'string' # Can we be more specific here? } } }, { - 'patternProperties': {r'\w[\w-]*': {'$ref': '#/definitions/module_file_configuration'}} + 'patternProperties': + {r'\w[\w-]*': + {'$ref': '#/definitions/module_file_configuration'}} } ] } @@ -326,7 +356,8 @@ section_schemas = { 'prefix_inspections': { 'type': 'object', 'patternProperties': { - r'\w[\w-]*': { # path to be inspected for existence (relative to prefix) + r'\w[\w-]*': + { # path to be inspected (relative to prefix) '$ref': '#/definitions/array_of_strings' } } @@ -341,13 +372,15 @@ section_schemas = { }, 'tcl': { 'allOf': [ - {'$ref': '#/definitions/module_type_configuration'}, # Base configuration + {'$ref': '#/definitions/module_type_configuration' + }, # Base configuration {} # Specific tcl extensions ] }, 'dotkit': { 'allOf': [ - {'$ref': '#/definitions/module_type_configuration'}, # Base configuration + {'$ref': '#/definitions/module_type_configuration' + }, # Base configuration {} # Specific dotkit extensions ] }, @@ -356,7 +389,6 @@ section_schemas = { }, }, } - """OrderedDict of config scopes keyed by name. Later scopes will override earlier scopes. """ @@ -366,12 +398,13 @@ config_scopes = OrderedDict() def validate_section_name(section): """Raise a ValueError if the section is not a valid section.""" if section not in section_schemas: - raise ValueError("Invalid config section: '%s'. Options are %s" - % (section, section_schemas)) + raise ValueError("Invalid config section: '%s'. Options are %s" % + (section, section_schemas)) def extend_with_default(validator_class): - """Add support for the 'default' attribute for properties and patternProperties. + """Add support for the 'default' attribute for + properties and patternProperties jsonschema does not handle this out of the box -- it only validates. This allows us to set default values for configs @@ -380,13 +413,15 @@ def extend_with_default(validator_class): """ validate_properties = validator_class.VALIDATORS["properties"] - validate_pattern_properties = validator_class.VALIDATORS["patternProperties"] + validate_pattern_properties = validator_class.VALIDATORS[ + "patternProperties"] def set_defaults(validator, properties, instance, schema): for property, subschema in properties.iteritems(): if "default" in subschema: instance.setdefault(property, subschema["default"]) - for err in validate_properties(validator, properties, instance, schema): + for err in validate_properties(validator, properties, instance, + schema): yield err def set_pp_defaults(validator, properties, instance, schema): @@ -397,17 +432,19 @@ def extend_with_default(validator_class): if re.match(property, key) and val is None: instance[key] = subschema["default"] - for err in validate_pattern_properties(validator, properties, instance, schema): + for err in validate_pattern_properties(validator, properties, instance, + schema): yield err return validators.extend(validator_class, { - "properties" : set_defaults, - "patternProperties" : set_pp_defaults + "properties": set_defaults, + "patternProperties": set_pp_defaults }) DefaultSettingValidator = extend_with_default(Draft4Validator) + def validate_section(data, schema): """Validate data read in from a Spack YAML file. @@ -429,9 +466,9 @@ class ConfigScope(object): """ def __init__(self, name, path): - self.name = name # scope name. - self.path = path # path to directory containing configs. - self.sections = {} # sections read from config files. + self.name = name # scope name. + self.path = path # path to directory containing configs. + self.sections = {} # sections read from config files. # Register in a dict of all ConfigScopes # TODO: make this cleaner. Mocking up for testing is brittle. @@ -442,16 +479,14 @@ class ConfigScope(object): validate_section_name(section) return os.path.join(self.path, "%s.yaml" % section) - def get_section(self, section): - if not section in self.sections: - path = self.get_section_filename(section) + if section not in self.sections: + path = self.get_section_filename(section) schema = section_schemas[section] - data = _read_config_file(path, schema) + data = _read_config_file(path, schema) self.sections[section] = data return self.sections[section] - def write_section(self, section): filename = self.get_section_filename(section) data = self.get_section(section) @@ -463,8 +498,8 @@ class ConfigScope(object): except jsonschema.ValidationError as e: raise ConfigSanityError(e, data) except (yaml.YAMLError, IOError) as e: - raise ConfigFileError("Error writing to config file: '%s'" % str(e)) - + raise ConfigFileError("Error writing to config file: '%s'" % + str(e)) def clear(self): """Empty cached config information.""" @@ -496,8 +531,8 @@ def validate_scope(scope): return config_scopes[scope] else: - raise ValueError("Invalid config scope: '%s'. Must be one of %s" - % (scope, config_scopes.keys())) + raise ValueError("Invalid config scope: '%s'. Must be one of %s" % + (scope, config_scopes.keys())) def _read_config_file(filename, schema): @@ -523,12 +558,12 @@ def _read_config_file(filename, schema): return data except MarkedYAMLError as e: - raise ConfigFileError( - "Error parsing yaml%s: %s" % (str(e.context_mark), e.problem)) + raise ConfigFileError("Error parsing yaml%s: %s" % + (str(e.context_mark), e.problem)) except IOError as e: - raise ConfigFileError( - "Error reading configuration file %s: %s" % (filename, str(e))) + raise ConfigFileError("Error reading configuration file %s: %s" % + (filename, str(e))) def clear_config_caches(): @@ -551,6 +586,7 @@ def _merge_yaml(dest, source): parent instead of merging. """ + def they_are(t): return isinstance(dest, t) and isinstance(source, t) @@ -571,7 +607,7 @@ def _merge_yaml(dest, source): # Source dict is merged into dest. elif they_are(dict): for sk, sv in source.iteritems(): - if not sk in dest: + if sk not in dest: dest[sk] = copy.copy(sv) else: dest[sk] = _merge_yaml(dest[sk], source[sk]) @@ -653,7 +689,7 @@ def print_section(section): data = syaml.syaml_dict() data[section] = get_config(section) syaml.dump(data, stream=sys.stdout, default_flow_style=False) - except (yaml.YAMLError, IOError) as e: + except (yaml.YAMLError, IOError): raise ConfigError("Error reading configuration: %s" % section) @@ -683,15 +719,22 @@ def is_spec_buildable(spec): """Return true if the spec pkgspec is configured as buildable""" allpkgs = get_config('packages') name = spec.name - if not spec.name in allpkgs: + if name not in allpkgs: return True - if not 'buildable' in allpkgs[spec.name]: + if 'buildable' not in allpkgs[name]: return True return allpkgs[spec.name]['buildable'] -class ConfigError(SpackError): pass -class ConfigFileError(ConfigError): pass +class ConfigError(SpackError): + + pass + + +class ConfigFileError(ConfigError): + + pass + def get_path(path, data): if path: @@ -699,8 +742,10 @@ def get_path(path, data): else: return data + class ConfigFormatError(ConfigError): """Raised when a configuration format does not match its schema.""" + def __init__(self, validation_error, data): # Try to get line number from erroneous instance and its parent instance_mark = getattr(validation_error.instance, '_start_mark', None) @@ -733,5 +778,6 @@ class ConfigFormatError(ConfigError): message = '%s: %s' % (location, validation_error.message) super(ConfigError, self).__init__(message) + class ConfigSanityError(ConfigFormatError): """Same as ConfigFormatError, raised when config is written by Spack.""" diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 92ab4e6bea..9748b4033a 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -26,7 +26,8 @@ class SetEnv(NameValueModifier): class UnsetEnv(NameModifier): def execute(self): - os.environ.pop(self.name, None) # Avoid throwing if the variable was not set + # Avoid throwing if the variable was not set + os.environ.pop(self.name, None) class SetPath(NameValueModifier): @@ -55,7 +56,9 @@ class RemovePath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split(':') if environment_value else [] - directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.value)] + directories = [os.path.normpath(x) + for x in directories + if x != os.path.normpath(self.value)] os.environ[self.name] = ':'.join(directories) @@ -63,7 +66,8 @@ class EnvironmentModifications(object): """ Keeps track of requests to modify the current environment. - Each call to a method to modify the environment stores the extra information on the caller in the request: + Each call to a method to modify the environment stores the extra + information on the caller in the request: - 'filename' : filename of the module where the caller is defined - 'lineno': line number where the request occurred - 'context' : line of code that issued the request that failed @@ -71,10 +75,10 @@ class EnvironmentModifications(object): def __init__(self, other=None): """ - Initializes a new instance, copying commands from other if it is not None + Initializes a new instance, copying commands from other if not None Args: - other: another instance of EnvironmentModifications from which (optional) + other: another instance of EnvironmentModifications """ self.env_modifications = [] if other is not None: @@ -93,7 +97,7 @@ class EnvironmentModifications(object): @staticmethod def _check_other(other): if not isinstance(other, EnvironmentModifications): - raise TypeError('other must be an instance of EnvironmentModifications') + raise TypeError('not an instance of EnvironmentModifications') def _get_outside_caller_attributes(self): stack = inspect.stack() @@ -101,12 +105,10 @@ class EnvironmentModifications(object): _, filename, lineno, _, context, index = stack[2] context = context[index].strip() except Exception: - filename, lineno, context = 'unknown file', 'unknown line', 'unknown context' - args = { - 'filename': filename, - 'lineno': lineno, - 'context': context - } + filename = 'unknown file' + lineno = 'unknown line' + context = 'unknown context' + args = {'filename': filename, 'lineno': lineno, 'context': context} return args def set(self, name, value, **kwargs): @@ -170,7 +172,7 @@ class EnvironmentModifications(object): def remove_path(self, name, path, **kwargs): """ - Stores in the current object a request to remove a path from a path list + Stores in the current object a request to remove a path from a list Args: name: name of the path list in the environment @@ -185,7 +187,8 @@ class EnvironmentModifications(object): Returns a dict of the modifications grouped by variable name Returns: - dict mapping the environment variable name to the modifications to be done on it + dict mapping the environment variable name to the modifications + to be done on it """ modifications = collections.defaultdict(list) for item in self: @@ -203,7 +206,8 @@ class EnvironmentModifications(object): Applies the modifications and clears the list """ modifications = self.group_by_name() - # Apply the modifications to the environment variables one variable at a time + # Apply the modifications to the environment variables one variable + # at a time for name, actions in sorted(modifications.items()): for x in actions: x.execute() @@ -224,13 +228,17 @@ def concatenate_paths(paths): def set_or_unset_not_first(variable, changes, errstream): """ - Check if we are going to set or unset something after other modifications have already been requested + Check if we are going to set or unset something after other modifications + have already been requested """ - indexes = [ii for ii, item in enumerate(changes) if ii != 0 and type(item) in [SetEnv, UnsetEnv]] + indexes = [ii + for ii, item in enumerate(changes) + if ii != 0 and type(item) in [SetEnv, UnsetEnv]] if indexes: good = '\t \t{context} at {filename}:{lineno}' nogood = '\t--->\t{context} at {filename}:{lineno}' - errstream('Suspicious requests to set or unset the variable \'{var}\' found'.format(var=variable)) + message = 'Suspicious requests to set or unset the variable \'{var}\' found' # NOQA: ignore=E501 + errstream(message.format(var=variable)) for ii, item in enumerate(changes): print_format = nogood if ii in indexes else good errstream(print_format.format(**item.args)) @@ -238,8 +246,8 @@ def set_or_unset_not_first(variable, changes, errstream): def validate(env, errstream): """ - Validates the environment modifications to check for the presence of suspicious patterns. Prompts a warning for - everything that was found + Validates the environment modifications to check for the presence of + suspicious patterns. Prompts a warning for everything that was found Current checks: - set or unset variables after other changes on the same variable @@ -254,7 +262,8 @@ def validate(env, errstream): def filter_environment_blacklist(env, variables): """ - Generator that filters out any change to environment variables present in the input list + Generator that filters out any change to environment variables present in + the input list Args: env: list of environment modifications diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index ffed469b20..0dc6f06f55 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -23,36 +23,34 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## """ -This module contains code for creating environment modules, which can include dotkits, tcl modules, lmod, and others. +This module contains code for creating environment modules, which can include +dotkits, tcl modules, lmod, and others. -The various types of modules are installed by post-install hooks and removed after an uninstall by post-uninstall hooks. -This class consolidates the logic for creating an abstract description of the information that module systems need. -Currently that includes a number of directories to be appended to paths in the user's environment: +The various types of modules are installed by post-install hooks and removed +after an uninstall by post-uninstall hooks. This class consolidates the logic +for creating an abstract description of the information that module systems +need. - * /bin directories to be appended to PATH - * /lib* directories for LD_LIBRARY_PATH - * /include directories for CPATH - * /man* and /share/man* directories for MANPATH - * the package prefix for CMAKE_PREFIX_PATH +This module also includes logic for coming up with unique names for the module +files so that they can be found by the various shell-support files in +$SPACK/share/spack/setup-env.*. -This module also includes logic for coming up with unique names for the module files so that they can be found by the -various shell-support files in $SPACK/share/spack/setup-env.*. - -Each hook in hooks/ implements the logic for writing its specific type of module file. +Each hook implements the logic for writing its specific type of module. """ import copy import datetime import os import os.path import re -import textwrap import string +import textwrap import llnl.util.tty as tty import spack import spack.config from llnl.util.filesystem import join_path, mkdirp -from spack.build_environment import parent_class_modules, set_module_variables_for_package +from spack.build_environment import parent_class_modules +from spack.build_environment import set_module_variables_for_package from spack.environment import * __all__ = ['EnvModule', 'Dotkit', 'TclModule'] @@ -67,30 +65,26 @@ def print_help(): """ For use by commands to tell user how to activate shell support. """ - tty.msg("This command requires spack's shell integration.", - "", + tty.msg("This command requires spack's shell integration.", "", "To initialize spack's shell commands, you must run one of", "the commands below. Choose the right command for your shell.", - "", - "For bash and zsh:", - " . %s/setup-env.sh" % spack.share_path, - "", - "For csh and tcsh:", - " setenv SPACK_ROOT %s" % spack.prefix, - " source %s/setup-env.csh" % spack.share_path, - "") + "", "For bash and zsh:", + " . %s/setup-env.sh" % spack.share_path, "", + "For csh and tcsh:", " setenv SPACK_ROOT %s" % spack.prefix, + " source %s/setup-env.csh" % spack.share_path, "") def inspect_path(prefix): """ - Inspects the prefix of an installation to search for common layouts. Issues a request to modify the environment - accordingly when an item is found. + Inspects the prefix of an installation to search for common layouts. + Issues a request to modify the environment when an item is found. Args: prefix: prefix of the installation Returns: - instance of EnvironmentModifications containing the requested modifications + instance of EnvironmentModifications containing the requested + modifications """ env = EnvironmentModifications() # Inspect the prefix to check for the existence of common directories @@ -105,18 +99,22 @@ def inspect_path(prefix): def dependencies(spec, request='all'): """ - Returns the list of dependent specs for a given spec, according to the given request + Returns the list of dependent specs for a given spec, according to the + given request Args: spec: target spec request: either 'none', 'direct' or 'all' Returns: - empty list if 'none', direct dependency list if 'direct', all dependencies if 'all' + empty list if 'none', direct dependency list if 'direct', all + dependencies if 'all' """ if request not in ('none', 'direct', 'all'): - raise tty.error("Wrong value for argument 'request' : should be one of ('none', 'direct', 'all') " - " [current value is '%s']" % request) + message = "Wrong value for argument 'request' : " + message += "should be one of ('none', 'direct', 'all')" + message += " [current value is '{0}']" + raise tty.error(message.format(request)) if request == 'none': return [] @@ -124,12 +122,19 @@ def dependencies(spec, request='all'): if request == 'direct': return [xx for _, xx in spec.dependencies.items()] - # FIXME : during module file creation nodes seem to be visited multiple times even if cover='nodes' - # FIXME : is given. This work around permits to get a unique list of spec anyhow. - # FIXME : Possibly we miss a merge step among nodes that refer to the same package. + # FIXME : during module file creation nodes seem to be visited + # FIXME : multiple times even if cover='nodes' is given. This work around + # FIXME : permits to get a unique list of spec anyhow. Maybe we miss a + # FIXME : merge step among nodes that refer to the same package? seen = set() seen_add = seen.add - l = [xx for xx in sorted(spec.traverse(order='post', depth=True, cover='nodes', root=False), reverse=True)] + l = [xx + for xx in sorted( + spec.traverse(order='post', + depth=True, + cover='nodes', + root=False), + reverse=True)] return [xx for ii, xx in l if not (xx in seen or seen_add(xx))] @@ -146,7 +151,8 @@ def update_dictionary_extending_lists(target, update): def parse_config_options(module_generator): """ - Parse the configuration file and returns a bunch of items that will be needed during module file generation + Parse the configuration file and returns a bunch of items that will be + needed during module file generation Args: module_generator: module generator for a given spec @@ -154,11 +160,14 @@ def parse_config_options(module_generator): Returns: autoloads: list of specs to be autoloaded prerequisites: list of specs to be marked as prerequisite - filters: list of environment variables whose modification is blacklisted in module files - env: list of custom environment modifications to be applied in the module file + filters: list of environment variables whose modification is + blacklisted in module files + env: list of custom environment modifications to be applied in the + module file """ # Get the configuration for this kind of generator - module_configuration = copy.deepcopy(CONFIGURATION.get(module_generator.name, {})) + module_configuration = copy.deepcopy(CONFIGURATION.get( + module_generator.name, {})) ##### # Merge all the rules @@ -179,9 +188,12 @@ def parse_config_options(module_generator): ##### # Automatic loading loads - module_file_actions['autoload'] = dependencies(module_generator.spec, module_file_actions.get('autoload', 'none')) + module_file_actions['autoload'] = dependencies( + module_generator.spec, module_file_actions.get('autoload', 'none')) # Prerequisites - module_file_actions['prerequisites'] = dependencies(module_generator.spec, module_file_actions.get('prerequisites', 'none')) + module_file_actions['prerequisites'] = dependencies( + module_generator.spec, module_file_actions.get('prerequisites', + 'none')) # Environment modifications environment_actions = module_file_actions.pop('environment', {}) env = EnvironmentModifications() @@ -189,7 +201,7 @@ def parse_config_options(module_generator): def process_arglist(arglist): if method == 'unset': for x in arglist: - yield (x,) + yield (x, ) else: for x in arglist.iteritems(): yield x @@ -198,19 +210,20 @@ def parse_config_options(module_generator): for args in process_arglist(arglist): getattr(env, method)(*args) - # for item in arglist: - # if method == 'unset': - # args = [item] - # else: - # args = item.split(',') - # getattr(env, method)(*args) + # for item in arglist: + # if method == 'unset': + # args = [item] + # else: + # args = item.split(',') + # getattr(env, method)(*args) return module_file_actions, env def filter_blacklisted(specs, module_name): """ - Given a sequence of specs, filters the ones that are blacklisted in the module configuration file. + Given a sequence of specs, filters the ones that are blacklisted in the + module configuration file. Args: specs: sequence of spec instances @@ -233,7 +246,8 @@ class EnvModule(object): class __metaclass__(type): def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) - if cls.name != 'env_module' and cls.name in CONFIGURATION['enable']: + if cls.name != 'env_module' and cls.name in CONFIGURATION[ + 'enable']: module_types[cls.name] = cls def __init__(self, spec=None): @@ -249,7 +263,8 @@ class EnvModule(object): # long description is the docstring with reduced whitespace. self.long_description = None if self.spec.package.__doc__: - self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) + self.long_description = re.sub(r'\s+', ' ', + self.spec.package.__doc__) @property def naming_scheme(self): @@ -271,12 +286,14 @@ class EnvModule(object): @property def use_name(self): """ - Subclasses should implement this to return the name the module command uses to refer to the package. + Subclasses should implement this to return the name the module command + uses to refer to the package. """ naming_tokens = self.tokens naming_scheme = self.naming_scheme name = naming_scheme.format(**naming_tokens) - name += '-' + self.spec.dag_hash() # Always append the hash to make the module file unique + name += '-' + self.spec.dag_hash( + ) # Always append the hash to make the module file unique # Not everybody is working on linux... parts = name.split('/') name = join_path(*parts) @@ -296,8 +313,12 @@ class EnvModule(object): @property def blacklisted(self): configuration = CONFIGURATION.get(self.name, {}) - whitelist_matches = [x for x in configuration.get('whitelist', []) if self.spec.satisfies(x)] - blacklist_matches = [x for x in configuration.get('blacklist', []) if self.spec.satisfies(x)] + whitelist_matches = [x + for x in configuration.get('whitelist', []) + if self.spec.satisfies(x)] + blacklist_matches = [x + for x in configuration.get('blacklist', []) + if self.spec.satisfies(x)] if whitelist_matches: message = '\tWHITELIST : %s [matches : ' % self.spec.cshort_spec for rule in whitelist_matches: @@ -327,7 +348,8 @@ class EnvModule(object): """ if self.blacklisted: return - tty.debug("\tWRITE : %s [%s]" % (self.spec.cshort_spec, self.file_name)) + tty.debug("\tWRITE : %s [%s]" % + (self.spec.cshort_spec, self.file_name)) module_dir = os.path.dirname(self.file_name) if not os.path.exists(module_dir): @@ -337,11 +359,12 @@ class EnvModule(object): # installation prefix env = inspect_path(self.spec.prefix) - # Let the extendee/dependency modify their extensions/dependencies before asking for - # package-specific modifications + # Let the extendee/dependency modify their extensions/dependencies + # before asking for package-specific modifications spack_env = EnvironmentModifications() - # TODO : the code down below is quite similar to build_environment.setup_package and needs to be - # TODO : factored out to a single place + # TODO : the code down below is quite similar to + # TODO : build_environment.setup_package and needs to be factored out + # TODO : to a single place for item in dependencies(self.spec, 'all'): package = self.spec[item.name].package modules = parent_class_modules(package.__class__) @@ -358,14 +381,18 @@ class EnvModule(object): # Parse configuration file module_configuration, conf_env = parse_config_options(self) env.extend(conf_env) - filters = module_configuration.get('filter', {}).get('environment_blacklist',{}) + filters = module_configuration.get('filter', {}).get( + 'environment_blacklist', {}) # Build up the module file content module_file_content = self.header - for x in filter_blacklisted(module_configuration.pop('autoload', []), self.name): + for x in filter_blacklisted( + module_configuration.pop('autoload', []), self.name): module_file_content += self.autoload(x) - for x in filter_blacklisted(module_configuration.pop('prerequisites', []), self.name): + for x in filter_blacklisted( + module_configuration.pop('prerequisites', []), self.name): module_file_content += self.prerequisite(x) - for line in self.process_environment_command(filter_environment_blacklist(env, filters)): + for line in self.process_environment_command( + filter_environment_blacklist(env, filters)): module_file_content += line for line in self.module_specific_content(module_configuration): module_file_content += line @@ -392,10 +419,13 @@ class EnvModule(object): def process_environment_command(self, env): for command in env: try: - yield self.environment_modifications_formats[type(command)].format(**command.args) + yield self.environment_modifications_formats[type( + command)].format(**command.args) except KeyError: - tty.warn('Cannot handle command of type {command} : skipping request'.format(command=type(command))) - tty.warn('{context} at {filename}:{lineno}'.format(**command.args)) + message = 'Cannot handle command of type {command} : skipping request' # NOQA: ignore=E501 + tty.warn(message.format(command=type(command))) + context = '{context} at {filename}:{lineno}' + tty.warn(context.format(**command.args)) @property def file_name(self): @@ -408,9 +438,12 @@ class EnvModule(object): if os.path.exists(mod_file): try: os.remove(mod_file) # Remove the module file - os.removedirs(os.path.dirname(mod_file)) # Remove all the empty directories from the leaf up + os.removedirs( + os.path.dirname(mod_file) + ) # Remove all the empty directories from the leaf up except OSError: - pass # removedirs throws OSError on first non-empty directory found + # removedirs throws OSError on first non-empty directory found + pass class Dotkit(EnvModule): @@ -424,13 +457,12 @@ class Dotkit(EnvModule): autoload_format = 'dk_op {module_file}\n' - prerequisite_format = None # TODO : does something like prerequisite exist for dotkit? - - default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' + default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' # NOQA: ignore=E501 @property def file_name(self): - return join_path(Dotkit.path, self.spec.architecture, '%s.dk' % self.use_name) + return join_path(Dotkit.path, self.spec.architecture, + '%s.dk' % self.use_name) @property def header(self): @@ -474,7 +506,7 @@ class TclModule(EnvModule): prerequisite_format = 'prereq {module_file}\n' - default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' + default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' # NOQA: ignore=E501 @property def file_name(self): @@ -482,9 +514,10 @@ class TclModule(EnvModule): @property def header(self): + timestamp = datetime.datetime.now() # TCL Modulefile header header = '#%Module1.0\n' - header += '## Module file created by spack (https://github.com/LLNL/spack) on %s\n' % datetime.datetime.now() + header += '## Module file created by spack (https://github.com/LLNL/spack) on %s\n' % timestamp # NOQA: ignore=E501 header += '##\n' header += '## %s\n' % self.spec.short_spec header += '##\n' @@ -509,16 +542,19 @@ class TclModule(EnvModule): f = string.Formatter() for item in conflict_format: line = 'conflict ' + item + '\n' - if len([x for x in f.parse(line)]) > 1: # We do have placeholder to substitute - for naming_dir, conflict_dir in zip(self.naming_scheme.split('/'), item.split('/')): + if len([x for x in f.parse(line) + ]) > 1: # We do have placeholder to substitute + for naming_dir, conflict_dir in zip( + self.naming_scheme.split('/'), item.split('/')): if naming_dir != conflict_dir: - message = 'conflict scheme does not match naming scheme [{spec}]\n\n' + message = 'conflict scheme does not match naming' + message += ' [{spec}]\n\n' message += 'naming scheme : "{nformat}"\n' message += 'conflict scheme : "{cformat}"\n\n' - message += '** You may want to check your `modules.yaml` configuration file **\n' - tty.error( - message.format(spec=self.spec, nformat=self.naming_scheme, cformat=item) - ) + message += '** You may want to check your `modules.yaml` configuration file **\n' # NOQA: ignore=E501 + tty.error(message.format(spec=self.spec, + nformat=self.naming_scheme, + cformat=item)) raise SystemExit('Module generation aborted.') line = line.format(**naming_tokens) yield line diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 05f58ab7b1..395ca0c87a 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -23,53 +23,23 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -import unittest -import nose -from spack.test.tally_plugin import Tally -from llnl.util.filesystem import join_path import llnl.util.tty as tty -from llnl.util.tty.colify import colify - +import nose import spack - +from llnl.util.filesystem import join_path +from llnl.util.tty.colify import colify +from spack.test.tally_plugin import Tally """Names of tests to be included in Spack's test suite""" -test_names = ['versions', - 'url_parse', - 'url_substitution', - 'packages', - 'stage', - 'spec_syntax', - 'spec_semantics', - 'spec_dag', - 'concretize', - 'multimethod', - 'install', - 'package_sanity', - 'config', - 'directory_layout', - 'pattern', - 'python_version', - 'git_fetch', - 'svn_fetch', - 'hg_fetch', - 'mirror', - 'modules', - 'url_extrapolate', - 'cc', - 'link_tree', - 'spec_yaml', - 'optional_deps', - 'make_executable', - 'configure_guess', - 'lock', - 'database', - 'namespace_trie', - 'yaml', - 'sbang', - 'environment', - 'cmd.uninstall', - 'cmd.test_install'] +test_names = ['versions', 'url_parse', 'url_substitution', 'packages', 'stage', + 'spec_syntax', 'spec_semantics', 'spec_dag', 'concretize', + 'multimethod', 'install', 'package_sanity', 'config', + 'directory_layout', 'pattern', 'python_version', 'git_fetch', + 'svn_fetch', 'hg_fetch', 'mirror', 'modules', 'url_extrapolate', + 'cc', 'link_tree', 'spec_yaml', 'optional_deps', + 'make_executable', 'configure_guess', 'lock', 'database', + 'namespace_trie', 'yaml', 'sbang', 'environment', + 'cmd.uninstall', 'cmd.test_install'] def list_tests(): @@ -80,7 +50,6 @@ def list_tests(): def run(names, outputDir, verbose=False): """Run tests with the supplied names. Names should be a list. If it's empty, run ALL of Spack's tests.""" - verbosity = 1 if not verbose else 2 if not names: names = test_names @@ -95,7 +64,7 @@ def run(names, outputDir, verbose=False): tally = Tally() for test in names: module = 'spack.test.' + test - print module + print(module) tty.msg("Running test: %s" % test) @@ -105,15 +74,13 @@ def run(names, outputDir, verbose=False): xmlOutputFname = "unittests-{0}.xml".format(test) xmlOutputPath = join_path(outputDir, xmlOutputFname) runOpts += ["--with-xunit", - "--xunit-file={0}".format(xmlOutputPath)] + "--xunit-file={0}".format(xmlOutputPath)] argv = [""] + runOpts + [module] - result = nose.run(argv=argv, addplugins=[tally]) + nose.run(argv=argv, addplugins=[tally]) succeeded = not tally.failCount and not tally.errorCount - tty.msg("Tests Complete.", - "%5d tests run" % tally.numberOfTestsRun, - "%5d failures" % tally.failCount, - "%5d errors" % tally.errorCount) + tty.msg("Tests Complete.", "%5d tests run" % tally.numberOfTestsRun, + "%5d failures" % tally.failCount, "%5d errors" % tally.errorCount) if succeeded: tty.info("OK", format='g') diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index b8b0d6fc6a..f0ff778a10 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -2,14 +2,18 @@ import collections from contextlib import contextmanager import StringIO +import spack.modules +from spack.test.mock_packages_test import MockPackagesTest FILE_REGISTRY = collections.defaultdict(StringIO.StringIO) + # Monkey-patch open to write module files to a StringIO instance @contextmanager def mock_open(filename, mode): if not mode == 'w': - raise RuntimeError('test.modules : unexpected opening mode for monkey-patched open') + message = 'test.modules : unexpected opening mode [mock_open]' + raise RuntimeError(message) FILE_REGISTRY[filename] = StringIO.StringIO() @@ -20,7 +24,6 @@ def mock_open(filename, mode): FILE_REGISTRY[filename] = handle.getvalue() handle.close() -import spack.modules configuration_autoload_direct = { 'enable': ['tcl'], @@ -47,7 +50,8 @@ configuration_alter_environment = { 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']} }, '=x86-linux': { - 'environment': {'set': {'FOO': 'foo'}, 'unset': ['BAR']} + 'environment': {'set': {'FOO': 'foo'}, + 'unset': ['BAR']} } } } @@ -72,15 +76,14 @@ configuration_conflicts = { } } -from spack.test.mock_packages_test import MockPackagesTest - class TclTests(MockPackagesTest): def setUp(self): super(TclTests, self).setUp() self.configuration_obj = spack.modules.CONFIGURATION spack.modules.open = mock_open - spack.modules.CONFIGURATION = None # Make sure that a non-mocked configuration will trigger an error + # Make sure that a non-mocked configuration will trigger an error + spack.modules.CONFIGURATION = None def tearDown(self): del spack.modules.open @@ -98,7 +101,7 @@ class TclTests(MockPackagesTest): spack.modules.CONFIGURATION = configuration_autoload_direct spec = spack.spec.Spec('mpich@3.0.4=x86-linux') content = self.get_modulefile_content(spec) - self.assertTrue('module-whatis "mpich @3.0.4"' in content ) + self.assertTrue('module-whatis "mpich @3.0.4"' in content) def test_autoload(self): spack.modules.CONFIGURATION = configuration_autoload_direct @@ -117,14 +120,22 @@ class TclTests(MockPackagesTest): spack.modules.CONFIGURATION = configuration_alter_environment spec = spack.spec.Spec('mpileaks=x86-linux') content = self.get_modulefile_content(spec) - self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) - self.assertEqual(len([x for x in content if 'setenv FOO "foo"' in x]), 1) + self.assertEqual( + len([x + for x in content + if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) + self.assertEqual( + len([x for x in content if 'setenv FOO "foo"' in x]), 1) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 1) spec = spack.spec.Spec('libdwarf=x64-linux') content = self.get_modulefile_content(spec) - self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) - self.assertEqual(len([x for x in content if 'setenv FOO "foo"' in x]), 0) + self.assertEqual( + len([x + for x in content + if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) + self.assertEqual( + len([x for x in content if 'setenv FOO "foo"' in x]), 0) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 0) def test_blacklist(self): @@ -138,6 +149,9 @@ class TclTests(MockPackagesTest): spack.modules.CONFIGURATION = configuration_conflicts spec = spack.spec.Spec('mpileaks=x86-linux') content = self.get_modulefile_content(spec) - self.assertEqual(len([x for x in content if x.startswith('conflict')]), 2) - self.assertEqual(len([x for x in content if x == 'conflict mpileaks']), 1) - self.assertEqual(len([x for x in content if x == 'conflict intel/14.0.1']), 1) + self.assertEqual( + len([x for x in content if x.startswith('conflict')]), 2) + self.assertEqual( + len([x for x in content if x == 'conflict mpileaks']), 1) + self.assertEqual( + len([x for x in content if x == 'conflict intel/14.0.1']), 1) -- cgit v1.2.3-70-g09d2 From fa44cd5cefae8890ece95b9870e24058dcd0a10f Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Tue, 10 May 2016 17:30:44 +0200 Subject: renaming py-Genshi into py-genshi --- var/spack/repos/builtin/packages/py-Genshi/package.py | 18 ------------------ var/spack/repos/builtin/packages/py-genshi/package.py | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 18 deletions(-) delete mode 100644 var/spack/repos/builtin/packages/py-Genshi/package.py create mode 100644 var/spack/repos/builtin/packages/py-genshi/package.py diff --git a/var/spack/repos/builtin/packages/py-Genshi/package.py b/var/spack/repos/builtin/packages/py-Genshi/package.py deleted file mode 100644 index d485c89429..0000000000 --- a/var/spack/repos/builtin/packages/py-Genshi/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import version, extends, depends_on -from spack import Package - - -class PyGenshi(Package): - """Python toolkit for generation of output for the web""" - homepage = "https://genshi.edgewall.org/" - url = "http://ftp.edgewall.com/pub/genshi/Genshi-0.7.tar.gz" - - version('0.7', '54e64dd69da3ec961f86e686e0848a82') - version('0.6.1', '372c368c8931110b0a521fa6091742d7') - version('0.6', '604e8b23b4697655d36a69c2d8ef7187') - - extends("python") - depends_on("py-setuptools") - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-genshi/package.py b/var/spack/repos/builtin/packages/py-genshi/package.py new file mode 100644 index 0000000000..d485c89429 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-genshi/package.py @@ -0,0 +1,18 @@ +from spack import version, extends, depends_on +from spack import Package + + +class PyGenshi(Package): + """Python toolkit for generation of output for the web""" + homepage = "https://genshi.edgewall.org/" + url = "http://ftp.edgewall.com/pub/genshi/Genshi-0.7.tar.gz" + + version('0.7', '54e64dd69da3ec961f86e686e0848a82') + version('0.6.1', '372c368c8931110b0a521fa6091742d7') + version('0.6', '604e8b23b4697655d36a69c2d8ef7187') + + extends("python") + depends_on("py-setuptools") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) -- cgit v1.2.3-70-g09d2 From c90ecc6847c034f648d49c813e4a633bf16322fb Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 10 May 2016 08:58:30 -0700 Subject: Add F821 to ignore list. - Consider removing this once names like `cmake` and `configure` are moved to a sensible namespace. --- flake8.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake8.ini b/flake8.ini index f8bff04355..32ced73dd7 100644 --- a/flake8.ini +++ b/flake8.ini @@ -1,3 +1,3 @@ [flake8] -ignore = W391,F403,E221 +ignore = W391,F403,E221,F821 max-line-length = 79 -- cgit v1.2.3-70-g09d2 From c8d88607425866413c0d3c931e760c89c01a00fd Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 10 May 2016 17:46:11 +0200 Subject: astyle: fix formatting --- var/spack/repos/builtin/packages/astyle/package.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/astyle/package.py b/var/spack/repos/builtin/packages/astyle/package.py index c8e8c3cbec..57083bb947 100644 --- a/var/spack/repos/builtin/packages/astyle/package.py +++ b/var/spack/repos/builtin/packages/astyle/package.py @@ -1,8 +1,11 @@ from spack import * -import os + class Astyle(Package): - """A Free, Fast, and Small Automatic Formatter for C, C++, C++/CLI, Objective-C, C#, and Java Source Code.""" + """ + A Free, Fast, and Small Automatic Formatter for C, C++, C++/CLI, + Objective-C, C#, and Java Source Code. + """ homepage = "http://astyle.sourceforge.net/" url = "http://downloads.sourceforge.net/project/astyle/astyle/astyle%202.04/astyle_2.04_linux.tar.gz" @@ -11,13 +14,15 @@ class Astyle(Package): def install(self, spec, prefix): with working_dir('src'): - # unfortunately we need to edit the makefile in place to set compiler: - make_file = join_path(self.stage.source_path,'build','gcc','Makefile') - filter_file(r'^CXX\s*=.*', 'CXX=%s' % spack_cxx, make_file) + # we need to edit the makefile in place to set compiler: + make_file = join_path(self.stage.source_path, + 'build', 'gcc', 'Makefile') + filter_file(r'^CXX\s*=.*', 'CXX=%s'.format(spack_cxx), make_file) make('-f', - make_file, - parallel=False) + make_file, + parallel=False) mkdirp(self.prefix.bin) - install(join_path(self.stage.source_path, 'src','bin','astyle'), self.prefix.bin) + install(join_path(self.stage.source_path, 'src', 'bin', 'astyle'), + self.prefix.bin) -- cgit v1.2.3-70-g09d2 From 979b53cab605d250dfe23edaacc6b0d2ff88d978 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 10 May 2016 18:33:07 +0200 Subject: qa : modifying 2 packages and a framework file just for the sake of it --- lib/spack/spack/architecture.py | 1 + var/spack/repos/builtin/packages/ImageMagick/package.py | 4 +++- var/spack/repos/builtin/packages/Mitos/package.py | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 2701fab90c..eec7636d00 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -44,6 +44,7 @@ class NoSysTypeError(serr.SpackError): "Could not determine sys_type for this machine.") + def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): diff --git a/var/spack/repos/builtin/packages/ImageMagick/package.py b/var/spack/repos/builtin/packages/ImageMagick/package.py index 753ea80ca6..d0a820b5fc 100644 --- a/var/spack/repos/builtin/packages/ImageMagick/package.py +++ b/var/spack/repos/builtin/packages/ImageMagick/package.py @@ -23,8 +23,10 @@ class Imagemagick(Package): version('6.8.9-10', 'aa050bf9785e571c956c111377bbf57c', url="http://sourceforge.net/projects/imagemagick/files/old-sources/6.x/6.8/ImageMagick-6.8.9-10.tar.gz/download") - depends_on('libtool') + + depends_on('jpeg') + depends_on('libtool') depends_on('libpng') depends_on('freetype') depends_on('fontconfig') diff --git a/var/spack/repos/builtin/packages/Mitos/package.py b/var/spack/repos/builtin/packages/Mitos/package.py index ea131872dd..073c473b93 100644 --- a/var/spack/repos/builtin/packages/Mitos/package.py +++ b/var/spack/repos/builtin/packages/Mitos/package.py @@ -19,6 +19,8 @@ class Mitos(Package): depends_on('hwloc') depends_on('mpi') + + def install(self, spec, prefix): with working_dir('spack-build', create=True): cmake('..', *std_cmake_args) -- cgit v1.2.3-70-g09d2 From 190c437980ecb7e1710643b05278cfd4f4b77b1b Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 10 May 2016 18:38:30 +0200 Subject: qa : copied flake8 configuration, updated travis.yml --- .travis.yml | 2 +- flake8-framework.ini | 3 +++ flake8-packages.ini | 3 +++ flake8.ini | 3 --- 4 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 flake8-framework.ini create mode 100644 flake8-packages.ini delete mode 100644 flake8.ini diff --git a/.travis.yml b/.travis.yml index 4ff4d5f483..96138813e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ script: - coverage run bin/spack test # Checks if the file that have been changed are flake8 conformant - CHANGED_PYTHON_FILES=`git diff develop... --name-only | perl -ne 'print if /\.py$/'` - - if [[ ${CHANGED_PYTHON_FILES} ]] ; then flake8 --format pylint --config flake8.ini ${CHANGED_PYTHON_FILES} ; fi + - if [[ ${CHANGED_PYTHON_FILES} ]] ; then flake8 --format pylint --config flake8-framework.ini ${CHANGED_PYTHON_FILES} ; fi after_success: diff --git a/flake8-framework.ini b/flake8-framework.ini new file mode 100644 index 0000000000..32ced73dd7 --- /dev/null +++ b/flake8-framework.ini @@ -0,0 +1,3 @@ +[flake8] +ignore = W391,F403,E221,F821 +max-line-length = 79 diff --git a/flake8-packages.ini b/flake8-packages.ini new file mode 100644 index 0000000000..32ced73dd7 --- /dev/null +++ b/flake8-packages.ini @@ -0,0 +1,3 @@ +[flake8] +ignore = W391,F403,E221,F821 +max-line-length = 79 diff --git a/flake8.ini b/flake8.ini deleted file mode 100644 index 32ced73dd7..0000000000 --- a/flake8.ini +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -ignore = W391,F403,E221,F821 -max-line-length = 79 -- cgit v1.2.3-70-g09d2 From 3f20014f335f50e6bac8f8ced7a36fae00537c61 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 10 May 2016 18:45:11 +0200 Subject: qa : differentiate framework from packages --- .travis.yml | 7 +++++-- flake8-framework.ini | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 96138813e2..c41692c08e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,9 +25,12 @@ script: - spack install -v libdwarf # Run unit tests with code coverage - coverage run bin/spack test - # Checks if the file that have been changed are flake8 conformant - - CHANGED_PYTHON_FILES=`git diff develop... --name-only | perl -ne 'print if /\.py$/'` + # Checks if the file that have been changed are flake8 conformant [framework] + - CHANGED_PYTHON_FILES=`git diff develop... --name-only | perl -ne 'print if /\.py$/' | perl -ne 'print if not /var/' ` - if [[ ${CHANGED_PYTHON_FILES} ]] ; then flake8 --format pylint --config flake8-framework.ini ${CHANGED_PYTHON_FILES} ; fi + # Checks if the file that have been changed are flake8 conformant [packages] + - CHANGED_PACKAGES=`git diff develop... --name-only | perl -ne 'print if /\.py$/' | perl -ne 'print if /var/' ` + - if [[ ${CHANGED_PACKAGES} ]] ; then flake8 --format pylint --config flake8-packages.ini ${CHANGED_PACKAGES} ; fi after_success: diff --git a/flake8-framework.ini b/flake8-framework.ini index 32ced73dd7..f8bff04355 100644 --- a/flake8-framework.ini +++ b/flake8-framework.ini @@ -1,3 +1,3 @@ [flake8] -ignore = W391,F403,E221,F821 +ignore = W391,F403,E221 max-line-length = 79 -- cgit v1.2.3-70-g09d2 From 7d74e209f365ba135acd9dfe738737bcf37d4b79 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 10 May 2016 19:20:26 +0200 Subject: qa : this should still fail due to F821 --- lib/spack/spack/architecture.py | 15 +++++++-------- var/spack/repos/builtin/packages/ImageMagick/package.py | 14 +++++++------- var/spack/repos/builtin/packages/Mitos/package.py | 9 +++------ 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index eec7636d00..62c25c8003 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -34,15 +34,14 @@ import spack.error as serr class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__( - "Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, + self).__init__("Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__( - "Could not determine sys_type for this machine.") - + super(NoSysTypeError, + self).__init__("Could not determine sys_type for this machine.") def get_sys_type_from_spack_globals(): @@ -70,15 +69,15 @@ def get_sys_type_from_platform(): @memoized def sys_type(): """Returns a SysType for the current machine.""" - methods = [get_sys_type_from_spack_globals, - get_sys_type_from_environment, + methods = [get_sys_type_from_spack_globals, get_sys_type_from_environment, get_sys_type_from_platform] # search for a method that doesn't return None sys_type = None for method in methods: sys_type = method() - if sys_type: break + if sys_type: + break # Couldn't determine the sys_type for this machine. if sys_type is None: diff --git a/var/spack/repos/builtin/packages/ImageMagick/package.py b/var/spack/repos/builtin/packages/ImageMagick/package.py index d0a820b5fc..7a3d12e364 100644 --- a/var/spack/repos/builtin/packages/ImageMagick/package.py +++ b/var/spack/repos/builtin/packages/ImageMagick/package.py @@ -1,10 +1,11 @@ from spack import * + class Imagemagick(Package): """ImageMagick is a image processing library""" homepage = "http://www.imagemagic.org" - #------------------------------------------------------------------------- + # ------------------------------------------------------------------------- # ImageMagick does not keep around anything but *-10 versions, so # this URL may change. If you want the bleeding edge, you can # uncomment it and see if it works but you may need to try to @@ -17,13 +18,13 @@ class Imagemagick(Package): # version('6.9.0-6', 'c1bce7396c22995b8bdb56b7797b4a1b', # url="http://www.imagemagick.org/download/ImageMagick-6.9.0-6.tar.bz2") - #------------------------------------------------------------------------- + # ------------------------------------------------------------------------- # *-10 versions are archived, so these versions should fetch reliably. # ------------------------------------------------------------------------- - version('6.8.9-10', 'aa050bf9785e571c956c111377bbf57c', - url="http://sourceforge.net/projects/imagemagick/files/old-sources/6.x/6.8/ImageMagick-6.8.9-10.tar.gz/download") - - + version( + '6.8.9-10', + 'aa050bf9785e571c956c111377bbf57c', + url="http://sourceforge.net/projects/imagemagick/files/old-sources/6.x/6.8/ImageMagick-6.8.9-10.tar.gz/download") # NOQA: ignore=E501 depends_on('jpeg') depends_on('libtool') @@ -34,6 +35,5 @@ class Imagemagick(Package): def install(self, spec, prefix): configure("--prefix=%s" % prefix) - make() make("install") diff --git a/var/spack/repos/builtin/packages/Mitos/package.py b/var/spack/repos/builtin/packages/Mitos/package.py index 073c473b93..ec1d56a5c7 100644 --- a/var/spack/repos/builtin/packages/Mitos/package.py +++ b/var/spack/repos/builtin/packages/Mitos/package.py @@ -1,26 +1,23 @@ from spack import * + class Mitos(Package): """Mitos is a library and a tool for collecting sampled memory performance data to view with MemAxes""" homepage = "https://github.com/llnl/Mitos" - url = "https://github.com/llnl/Mitos" + url = "https://github.com/llnl/Mitos" version('0.9.2', git='https://github.com/llnl/Mitos.git', commit='8cb143a2e8c00353ff531a781a9ca0992b0aaa3d') - version('0.9.1', - git='https://github.com/llnl/Mitos.git', - tag='v0.9.1') + version('0.9.1', git='https://github.com/llnl/Mitos.git', tag='v0.9.1') depends_on('dyninst@8.2.1:') depends_on('hwloc') depends_on('mpi') - - def install(self, spec, prefix): with working_dir('spack-build', create=True): cmake('..', *std_cmake_args) -- cgit v1.2.3-70-g09d2 From 4cb91d6f7b6680140e0011da1f7965c430920067 Mon Sep 17 00:00:00 2001 From: Luca Heltai Date: Tue, 10 May 2016 19:46:24 +0200 Subject: Added documentation on lockf/flock. --- lib/spack/docs/basic_usage.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 29791d98c4..58e67e4550 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -1246,6 +1246,38 @@ several variants: spack deactivate -a python +A word of warning +----------------------- + +If you run `spack find` and you get an error similar to the following: + +.. code-block:: sh + + $ ./spack find + Traceback (most recent call last): + File "./spack", line 176, in + main() + File "./spack", line 154, in main + return_val = command(parser, args) + File "./spack/lib/spack/spack/cmd/find.py", line 170, in find + specs = set(spack.installed_db.query(**q_args)) + File "./spack/lib/spack/spack/database.py", line 551, in query + with self.read_transaction(): + File "./spack/lib/spack/spack/database.py", line 598, in __enter__ + if self._enter() and self._acquire_fn: + File "./spack/lib/spack/spack/database.py", line 608, in _enter + return self._db.lock.acquire_read(self._timeout) + File "./spack/lib/spack/llnl/util/lock.py", line 103, in acquire_read + self._lock(fcntl.LOCK_SH, timeout) # can raise LockError. + File "./spack/lib/spack/llnl/util/lock.py", line 64, in _lock + fcntl.lockf(self._fd, op | fcntl.LOCK_NB) + IOError: [Errno 38] Function not implemented + + +it most likely means that you are trying to run spack in a shared +network filesystem without support for lockf/flock. Currently this is not +supported, and you should ask your system administrator to enable lockf/flock. + Getting Help ----------------------- -- cgit v1.2.3-70-g09d2 From 98faee1d5c24ddacaeb1bc630ff8c31ba0d74f9b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 10 May 2016 11:19:17 -0700 Subject: Minor tweaks to `flock` docs. --- lib/spack/docs/basic_usage.rst | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 58e67e4550..f08a2bb675 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -1246,13 +1246,30 @@ several variants: spack deactivate -a python -A word of warning ------------------------ +Filesystem requirements +-------------------------- -If you run `spack find` and you get an error similar to the following: +Spack currently needs to be run from a filesystem that supports +``flock`` locking semantics. Nearly all local filesystems and recent +versions of NFS support this, but parallel filesystems may be mounted +without ``flock`` support enabled. You can determine how your +filesystems are mounted with ``mount -p``. The output for a Lustre +filesystem might look like this: .. code-block:: sh - + + $ mount -l | grep lscratch + pilsner-mds1-lnet0@o2ib100:/lsd on /p/lscratchd type lustre (rw,nosuid,noauto,_netdev,lazystatfs,flock) + porter-mds1-lnet0@o2ib100:/lse on /p/lscratche type lustre (rw,nosuid,noauto,_netdev,lazystatfs,flock) + +Note the ``flock`` option on both Lustre mounts. If you do not see +this or a similar option for your filesystem, you may need ot ask your +system administrator to enable ``flock``. + +This issue typically manifests with the error below: + +.. code-block:: sh + $ ./spack find Traceback (most recent call last): File "./spack", line 176, in @@ -1273,11 +1290,7 @@ If you run `spack find` and you get an error similar to the following: fcntl.lockf(self._fd, op | fcntl.LOCK_NB) IOError: [Errno 38] Function not implemented - -it most likely means that you are trying to run spack in a shared -network filesystem without support for lockf/flock. Currently this is not -supported, and you should ask your system administrator to enable lockf/flock. - +A nicer error message is TBD in future versions of Spack. Getting Help ----------------------- -- cgit v1.2.3-70-g09d2 From 5d477bc956e0033bd2816d48df1f4096856bc5fe Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Tue, 10 May 2016 23:19:30 +0200 Subject: fix setting config for list parameters --- lib/spack/spack/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 336d47cbb7..06954f535b 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -545,7 +545,10 @@ def update_config(section, update_data, scope=None): # read in the config to ensure we've got current data configuration = get_config(section) - configuration.update(update_data) + if isinstance(update_data, list): + configuration = update_data + else: + configuration.extend(update_data) # read only the requested section's data. scope.sections[section] = {section: configuration} -- cgit v1.2.3-70-g09d2 From 6e462abb4d208584cb5f984f2f4c177bcec37dd4 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Tue, 10 May 2016 23:31:29 +0200 Subject: hmm... test before commit --- lib/spack/spack/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 06954f535b..3bdefd3a6c 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -548,7 +548,7 @@ def update_config(section, update_data, scope=None): if isinstance(update_data, list): configuration = update_data else: - configuration.extend(update_data) + configuration.update(update_data) # read only the requested section's data. scope.sections[section] = {section: configuration} -- cgit v1.2.3-70-g09d2 From 05b6c3f8cfd065dcf4173613a64ed94fa1815d31 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Wed, 11 May 2016 00:14:24 +0200 Subject: add test for list parameters --- lib/spack/spack/test/config.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 3977f0e7d4..ed0797a541 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -72,6 +72,10 @@ b_comps = { } } +# Some Sample repo data +repos_low = [ "/some/path" ] +repos_high = [ "/some/other/path" ] + class ConfigTest(MockPackagesTest): def setUp(self): @@ -95,6 +99,12 @@ class ConfigTest(MockPackagesTest): actual = config[arch][key][c] self.assertEqual(expected, actual) + def test_write_list_in_memory(self): + spack.config.update_config('repos', repos_low, 'test_low_priority') + spack.config.update_config('repos', repos_high, 'test_high_priority') + config = spack.config.get_config('repos') + self.assertEqual(config, repos_high+repos_low) + def test_write_key_in_memory(self): # Write b_comps "on top of" a_comps. spack.config.update_config('compilers', a_comps, 'test_low_priority') -- cgit v1.2.3-70-g09d2 From c1e6b5218438988d7dc2da5ca577fe5a8752d6e1 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Wed, 11 May 2016 00:28:12 +0200 Subject: first round of coding rules --- lib/spack/spack/config.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3bdefd3a6c..9da5d7aaf0 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -129,7 +129,6 @@ from ordereddict_backport import OrderedDict import llnl.util.tty as tty from llnl.util.filesystem import mkdirp -import copy import spack from spack.error import SpackError @@ -306,13 +305,14 @@ def extend_with_default(validator_class): yield err return validators.extend(validator_class, { - "properties" : set_defaults, - "patternProperties" : set_pp_defaults + "properties": set_defaults, + "patternProperties": set_pp_defaults }) DefaultSettingValidator = extend_with_default(Draft4Validator) + def validate_section(data, schema): """Validate data read in from a Spack YAML file. @@ -347,16 +347,14 @@ class ConfigScope(object): validate_section_name(section) return os.path.join(self.path, "%s.yaml" % section) - def get_section(self, section): - if not section in self.sections: + if section not in self.sections: path = self.get_section_filename(section) schema = section_schemas[section] data = _read_config_file(path, schema) self.sections[section] = data return self.sections[section] - def write_section(self, section): filename = self.get_section_filename(section) data = self.get_section(section) @@ -370,7 +368,6 @@ class ConfigScope(object): except (yaml.YAMLError, IOError) as e: raise ConfigFileError("Error writing to config file: '%s'" % str(e)) - def clear(self): """Empty cached config information.""" self.sections = {} @@ -476,7 +473,7 @@ def _merge_yaml(dest, source): # Source dict is merged into dest. elif they_are(dict): for sk, sv in source.iteritems(): - if not sk in dest: + if sk not in dest: dest[sk] = copy.copy(sv) else: dest[sk] = _merge_yaml(dest[sk], source[sk]) @@ -590,16 +587,20 @@ def spec_externals(spec): def is_spec_buildable(spec): """Return true if the spec pkgspec is configured as buildable""" allpkgs = get_config('packages') - name = spec.name - if not spec.name in allpkgs: + if spec.name not in allpkgs: return True - if not 'buildable' in allpkgs[spec.name]: + if 'buildable' not in allpkgs[spec.name]: return True return allpkgs[spec.name]['buildable'] -class ConfigError(SpackError): pass -class ConfigFileError(ConfigError): pass +class ConfigError(SpackError): + pass + + +class ConfigFileError(ConfigError): + pass + def get_path(path, data): if path: @@ -607,6 +608,7 @@ def get_path(path, data): else: return data + class ConfigFormatError(ConfigError): """Raised when a configuration format does not match its schema.""" def __init__(self, validation_error, data): @@ -641,5 +643,6 @@ class ConfigFormatError(ConfigError): message = '%s: %s' % (location, validation_error.message) super(ConfigError, self).__init__(message) + class ConfigSanityError(ConfigFormatError): """Same as ConfigFormatError, raised when config is written by Spack.""" -- cgit v1.2.3-70-g09d2 From 2aa4387ebac307285c1b2c90bc6527b82fe83839 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 10 May 2016 23:45:41 -0700 Subject: Update code style checking. - Exempt overlong URL lines from checks. - Omit some of the more painful PEP items. --- .style.yapf | 3 --- .travis.yml | 24 ++++++++++++++++++------ flake8-framework.ini | 3 --- flake8-packages.ini | 3 --- share/spack/qa/flake8-framework | 10 ++++++++++ share/spack/qa/flake8-packages | 11 +++++++++++ 6 files changed, 39 insertions(+), 15 deletions(-) delete mode 100644 .style.yapf delete mode 100644 flake8-framework.ini delete mode 100644 flake8-packages.ini create mode 100644 share/spack/qa/flake8-framework create mode 100644 share/spack/qa/flake8-packages diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index a4b3f65252..0000000000 --- a/.style.yapf +++ /dev/null @@ -1,3 +0,0 @@ -[style] -based_on_style = pep8 -column_limit = 79 diff --git a/.travis.yml b/.travis.yml index c41692c08e..9acba991c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,23 +15,35 @@ before_install: # Need this for the git tests to succeed. - git config --global user.email "spack@example.com" - git config --global user.name "Test User" + # Need this to be able to compute the list of changed files - git fetch origin develop:develop script: + # Regular spack setup and tests - . share/spack/setup-env.sh - spack compilers - spack config get compilers - spack install -v libdwarf + # Run unit tests with code coverage - coverage run bin/spack test - # Checks if the file that have been changed are flake8 conformant [framework] - - CHANGED_PYTHON_FILES=`git diff develop... --name-only | perl -ne 'print if /\.py$/' | perl -ne 'print if not /var/' ` - - if [[ ${CHANGED_PYTHON_FILES} ]] ; then flake8 --format pylint --config flake8-framework.ini ${CHANGED_PYTHON_FILES} ; fi - # Checks if the file that have been changed are flake8 conformant [packages] - - CHANGED_PACKAGES=`git diff develop... --name-only | perl -ne 'print if /\.py$/' | perl -ne 'print if /var/' ` - - if [[ ${CHANGED_PACKAGES} ]] ; then flake8 --format pylint --config flake8-packages.ini ${CHANGED_PACKAGES} ; fi + # Check if changed files are flake8 conformant [framework] + - changed=$(git diff --name-only develop... | grep '.py$' | grep -v ^var/) + - [[ $changed ]] && \ + flake8 --format pylint --config share/spack/qa/flake8-framework $changed + + # Check if changed files are flake8 conformant [packages] + - changed=$(git diff --name-only develop... | grep '.py$' | grep ^var/) + + # Exempt url lines in changed packages from overlong errors. + - for file in $changed; do \ + [[ file = *package.py ]] && \ + perl -i~ -pe 's/^(\s*url\s*=.*)$/\1 # NOQA/' $file; \ + done + - [[ $changed ]] && \ + flake8 --format pylint --config share/spack/qa/flake8-packages $changed after_success: - coveralls diff --git a/flake8-framework.ini b/flake8-framework.ini deleted file mode 100644 index f8bff04355..0000000000 --- a/flake8-framework.ini +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -ignore = W391,F403,E221 -max-line-length = 79 diff --git a/flake8-packages.ini b/flake8-packages.ini deleted file mode 100644 index 32ced73dd7..0000000000 --- a/flake8-packages.ini +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -ignore = W391,F403,E221,F821 -max-line-length = 79 diff --git a/share/spack/qa/flake8-framework b/share/spack/qa/flake8-framework new file mode 100644 index 0000000000..b1ad00b80f --- /dev/null +++ b/share/spack/qa/flake8-framework @@ -0,0 +1,10 @@ +# -*- conf -*- +[flake8] +# Descriptions of ignored checks: +# +# E221: multiple spaces before operator +# E241: multiple spaces after ‘,’ +# F403: disable wildcard import +# +ignore = E221,E241,F403 +max-line-length = 79 diff --git a/share/spack/qa/flake8-packages b/share/spack/qa/flake8-packages new file mode 100644 index 0000000000..0e5be249fb --- /dev/null +++ b/share/spack/qa/flake8-packages @@ -0,0 +1,11 @@ +# -*- conf -*- +[flake8] +# Descriptions of ignored checks: +# +# E221: multiple spaces before operator +# E241: multiple spaces after ‘,’ +# F403: disable wildcard import +# F821: undefined name (needed for build commands) +# +ignore = E221,E241,F403,F821 +max-line-length = 79 -- cgit v1.2.3-70-g09d2 From d7847ff7685a52670a9d01cf39bd44eff1a57684 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 11 May 2016 01:16:41 -0700 Subject: Use just one flake8 file; mark package.py files to avoid E501 w/url. --- .flake8 | 12 ++++++++++++ share/spack/qa/flake8-framework | 10 ---------- share/spack/qa/flake8-packages | 11 ----------- var/spack/repos/builtin/packages/ImageMagick/package.py | 2 +- 4 files changed, 13 insertions(+), 22 deletions(-) create mode 100644 .flake8 delete mode 100644 share/spack/qa/flake8-framework delete mode 100644 share/spack/qa/flake8-packages diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000000..ab1e14b66e --- /dev/null +++ b/.flake8 @@ -0,0 +1,12 @@ +# -*- conf -*- +[flake8] +# Descriptions of ignored checks: +# +# E221: multiple spaces before operator +# E241: multiple spaces after ‘,’ +# F403: disable wildcard import +# F821: undefined name (needed for build commands) +# F999: Undefined undefined name (needed for build commands) +# +ignore = E221,E241,F403,F821,F999 +max-line-length = 79 diff --git a/share/spack/qa/flake8-framework b/share/spack/qa/flake8-framework deleted file mode 100644 index b1ad00b80f..0000000000 --- a/share/spack/qa/flake8-framework +++ /dev/null @@ -1,10 +0,0 @@ -# -*- conf -*- -[flake8] -# Descriptions of ignored checks: -# -# E221: multiple spaces before operator -# E241: multiple spaces after ‘,’ -# F403: disable wildcard import -# -ignore = E221,E241,F403 -max-line-length = 79 diff --git a/share/spack/qa/flake8-packages b/share/spack/qa/flake8-packages deleted file mode 100644 index 0e5be249fb..0000000000 --- a/share/spack/qa/flake8-packages +++ /dev/null @@ -1,11 +0,0 @@ -# -*- conf -*- -[flake8] -# Descriptions of ignored checks: -# -# E221: multiple spaces before operator -# E241: multiple spaces after ‘,’ -# F403: disable wildcard import -# F821: undefined name (needed for build commands) -# -ignore = E221,E241,F403,F821 -max-line-length = 79 diff --git a/var/spack/repos/builtin/packages/ImageMagick/package.py b/var/spack/repos/builtin/packages/ImageMagick/package.py index 7a3d12e364..3a86d9fb7c 100644 --- a/var/spack/repos/builtin/packages/ImageMagick/package.py +++ b/var/spack/repos/builtin/packages/ImageMagick/package.py @@ -24,7 +24,7 @@ class Imagemagick(Package): version( '6.8.9-10', 'aa050bf9785e571c956c111377bbf57c', - url="http://sourceforge.net/projects/imagemagick/files/old-sources/6.x/6.8/ImageMagick-6.8.9-10.tar.gz/download") # NOQA: ignore=E501 + url="http://sourceforge.net/projects/imagemagick/files/old-sources/6.x/6.8/ImageMagick-6.8.9-10.tar.gz/download") depends_on('jpeg') depends_on('libtool') -- cgit v1.2.3-70-g09d2 From f15837505420e878e7b5ec2139686985deb54401 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 11 May 2016 01:17:52 -0700 Subject: .travis.yml uses a script now. - user can run the same script. --- .travis.yml | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9acba991c7..904143a00f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,21 +29,8 @@ script: # Run unit tests with code coverage - coverage run bin/spack test - # Check if changed files are flake8 conformant [framework] - - changed=$(git diff --name-only develop... | grep '.py$' | grep -v ^var/) - - [[ $changed ]] && \ - flake8 --format pylint --config share/spack/qa/flake8-framework $changed - - # Check if changed files are flake8 conformant [packages] - - changed=$(git diff --name-only develop... | grep '.py$' | grep ^var/) - - # Exempt url lines in changed packages from overlong errors. - - for file in $changed; do \ - [[ file = *package.py ]] && \ - perl -i~ -pe 's/^(\s*url\s*=.*)$/\1 # NOQA/' $file; \ - done - - [[ $changed ]] && \ - flake8 --format pylint --config share/spack/qa/flake8-packages $changed + # Run flake8 code style checks. + - share/spack/qa/run-flake8 after_success: - coveralls -- cgit v1.2.3-70-g09d2 From c512b37a4e26bb639893d06c9f2df30fb6dd6892 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 11 May 2016 01:25:40 -0700 Subject: format .flake8 more nicely. --- .flake8 | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/.flake8 b/.flake8 index ab1e14b66e..a1e2fcc1f8 100644 --- a/.flake8 +++ b/.flake8 @@ -1,12 +1,20 @@ # -*- conf -*- -[flake8] -# Descriptions of ignored checks: +# flake8 settings for Spack. +# +# Below we describe which flake8 checks Spack ignores and what the +# rationale is. +# +# Let people line things up nicely: +# - E221: multiple spaces before operator +# - E241: multiple spaces after ‘,’ # -# E221: multiple spaces before operator -# E241: multiple spaces after ‘,’ -# F403: disable wildcard import -# F821: undefined name (needed for build commands) -# F999: Undefined undefined name (needed for build commands) +# Spack allows wildcard imports: +# - F403: disable wildcard import # +# These are required to get the package.py files to test clean. +# - F821: undefined name (needed for cmake, configure, etc.) +# - F999: name name be undefined or undefined from star imports. +# +[flake8] ignore = E221,E241,F403,F821,F999 max-line-length = 79 -- cgit v1.2.3-70-g09d2 From 78d25ad337e2faaa8cfd589607801acb78aec4d9 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 11 May 2016 02:01:24 -0700 Subject: Add run-flake8 script. - was missing the obvious. --- share/spack/qa/run-flake8 | 55 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100755 share/spack/qa/run-flake8 diff --git a/share/spack/qa/run-flake8 b/share/spack/qa/run-flake8 new file mode 100755 index 0000000000..722c7fcba6 --- /dev/null +++ b/share/spack/qa/run-flake8 @@ -0,0 +1,55 @@ +#!/bin/bash +# +# This script runs source code style checks on Spack. +# +# It should be executed from the top-level directory of the repo, +# e.g.: +# +# share/spack/qa/run-flake8 +# +# To run it, you'll need to have the Python flake8 installed locally. +# +PYTHONPATH=./lib/spack:$PYTHONPATH + +flake8="$(which flake8)" +if [[ ! $flake8 ]]; then + echo "ERROR: flake8 is required to run this script." + exit 1 +fi + +# Check if changed files are flake8 conformant [framework] +changed=$(git diff --name-only develop... | grep '.py$') + +# Exempt url lines in changed packages from overlong line errors. +for file in $changed; do + if [[ $file = *package.py ]]; then + perl -i~ -pe 's/^(\s*url\s*=.*)$/\1 # NOQA: ignore=E501/' $file; + fi +done + +return_code=0 +if [[ $changed ]]; then + echo ======================================================= + echo flake8: running flake8 code checks on spack. + echo + echo Modified files: + echo $changed | perl -pe 's/^/ /;s/ +/\n /g' + echo ======================================================= + if flake8 --format pylint $changed; then + echo "Flake8 checks were clean." + else + echo "Flake8 found errors." + return_code=1 + fi +else + echo No core framework files modified. +fi + +# Restore original package files after modifying them. +for file in $changed; do + if [[ $file = *package.py ]]; then + mv "${file}~" "${file}" + fi +done + +exit $return_code -- cgit v1.2.3-70-g09d2 From 6d6eb0f2d1e1f5f4fd340f8e5f3a5159be7e7f40 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 11 May 2016 02:32:13 -0700 Subject: Remove config.py from flake checking for now. --- lib/spack/spack/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 9da5d7aaf0..21e08ff666 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1,3 +1,4 @@ +# flake8: noqa ############################################################################## # Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. -- cgit v1.2.3-70-g09d2 From 3d3a520a7d79f40f167f2856c5787ef94739eedc Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 11 May 2016 02:59:08 -0700 Subject: update readme for flake8 checking --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 1f7bcc5c61..fe00e2af27 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,11 @@ a [pull request](https://help.github.com/articles/using-pull-requests/). When you send your request, make ``develop`` the destination branch on the [Spack repository](https://github.com/LLNL/spack). +Your contribution will need to pass all the tests run by the `spack test` +command, as well as the formatting checks in `share/spack/qa/run-flake8`. +You should run both of these before submitting your pull request, to +ensure that the online checks succeed. + Spack is using a rough approximation of the [Git Flow](http://nvie.com/posts/a-successful-git-branching-model/) branching model. The ``develop`` branch contains the latest -- cgit v1.2.3-70-g09d2 From 0b7b25487f326b1ff6e0d04b41e1753398262b58 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Wed, 11 May 2016 13:29:27 +0200 Subject: improve error message for wrong config section names --- lib/spack/spack/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 21e08ff666..9488e49ab9 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -269,10 +269,10 @@ config_scopes = OrderedDict() def validate_section_name(section): - """Raise a ValueError if the section is not a valid section.""" + """Exit if the section is not a valid section.""" if section not in section_schemas: - raise ValueError("Invalid config section: '%s'. Options are %s" - % (section, section_schemas)) + tty.die("Invalid config section: '%s'. Options are: %s" + % (section, " ".join(section_schemas.keys()))) def extend_with_default(validator_class): -- cgit v1.2.3-70-g09d2 From 809ded74c99ac6883bc5b5e4c7a2da887131360c Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 11 May 2016 15:02:14 +0200 Subject: add functions for simple unit tests; refactor openblas to use them --- lib/spack/spack/package_test.py | 64 ++++++++++++++++++++++ .../repos/builtin/packages/openblas/package.py | 39 +++---------- 2 files changed, 72 insertions(+), 31 deletions(-) create mode 100644 lib/spack/spack/package_test.py diff --git a/lib/spack/spack/package_test.py b/lib/spack/spack/package_test.py new file mode 100644 index 0000000000..4b021684ee --- /dev/null +++ b/lib/spack/spack/package_test.py @@ -0,0 +1,64 @@ +############################################################################## +# Copyright (c) 2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +import os + +def compile_c_and_execute(source_file,include_flags, link_flags): + """Compile C @p source_file with @p include_flags and @p link_flags, + run and return the output. + """ + cc = which('cc') + flags = include_flags + flags.extend([source_file]) + cc('-c', *flags) + name = os.path.splitext(os.path.basename(source_file))[0] + cc('-o', "check", "%s.o" % name, + *link_flags) + + check = Executable('./check') + return check(return_output=True) + + +def compare_output(current_output, blessed_output): + """Compare blessed and current output of executables.""" + if not (current_output == blessed_output): + print "Produced output does not match expected output." + print "Expected output:" + print '-'*80 + print blessed_output + print '-'*80 + print "Produced output:" + print '-'*80 + print current_output + print '-'*80 + raise RuntimeError("Ouput check failed. See spack_output.log for details") + + +def compare_output_file(current_output, blessed_output_file): + """Same as above, but when the blessed output is given as a file.""" + with open(blessed_output_file, 'r') as f: + blessed_output = f.read() + + compare_output(current_output,blessed_output) diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index d147533491..47b30181a8 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -1,4 +1,5 @@ from spack import * +from spack.package_test import * import sys import os import shutil @@ -94,42 +95,18 @@ class Openblas(Package): self.spec.lapack_shared_lib = self.spec.blas_shared_lib def check_install(self, spec): - # TODO: Pull this out to the framework function which recieves a pair of xyz.c and xyz.output - print "Checking Openblas installation..." source_file = join_path(os.path.dirname(self.module.__file__), 'test_cblas_dgemm.c') - output_file = join_path(os.path.dirname(self.module.__file__), - 'test_cblas_dgemm.output') + blessed_file = join_path(os.path.dirname(self.module.__file__), + 'test_cblas_dgemm.output') - with open(output_file, 'r') as f: - expected = f.read() - - cc = which('cc') - cc('-c', "-I%s" % join_path(spec.prefix, "include"), source_file) + include_flags = ["-I%s" % join_path(spec.prefix, "include")] link_flags = ["-L%s" % join_path(spec.prefix, "lib"), "-llapack", "-lblas", - "-lpthread" - ] + "-lpthread"] if '+openmp' in spec: link_flags.extend([self.compiler.openmp_flag]) - cc('-o', "check", "test_cblas_dgemm.o", - *link_flags) - - try: - check = Executable('./check') - output = check(return_output=True) - except: - output = "" - success = output == expected - if not success: - print "Produced output does not match expected output." - print "Expected output:" - print '-'*80 - print expected - print '-'*80 - print "Produced output:" - print '-'*80 - print output - print '-'*80 - raise RuntimeError("Openblas install check failed") + + output = compile_c_and_execute(source_file,include_flags,link_flags) + compare_output_file(output,blessed_file) -- cgit v1.2.3-70-g09d2 From 361b6290341e1248b3c5a5c3895b03610cb7ae3f Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 11 May 2016 15:09:48 +0200 Subject: openblas: make unit test more robust by outputing only the solution vector from dgesv --- var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.c | 2 +- var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.output | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.c b/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.c index 3813a23b69..2cb90fb883 100644 --- a/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.c +++ b/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.c @@ -43,7 +43,7 @@ int main(void) { int ldb = 3; dgesv_(&n,&nrhs, &m[0], &lda, ipiv, &x[0], &ldb, &info); for (i=0; i<3; ++i) - printf("%5.1f %3d\n", x[i], ipiv[i]); + printf("%5.1f\n", x[i]); return 0; } diff --git a/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.output b/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.output index 9c235e314f..01404462c4 100644 --- a/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.output +++ b/var/spack/repos/builtin/packages/openblas/test_cblas_dgemm.output @@ -7,6 +7,6 @@ 5.000000 -1.000000 3.000000 - -0.3 1 - 3.0 1499101120 - -3.0 32767 + -0.3 + 3.0 + -3.0 -- cgit v1.2.3-70-g09d2 From b215b19cae1eaf665dfbec4e20c36497c57dd182 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 11 May 2016 15:42:48 +0200 Subject: modules : added docs --- lib/spack/docs/basic_usage.rst | 272 ++++++++++++++++++++++++++++++++++++----- lib/spack/spack/package.py | 2 +- 2 files changed, 242 insertions(+), 32 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 29791d98c4..ce79ee0929 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -788,7 +788,7 @@ versions are now filtered out. .. _shell-support: -Environment modules +Integration with module systems ------------------------------- .. note:: @@ -798,42 +798,50 @@ Environment modules interface and/or generated module names may change in future versions. -Spack provides some limited integration with environment module -systems to make it easier to use the packages it provides. +Spack provides some integration with +`Environment Modules `_ +and `Dotkit `_ to make +it easier to use the packages it installed. + Installing Environment Modules -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In order to use Spack's generated environment modules, you must have installed the *Environment Modules* package. On many Linux -distributions, this can be installed from the vendor's repository. -For example: ```yum install environment-modules`` -(Fedora/RHEL/CentOS). If your Linux distribution does not have -Environment Modules, you can get it with Spack: +distributions, this can be installed from the vendor's repository: + +.. code-block:: sh -1. Install with:: + yum install environment-modules # (Fedora/RHEL/CentOS) + apt-get install environment-modules # (Ubuntu/Debian) + +If your Linux distribution does not have +Environment Modules, you can get it with Spack: .. code-block:: sh spack install environment-modules -2. Activate with:: -Add the following two lines to your ``.bashrc`` profile (or similar): +In this case to activate it automatically you need to add the following two +lines to your ``.bashrc`` profile (or similar): .. code-block:: sh MODULES_HOME=`spack location -i environment-modules` source ${MODULES_HOME}/Modules/init/bash -In case you use a Unix shell other than bash, substitute ``bash`` by -the appropriate file in ``${MODULES_HOME}/Modules/init/``. +If you use a Unix shell other than ``bash``, modify the commands above +accordingly and source the appropriate file in +``${MODULES_HOME}/Modules/init/``. -Spack and Environment Modules -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. TODO : Add a similar section on how to install dotkit ? +Spack and module systems +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can enable shell support by sourcing some files in the ``/share/spack`` directory. @@ -841,7 +849,7 @@ For ``bash`` or ``ksh``, run: .. code-block:: sh - . $SPACK_ROOT/share/spack/setup-env.sh + . ${SPACK_ROOT}/share/spack/setup-env.sh For ``csh`` and ``tcsh`` run: @@ -853,17 +861,19 @@ For ``csh`` and ``tcsh`` run: You can put the above code in your ``.bashrc`` or ``.cshrc``, and Spack's shell support will be available on the command line. -When you install a package with Spack, it automatically generates an -environment module that lets you add the package to your environment. +When you install a package with Spack, it automatically generates a module file +that lets you add the package to your environment. -Currently, Spack supports the generation of `TCL Modules +Currently, Spack supports the generation of `Environment Modules `_ and `Dotkit `_. Generated module files for each of these systems can be found in these directories: - * ``$SPACK_ROOT/share/spack/modules`` - * ``$SPACK_ROOT/share/spack/dotkit`` +.. code-block:: sh + + ${SPACK_ROOT}/share/spack/modules + ${SPACK_ROOT}/share/spack/dotkit The directories are automatically added to your ``MODULEPATH`` and ``DK_NODE`` environment variables when you enable Spack's `shell @@ -919,8 +929,7 @@ of installed packages. The names here should look familiar, they're the same ones from ``spack find``. You *can* use the names here directly. For example, -you could type either of these commands to load the callpath module -(assuming dotkit and modules are installed): +you could type either of these commands to load the callpath module: .. code-block:: sh @@ -935,7 +944,7 @@ easy to type. Luckily, Spack has its own interface for using modules and dotkits. You can use the same spec syntax you're used to: ========================= ========================== - Modules Dotkit + Environment Modules Dotkit ========================= ========================== ``spack load `` ``spack use `` ``spack unload `` ``spack unuse `` @@ -1002,15 +1011,216 @@ used ``gcc``. You could therefore just type: To identify just the one built with the Intel compiler. +Module files generation and customization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Environment Modules and Dotkit files are generated when packages are installed, +and are placed in the following directories under the Spack root: + +.. code-block:: sh + + ${SPACK_ROOT}/share/spack/modules + ${SPACK_ROOT}/share/spack/dotkit + +The content that gets written in each module file can be customized in two ways: + + 1. overriding part of the ``spack.Package`` API within a ``package.py`` + 2. writing dedicated configuration files + +Override ``Package`` API +^^^^^^^^^^^^^^^^^^^^^^^^ +There are currently two methods in ``spack.Package`` that may affect the content +of module files: + +.. code-block:: python + + def setup_environment(self, spack_env, run_env): + """Set up the compile and runtime environments for a package.""" + pass + +.. code-block:: python + + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + """Set up the environment of packages that depend on this one""" + pass + +As briefly stated in the comments, the first method lets you customize the +module file content for the package you are currently writing, the second +allows for modifications to your dependees module file. In both cases one +needs to fill ``run_env`` with the desired list of environment modifications. + +Example : ``builtin/packages/python/package.py`` +"""""""""""""""""""""""""""""""""""""""""""""""" + +The ``python`` package that comes with the ``builtin`` Spack repository +overrides ``setup_dependent_environment`` in the following way: + +.. code-block:: python + + def setup_dependent_environment(self, spack_env, run_env, extension_spec): + # ... + if extension_spec.package.extends(self.spec): + run_env.prepend_path('PYTHONPATH', os.path.join(extension_spec.prefix, self.site_packages_dir)) + +to insert the appropriate ``PYTHONPATH`` modifications in the module +files of python packages. + +Configuration files +^^^^^^^^^^^^^^^^^^^ + +Another way of modifying the content of module files is writing a +``modules.yaml`` configuration file. Following usual Spack conventions, this +file can be placed either at *site* or *user* scope. + +The default site configuration reads: + + .. literalinclude:: ../../../etc/spack/modules.yaml + :language: yaml + +It basically inspects the installation prefixes for the +existence of a few folders and, if they exist, it prepends a path to a given +list of environment variables. + +For each module system that can be enabled a finer configuration is possible: + +.. code-block:: yaml + + modules: + tcl: + # contains environment modules specific customizations + dotkit: + # contains dotkit specific customizations -Regenerating Module files -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The structure under the ``tcl`` and ``dotkit`` keys is almost equal, and will +be showcased in the following by some examples. -Module and dotkit files are generated when packages are installed, and -are placed in the following directories under the Spack root: +Select module files by spec constraints +""""""""""""""""""""""""""""""""""""""" +Using spec syntax it's possible to have different customizations for different +groups of module files. - * ``$SPACK_ROOT/share/spack/modules`` - * ``$SPACK_ROOT/share/spack/dotkit`` +Considering : + +.. code-block:: yaml + + modules: + tcl: + all: # Default addition for every package + environment: + set: + BAR: 'bar' + ^openmpi:: # A double ':' overrides previous rules + environment: + set: + BAR: 'baz' + zlib: + environment: + prepend_path: + LD_LIBRARY_PATH: 'foo' + zlib%gcc@4.8: + environment: + unset: + - FOOBAR + +what will happen is that: + + - every module file will set ``BAR=bar`` + - unless the associated spec satisfies ``^openmpi`` in which case ``BAR=baz`` + - any spec that satisfies ``zlib`` will additionally prepend ``foo`` to ``LD_LIBRARY_PATH`` + - any spec that satisfies ``zlib%gcc@4.8`` will additionally unset ``FOOBAR`` + +.. note:: + Order does matter + The modifications associated with the ``all`` keyword are always evaluated + first, no matter where they appear in the configuration file. All the other + spec constraints are instead evaluated top to bottom. + +Filter modifications out of module files +"""""""""""""""""""""""""""""""""""""""" + +Modifications to certain environment variables in module files are generated by +default. Suppose you would like to avoid having ``CPATH`` and ``LIBRARY_PATH`` +modified by your dotkit modules. Then : + +.. code-block:: yaml + + modules: + dotkit: + all: + filter: + environment_blacklist: ['CPATH', 'LIBRARY_PATH'] # Exclude changes to any of these variables + +will generate dotkit module files that will not contain modifications to either +``CPATH`` or ``LIBRARY_PATH`` and environment module files that instead will +contain those modifications. + +Autoload dependencies +""""""""""""""""""""" + +The following lines in ``modules.yaml``: + +.. code-block:: yaml + + modules: + tcl: + all: + autoload: 'direct' + +will produce environment module files that will automatically load their direct +dependencies. + +.. note:: + Allowed values for ``autoload`` statements + Allowed values for ``autoload`` statements are either ``none``, ``direct`` + or ``all``. In ``tcl`` configuration it is possible to use the option + ``prerequisites`` that accepts the same values and will add ``prereq`` + statements instead of automatically loading other modules. + +Blacklist or whitelist the generation of specific module files +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Sometimes it is desirable not to generate module files, a common use case being +not providing the users with software built using the system compiler. + +A configuration file like: + +.. code-block:: yaml + + modules: + tcl: + whitelist: ['gcc', 'llvm'] # Whitelist will have precedence over blacklist + blacklist: ['%gcc@4.4.7'] # Assuming gcc@4.4.7 is the system compiler + +will skip module file generation for anything that satisfies ``%gcc@4.4.7``, +with the exception of specs that satisfy ``gcc`` or ``llvm``. + +Customize the naming scheme and insert conflicts +"""""""""""""""""""""""""""""""""""""""""""""""" + +A configuration file like: + +.. code-block:: yaml + + modules: + tcl: + naming_scheme: '{name}/{version}-{compiler.name}-{compiler.version}' + all: + conflict: ['{name}', 'intel/14.0.1'] + +will create module files that will conflict with ``intel/14.0.1`` and with the +base directory of the same module, effectively preventing the possibility to +load two or more versions of the same software at the same time. + +.. note:: + Tokens available for the naming scheme + currently only the tokens shown in the example are available to construct + the naming scheme + +.. note:: + The ``conflict`` option is ``tcl`` specific + +Regenerating module files +^^^^^^^^^^^^^^^^^^^^^^^^^ Sometimes you may need to regenerate the modules files. For example, if newer, fancier module support is added to Spack at some later date, @@ -1020,7 +1230,7 @@ new features. .. _spack-module: ``spack module refresh`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +"""""""""""""""""""""""" Running ``spack module refresh`` will remove the ``share/spack/modules`` and ``share/spack/dotkit`` directories, then diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 4065553131..3626a574c8 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1006,7 +1006,7 @@ class Package(object): fromlist=[self.__class__.__name__]) def setup_environment(self, spack_env, run_env): - """Set up the compile and runtime environemnts for a package. + """Set up the compile and runtime environments for a package. `spack_env` and `run_env` are `EnvironmentModifications` objects. Package authors can call methods on them to alter -- cgit v1.2.3-70-g09d2 From 22bb0562fea525afb329d5710970785189d3af63 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 11 May 2016 16:09:47 +0200 Subject: Revert "flake8 : fixed all issues?" This reverts commit 71e49e289a849b8aaa4f0d9a195d07569051ca88. --- lib/spack/spack/cmd/module.py | 16 ++- lib/spack/spack/config.py | 204 +++++++++++++++------------------------ lib/spack/spack/environment.py | 51 ++++------ lib/spack/spack/modules.py | 202 ++++++++++++++++---------------------- lib/spack/spack/test/__init__.py | 69 +++++++++---- lib/spack/spack/test/modules.py | 42 +++----- 6 files changed, 254 insertions(+), 330 deletions(-) diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index cfe59c8d98..f996f4eb84 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -32,21 +32,18 @@ from llnl.util.filesystem import mkdirp from spack.modules import module_types from spack.util.string import * -description = "Manipulate modules and dotkits." +description ="Manipulate modules and dotkits." def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='module_command') - sp.add_parser('refresh', help='Regenerate all module files.') + refresh_parser = sp.add_parser('refresh', help='Regenerate all module files.') find_parser = sp.add_parser('find', help='Find module files for packages.') - find_parser.add_argument('module_type', - help="Type of module to find file for. [" + - '|'.join(module_types) + "]") - find_parser.add_argument('spec', - nargs='+', - help='spec to find a module file for.') + find_parser.add_argument( + 'module_type', help="Type of module to find file for. [" + '|'.join(module_types) + "]") + find_parser.add_argument('spec', nargs='+', help='spec to find a module file for.') def module_find(mtype, spec_array): @@ -56,8 +53,7 @@ def module_find(mtype, spec_array): should type to use that package's module. """ if mtype not in module_types: - tty.die("Invalid module type: '%s'. Options are %s" % - (mtype, comma_or(module_types))) + tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types))) specs = spack.cmd.parse_specs(spec_array) if len(specs) > 1: diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 684a420b3b..6ddf07776b 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -134,6 +134,8 @@ from yaml.error import MarkedYAMLError # Hacked yaml for configuration files preserves line numbers. import spack.util.spack_yaml as syaml + + """Dict from section names -> schema for that section.""" section_schemas = { 'compilers': { @@ -147,31 +149,25 @@ section_schemas = { 'default': {}, 'additionalProperties': False, 'patternProperties': { - r'\w[\w-]*': { # architecture + r'\w[\w-]*': { # architecture 'type': 'object', 'additionalProperties': False, 'patternProperties': { - r'\w[\w-]*@\w[\w-]*': { # compiler spec + r'\w[\w-]*@\w[\w-]*': { # compiler spec 'type': 'object', 'additionalProperties': False, 'required': ['cc', 'cxx', 'f77', 'fc'], 'properties': { - 'cc': {'anyOf': [{'type': 'string'}, - {'type': 'null'}]}, - 'cxx': {'anyOf': [{'type': 'string'}, - {'type': 'null'}]}, - 'f77': {'anyOf': [{'type': 'string'}, - {'type': 'null'}]}, - 'fc': {'anyOf': [{'type': 'string'}, - {'type': 'null'}]}, - }, - }, - }, - }, - }, - }, - }, - }, + 'cc': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cxx': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'f77': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'fc': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + },},},},},},},}, + 'mirrors': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack mirror configuration file schema', @@ -184,12 +180,8 @@ section_schemas = { 'additionalProperties': False, 'patternProperties': { r'\w[\w-]*': { - 'type': 'string' - }, - }, - }, - }, - }, + 'type': 'string'},},},},}, + 'repos': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack repository configuration file schema', @@ -200,11 +192,8 @@ section_schemas = { 'type': 'array', 'default': [], 'items': { - 'type': 'string' - }, - }, - }, - }, + 'type': 'string'},},},}, + 'packages': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack package configuration file schema', @@ -216,48 +205,39 @@ section_schemas = { 'default': {}, 'additionalProperties': False, 'patternProperties': { - r'\w[\w-]*': { # package name + r'\w[\w-]*': { # package name 'type': 'object', 'default': {}, 'additionalProperties': False, 'properties': { 'version': { - 'type': 'array', - 'default': [], - 'items': {'anyOf': [{'type': 'string'}, - {'type': 'number'}]} - }, # version strings + 'type' : 'array', + 'default' : [], + 'items' : { 'anyOf' : [ { 'type' : 'string' }, + { 'type' : 'number'}]}}, #version strings 'compiler': { - 'type': 'array', - 'default': [], - 'items': {'type': 'string'} - }, # compiler specs + 'type' : 'array', + 'default' : [], + 'items' : { 'type' : 'string' } }, #compiler specs 'buildable': { - 'type': 'boolean', + 'type': 'boolean', 'default': True, - }, + }, 'providers': { - 'type': 'object', + 'type': 'object', 'default': {}, 'additionalProperties': False, 'patternProperties': { r'\w[\w-]*': { - 'type': 'array', - 'default': [], - 'items': {'type': 'string'}, - }, - }, - }, + 'type' : 'array', + 'default' : [], + 'items' : { 'type' : 'string' },},},}, 'paths': { - 'type': 'object', - 'default': {}, + 'type' : 'object', + 'default' : {}, } - }, - }, - }, - }, - }, - }, + },},},},},}, + 'modules': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack module file configuration file schema', @@ -303,22 +283,17 @@ section_schemas = { } }, 'autoload': {'$ref': '#/definitions/dependency_selection'}, - 'prerequisites': - {'$ref': '#/definitions/dependency_selection'}, + 'prerequisites': {'$ref': '#/definitions/dependency_selection'}, 'conflict': {'$ref': '#/definitions/array_of_strings'}, 'environment': { 'type': 'object', 'default': {}, 'additionalProperties': False, 'properties': { - 'set': - {'$ref': '#/definitions/dictionary_of_strings'}, - 'unset': - {'$ref': '#/definitions/array_of_strings'}, - 'prepend_path': - {'$ref': '#/definitions/dictionary_of_strings'}, - 'append_path': - {'$ref': '#/definitions/dictionary_of_strings'} + 'set': {'$ref': '#/definitions/dictionary_of_strings'}, + 'unset': {'$ref': '#/definitions/array_of_strings'}, + 'prepend_path': {'$ref': '#/definitions/dictionary_of_strings'}, + 'append_path': {'$ref': '#/definitions/dictionary_of_strings'} } } } @@ -329,20 +304,15 @@ section_schemas = { 'anyOf': [ { 'properties': { - 'whitelist': - {'$ref': '#/definitions/array_of_strings'}, - 'blacklist': - {'$ref': '#/definitions/array_of_strings'}, + 'whitelist': {'$ref': '#/definitions/array_of_strings'}, + 'blacklist': {'$ref': '#/definitions/array_of_strings'}, 'naming_scheme': { - 'type': - 'string' # Can we be more specific here? + 'type': 'string' # Can we be more specific here? } } }, { - 'patternProperties': - {r'\w[\w-]*': - {'$ref': '#/definitions/module_file_configuration'}} + 'patternProperties': {r'\w[\w-]*': {'$ref': '#/definitions/module_file_configuration'}} } ] } @@ -356,8 +326,7 @@ section_schemas = { 'prefix_inspections': { 'type': 'object', 'patternProperties': { - r'\w[\w-]*': - { # path to be inspected (relative to prefix) + r'\w[\w-]*': { # path to be inspected for existence (relative to prefix) '$ref': '#/definitions/array_of_strings' } } @@ -372,15 +341,13 @@ section_schemas = { }, 'tcl': { 'allOf': [ - {'$ref': '#/definitions/module_type_configuration' - }, # Base configuration + {'$ref': '#/definitions/module_type_configuration'}, # Base configuration {} # Specific tcl extensions ] }, 'dotkit': { 'allOf': [ - {'$ref': '#/definitions/module_type_configuration' - }, # Base configuration + {'$ref': '#/definitions/module_type_configuration'}, # Base configuration {} # Specific dotkit extensions ] }, @@ -389,6 +356,7 @@ section_schemas = { }, }, } + """OrderedDict of config scopes keyed by name. Later scopes will override earlier scopes. """ @@ -398,13 +366,12 @@ config_scopes = OrderedDict() def validate_section_name(section): """Raise a ValueError if the section is not a valid section.""" if section not in section_schemas: - raise ValueError("Invalid config section: '%s'. Options are %s" % - (section, section_schemas)) + raise ValueError("Invalid config section: '%s'. Options are %s" + % (section, section_schemas)) def extend_with_default(validator_class): - """Add support for the 'default' attribute for - properties and patternProperties + """Add support for the 'default' attribute for properties and patternProperties. jsonschema does not handle this out of the box -- it only validates. This allows us to set default values for configs @@ -413,15 +380,13 @@ def extend_with_default(validator_class): """ validate_properties = validator_class.VALIDATORS["properties"] - validate_pattern_properties = validator_class.VALIDATORS[ - "patternProperties"] + validate_pattern_properties = validator_class.VALIDATORS["patternProperties"] def set_defaults(validator, properties, instance, schema): for property, subschema in properties.iteritems(): if "default" in subschema: instance.setdefault(property, subschema["default"]) - for err in validate_properties(validator, properties, instance, - schema): + for err in validate_properties(validator, properties, instance, schema): yield err def set_pp_defaults(validator, properties, instance, schema): @@ -432,19 +397,17 @@ def extend_with_default(validator_class): if re.match(property, key) and val is None: instance[key] = subschema["default"] - for err in validate_pattern_properties(validator, properties, instance, - schema): + for err in validate_pattern_properties(validator, properties, instance, schema): yield err return validators.extend(validator_class, { - "properties": set_defaults, - "patternProperties": set_pp_defaults + "properties" : set_defaults, + "patternProperties" : set_pp_defaults }) DefaultSettingValidator = extend_with_default(Draft4Validator) - def validate_section(data, schema): """Validate data read in from a Spack YAML file. @@ -466,9 +429,9 @@ class ConfigScope(object): """ def __init__(self, name, path): - self.name = name # scope name. - self.path = path # path to directory containing configs. - self.sections = {} # sections read from config files. + self.name = name # scope name. + self.path = path # path to directory containing configs. + self.sections = {} # sections read from config files. # Register in a dict of all ConfigScopes # TODO: make this cleaner. Mocking up for testing is brittle. @@ -479,14 +442,16 @@ class ConfigScope(object): validate_section_name(section) return os.path.join(self.path, "%s.yaml" % section) + def get_section(self, section): - if section not in self.sections: - path = self.get_section_filename(section) + if not section in self.sections: + path = self.get_section_filename(section) schema = section_schemas[section] - data = _read_config_file(path, schema) + data = _read_config_file(path, schema) self.sections[section] = data return self.sections[section] + def write_section(self, section): filename = self.get_section_filename(section) data = self.get_section(section) @@ -498,8 +463,8 @@ class ConfigScope(object): except jsonschema.ValidationError as e: raise ConfigSanityError(e, data) except (yaml.YAMLError, IOError) as e: - raise ConfigFileError("Error writing to config file: '%s'" % - str(e)) + raise ConfigFileError("Error writing to config file: '%s'" % str(e)) + def clear(self): """Empty cached config information.""" @@ -531,8 +496,8 @@ def validate_scope(scope): return config_scopes[scope] else: - raise ValueError("Invalid config scope: '%s'. Must be one of %s" % - (scope, config_scopes.keys())) + raise ValueError("Invalid config scope: '%s'. Must be one of %s" + % (scope, config_scopes.keys())) def _read_config_file(filename, schema): @@ -558,12 +523,12 @@ def _read_config_file(filename, schema): return data except MarkedYAMLError as e: - raise ConfigFileError("Error parsing yaml%s: %s" % - (str(e.context_mark), e.problem)) + raise ConfigFileError( + "Error parsing yaml%s: %s" % (str(e.context_mark), e.problem)) except IOError as e: - raise ConfigFileError("Error reading configuration file %s: %s" % - (filename, str(e))) + raise ConfigFileError( + "Error reading configuration file %s: %s" % (filename, str(e))) def clear_config_caches(): @@ -586,7 +551,6 @@ def _merge_yaml(dest, source): parent instead of merging. """ - def they_are(t): return isinstance(dest, t) and isinstance(source, t) @@ -607,7 +571,7 @@ def _merge_yaml(dest, source): # Source dict is merged into dest. elif they_are(dict): for sk, sv in source.iteritems(): - if sk not in dest: + if not sk in dest: dest[sk] = copy.copy(sv) else: dest[sk] = _merge_yaml(dest[sk], source[sk]) @@ -689,7 +653,7 @@ def print_section(section): data = syaml.syaml_dict() data[section] = get_config(section) syaml.dump(data, stream=sys.stdout, default_flow_style=False) - except (yaml.YAMLError, IOError): + except (yaml.YAMLError, IOError) as e: raise ConfigError("Error reading configuration: %s" % section) @@ -719,22 +683,15 @@ def is_spec_buildable(spec): """Return true if the spec pkgspec is configured as buildable""" allpkgs = get_config('packages') name = spec.name - if name not in allpkgs: + if not spec.name in allpkgs: return True - if 'buildable' not in allpkgs[name]: + if not 'buildable' in allpkgs[spec.name]: return True return allpkgs[spec.name]['buildable'] -class ConfigError(SpackError): - - pass - - -class ConfigFileError(ConfigError): - - pass - +class ConfigError(SpackError): pass +class ConfigFileError(ConfigError): pass def get_path(path, data): if path: @@ -742,10 +699,8 @@ def get_path(path, data): else: return data - class ConfigFormatError(ConfigError): """Raised when a configuration format does not match its schema.""" - def __init__(self, validation_error, data): # Try to get line number from erroneous instance and its parent instance_mark = getattr(validation_error.instance, '_start_mark', None) @@ -778,6 +733,5 @@ class ConfigFormatError(ConfigError): message = '%s: %s' % (location, validation_error.message) super(ConfigError, self).__init__(message) - class ConfigSanityError(ConfigFormatError): """Same as ConfigFormatError, raised when config is written by Spack.""" diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 9748b4033a..92ab4e6bea 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -26,8 +26,7 @@ class SetEnv(NameValueModifier): class UnsetEnv(NameModifier): def execute(self): - # Avoid throwing if the variable was not set - os.environ.pop(self.name, None) + os.environ.pop(self.name, None) # Avoid throwing if the variable was not set class SetPath(NameValueModifier): @@ -56,9 +55,7 @@ class RemovePath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split(':') if environment_value else [] - directories = [os.path.normpath(x) - for x in directories - if x != os.path.normpath(self.value)] + directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.value)] os.environ[self.name] = ':'.join(directories) @@ -66,8 +63,7 @@ class EnvironmentModifications(object): """ Keeps track of requests to modify the current environment. - Each call to a method to modify the environment stores the extra - information on the caller in the request: + Each call to a method to modify the environment stores the extra information on the caller in the request: - 'filename' : filename of the module where the caller is defined - 'lineno': line number where the request occurred - 'context' : line of code that issued the request that failed @@ -75,10 +71,10 @@ class EnvironmentModifications(object): def __init__(self, other=None): """ - Initializes a new instance, copying commands from other if not None + Initializes a new instance, copying commands from other if it is not None Args: - other: another instance of EnvironmentModifications + other: another instance of EnvironmentModifications from which (optional) """ self.env_modifications = [] if other is not None: @@ -97,7 +93,7 @@ class EnvironmentModifications(object): @staticmethod def _check_other(other): if not isinstance(other, EnvironmentModifications): - raise TypeError('not an instance of EnvironmentModifications') + raise TypeError('other must be an instance of EnvironmentModifications') def _get_outside_caller_attributes(self): stack = inspect.stack() @@ -105,10 +101,12 @@ class EnvironmentModifications(object): _, filename, lineno, _, context, index = stack[2] context = context[index].strip() except Exception: - filename = 'unknown file' - lineno = 'unknown line' - context = 'unknown context' - args = {'filename': filename, 'lineno': lineno, 'context': context} + filename, lineno, context = 'unknown file', 'unknown line', 'unknown context' + args = { + 'filename': filename, + 'lineno': lineno, + 'context': context + } return args def set(self, name, value, **kwargs): @@ -172,7 +170,7 @@ class EnvironmentModifications(object): def remove_path(self, name, path, **kwargs): """ - Stores in the current object a request to remove a path from a list + Stores in the current object a request to remove a path from a path list Args: name: name of the path list in the environment @@ -187,8 +185,7 @@ class EnvironmentModifications(object): Returns a dict of the modifications grouped by variable name Returns: - dict mapping the environment variable name to the modifications - to be done on it + dict mapping the environment variable name to the modifications to be done on it """ modifications = collections.defaultdict(list) for item in self: @@ -206,8 +203,7 @@ class EnvironmentModifications(object): Applies the modifications and clears the list """ modifications = self.group_by_name() - # Apply the modifications to the environment variables one variable - # at a time + # Apply the modifications to the environment variables one variable at a time for name, actions in sorted(modifications.items()): for x in actions: x.execute() @@ -228,17 +224,13 @@ def concatenate_paths(paths): def set_or_unset_not_first(variable, changes, errstream): """ - Check if we are going to set or unset something after other modifications - have already been requested + Check if we are going to set or unset something after other modifications have already been requested """ - indexes = [ii - for ii, item in enumerate(changes) - if ii != 0 and type(item) in [SetEnv, UnsetEnv]] + indexes = [ii for ii, item in enumerate(changes) if ii != 0 and type(item) in [SetEnv, UnsetEnv]] if indexes: good = '\t \t{context} at {filename}:{lineno}' nogood = '\t--->\t{context} at {filename}:{lineno}' - message = 'Suspicious requests to set or unset the variable \'{var}\' found' # NOQA: ignore=E501 - errstream(message.format(var=variable)) + errstream('Suspicious requests to set or unset the variable \'{var}\' found'.format(var=variable)) for ii, item in enumerate(changes): print_format = nogood if ii in indexes else good errstream(print_format.format(**item.args)) @@ -246,8 +238,8 @@ def set_or_unset_not_first(variable, changes, errstream): def validate(env, errstream): """ - Validates the environment modifications to check for the presence of - suspicious patterns. Prompts a warning for everything that was found + Validates the environment modifications to check for the presence of suspicious patterns. Prompts a warning for + everything that was found Current checks: - set or unset variables after other changes on the same variable @@ -262,8 +254,7 @@ def validate(env, errstream): def filter_environment_blacklist(env, variables): """ - Generator that filters out any change to environment variables present in - the input list + Generator that filters out any change to environment variables present in the input list Args: env: list of environment modifications diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 0dc6f06f55..ffed469b20 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -23,34 +23,36 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## """ -This module contains code for creating environment modules, which can include -dotkits, tcl modules, lmod, and others. +This module contains code for creating environment modules, which can include dotkits, tcl modules, lmod, and others. -The various types of modules are installed by post-install hooks and removed -after an uninstall by post-uninstall hooks. This class consolidates the logic -for creating an abstract description of the information that module systems -need. +The various types of modules are installed by post-install hooks and removed after an uninstall by post-uninstall hooks. +This class consolidates the logic for creating an abstract description of the information that module systems need. +Currently that includes a number of directories to be appended to paths in the user's environment: -This module also includes logic for coming up with unique names for the module -files so that they can be found by the various shell-support files in -$SPACK/share/spack/setup-env.*. + * /bin directories to be appended to PATH + * /lib* directories for LD_LIBRARY_PATH + * /include directories for CPATH + * /man* and /share/man* directories for MANPATH + * the package prefix for CMAKE_PREFIX_PATH -Each hook implements the logic for writing its specific type of module. +This module also includes logic for coming up with unique names for the module files so that they can be found by the +various shell-support files in $SPACK/share/spack/setup-env.*. + +Each hook in hooks/ implements the logic for writing its specific type of module file. """ import copy import datetime import os import os.path import re -import string import textwrap +import string import llnl.util.tty as tty import spack import spack.config from llnl.util.filesystem import join_path, mkdirp -from spack.build_environment import parent_class_modules -from spack.build_environment import set_module_variables_for_package +from spack.build_environment import parent_class_modules, set_module_variables_for_package from spack.environment import * __all__ = ['EnvModule', 'Dotkit', 'TclModule'] @@ -65,26 +67,30 @@ def print_help(): """ For use by commands to tell user how to activate shell support. """ - tty.msg("This command requires spack's shell integration.", "", + tty.msg("This command requires spack's shell integration.", + "", "To initialize spack's shell commands, you must run one of", "the commands below. Choose the right command for your shell.", - "", "For bash and zsh:", - " . %s/setup-env.sh" % spack.share_path, "", - "For csh and tcsh:", " setenv SPACK_ROOT %s" % spack.prefix, - " source %s/setup-env.csh" % spack.share_path, "") + "", + "For bash and zsh:", + " . %s/setup-env.sh" % spack.share_path, + "", + "For csh and tcsh:", + " setenv SPACK_ROOT %s" % spack.prefix, + " source %s/setup-env.csh" % spack.share_path, + "") def inspect_path(prefix): """ - Inspects the prefix of an installation to search for common layouts. - Issues a request to modify the environment when an item is found. + Inspects the prefix of an installation to search for common layouts. Issues a request to modify the environment + accordingly when an item is found. Args: prefix: prefix of the installation Returns: - instance of EnvironmentModifications containing the requested - modifications + instance of EnvironmentModifications containing the requested modifications """ env = EnvironmentModifications() # Inspect the prefix to check for the existence of common directories @@ -99,22 +105,18 @@ def inspect_path(prefix): def dependencies(spec, request='all'): """ - Returns the list of dependent specs for a given spec, according to the - given request + Returns the list of dependent specs for a given spec, according to the given request Args: spec: target spec request: either 'none', 'direct' or 'all' Returns: - empty list if 'none', direct dependency list if 'direct', all - dependencies if 'all' + empty list if 'none', direct dependency list if 'direct', all dependencies if 'all' """ if request not in ('none', 'direct', 'all'): - message = "Wrong value for argument 'request' : " - message += "should be one of ('none', 'direct', 'all')" - message += " [current value is '{0}']" - raise tty.error(message.format(request)) + raise tty.error("Wrong value for argument 'request' : should be one of ('none', 'direct', 'all') " + " [current value is '%s']" % request) if request == 'none': return [] @@ -122,19 +124,12 @@ def dependencies(spec, request='all'): if request == 'direct': return [xx for _, xx in spec.dependencies.items()] - # FIXME : during module file creation nodes seem to be visited - # FIXME : multiple times even if cover='nodes' is given. This work around - # FIXME : permits to get a unique list of spec anyhow. Maybe we miss a - # FIXME : merge step among nodes that refer to the same package? + # FIXME : during module file creation nodes seem to be visited multiple times even if cover='nodes' + # FIXME : is given. This work around permits to get a unique list of spec anyhow. + # FIXME : Possibly we miss a merge step among nodes that refer to the same package. seen = set() seen_add = seen.add - l = [xx - for xx in sorted( - spec.traverse(order='post', - depth=True, - cover='nodes', - root=False), - reverse=True)] + l = [xx for xx in sorted(spec.traverse(order='post', depth=True, cover='nodes', root=False), reverse=True)] return [xx for ii, xx in l if not (xx in seen or seen_add(xx))] @@ -151,8 +146,7 @@ def update_dictionary_extending_lists(target, update): def parse_config_options(module_generator): """ - Parse the configuration file and returns a bunch of items that will be - needed during module file generation + Parse the configuration file and returns a bunch of items that will be needed during module file generation Args: module_generator: module generator for a given spec @@ -160,14 +154,11 @@ def parse_config_options(module_generator): Returns: autoloads: list of specs to be autoloaded prerequisites: list of specs to be marked as prerequisite - filters: list of environment variables whose modification is - blacklisted in module files - env: list of custom environment modifications to be applied in the - module file + filters: list of environment variables whose modification is blacklisted in module files + env: list of custom environment modifications to be applied in the module file """ # Get the configuration for this kind of generator - module_configuration = copy.deepcopy(CONFIGURATION.get( - module_generator.name, {})) + module_configuration = copy.deepcopy(CONFIGURATION.get(module_generator.name, {})) ##### # Merge all the rules @@ -188,12 +179,9 @@ def parse_config_options(module_generator): ##### # Automatic loading loads - module_file_actions['autoload'] = dependencies( - module_generator.spec, module_file_actions.get('autoload', 'none')) + module_file_actions['autoload'] = dependencies(module_generator.spec, module_file_actions.get('autoload', 'none')) # Prerequisites - module_file_actions['prerequisites'] = dependencies( - module_generator.spec, module_file_actions.get('prerequisites', - 'none')) + module_file_actions['prerequisites'] = dependencies(module_generator.spec, module_file_actions.get('prerequisites', 'none')) # Environment modifications environment_actions = module_file_actions.pop('environment', {}) env = EnvironmentModifications() @@ -201,7 +189,7 @@ def parse_config_options(module_generator): def process_arglist(arglist): if method == 'unset': for x in arglist: - yield (x, ) + yield (x,) else: for x in arglist.iteritems(): yield x @@ -210,20 +198,19 @@ def parse_config_options(module_generator): for args in process_arglist(arglist): getattr(env, method)(*args) - # for item in arglist: - # if method == 'unset': - # args = [item] - # else: - # args = item.split(',') - # getattr(env, method)(*args) + # for item in arglist: + # if method == 'unset': + # args = [item] + # else: + # args = item.split(',') + # getattr(env, method)(*args) return module_file_actions, env def filter_blacklisted(specs, module_name): """ - Given a sequence of specs, filters the ones that are blacklisted in the - module configuration file. + Given a sequence of specs, filters the ones that are blacklisted in the module configuration file. Args: specs: sequence of spec instances @@ -246,8 +233,7 @@ class EnvModule(object): class __metaclass__(type): def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) - if cls.name != 'env_module' and cls.name in CONFIGURATION[ - 'enable']: + if cls.name != 'env_module' and cls.name in CONFIGURATION['enable']: module_types[cls.name] = cls def __init__(self, spec=None): @@ -263,8 +249,7 @@ class EnvModule(object): # long description is the docstring with reduced whitespace. self.long_description = None if self.spec.package.__doc__: - self.long_description = re.sub(r'\s+', ' ', - self.spec.package.__doc__) + self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) @property def naming_scheme(self): @@ -286,14 +271,12 @@ class EnvModule(object): @property def use_name(self): """ - Subclasses should implement this to return the name the module command - uses to refer to the package. + Subclasses should implement this to return the name the module command uses to refer to the package. """ naming_tokens = self.tokens naming_scheme = self.naming_scheme name = naming_scheme.format(**naming_tokens) - name += '-' + self.spec.dag_hash( - ) # Always append the hash to make the module file unique + name += '-' + self.spec.dag_hash() # Always append the hash to make the module file unique # Not everybody is working on linux... parts = name.split('/') name = join_path(*parts) @@ -313,12 +296,8 @@ class EnvModule(object): @property def blacklisted(self): configuration = CONFIGURATION.get(self.name, {}) - whitelist_matches = [x - for x in configuration.get('whitelist', []) - if self.spec.satisfies(x)] - blacklist_matches = [x - for x in configuration.get('blacklist', []) - if self.spec.satisfies(x)] + whitelist_matches = [x for x in configuration.get('whitelist', []) if self.spec.satisfies(x)] + blacklist_matches = [x for x in configuration.get('blacklist', []) if self.spec.satisfies(x)] if whitelist_matches: message = '\tWHITELIST : %s [matches : ' % self.spec.cshort_spec for rule in whitelist_matches: @@ -348,8 +327,7 @@ class EnvModule(object): """ if self.blacklisted: return - tty.debug("\tWRITE : %s [%s]" % - (self.spec.cshort_spec, self.file_name)) + tty.debug("\tWRITE : %s [%s]" % (self.spec.cshort_spec, self.file_name)) module_dir = os.path.dirname(self.file_name) if not os.path.exists(module_dir): @@ -359,12 +337,11 @@ class EnvModule(object): # installation prefix env = inspect_path(self.spec.prefix) - # Let the extendee/dependency modify their extensions/dependencies - # before asking for package-specific modifications + # Let the extendee/dependency modify their extensions/dependencies before asking for + # package-specific modifications spack_env = EnvironmentModifications() - # TODO : the code down below is quite similar to - # TODO : build_environment.setup_package and needs to be factored out - # TODO : to a single place + # TODO : the code down below is quite similar to build_environment.setup_package and needs to be + # TODO : factored out to a single place for item in dependencies(self.spec, 'all'): package = self.spec[item.name].package modules = parent_class_modules(package.__class__) @@ -381,18 +358,14 @@ class EnvModule(object): # Parse configuration file module_configuration, conf_env = parse_config_options(self) env.extend(conf_env) - filters = module_configuration.get('filter', {}).get( - 'environment_blacklist', {}) + filters = module_configuration.get('filter', {}).get('environment_blacklist',{}) # Build up the module file content module_file_content = self.header - for x in filter_blacklisted( - module_configuration.pop('autoload', []), self.name): + for x in filter_blacklisted(module_configuration.pop('autoload', []), self.name): module_file_content += self.autoload(x) - for x in filter_blacklisted( - module_configuration.pop('prerequisites', []), self.name): + for x in filter_blacklisted(module_configuration.pop('prerequisites', []), self.name): module_file_content += self.prerequisite(x) - for line in self.process_environment_command( - filter_environment_blacklist(env, filters)): + for line in self.process_environment_command(filter_environment_blacklist(env, filters)): module_file_content += line for line in self.module_specific_content(module_configuration): module_file_content += line @@ -419,13 +392,10 @@ class EnvModule(object): def process_environment_command(self, env): for command in env: try: - yield self.environment_modifications_formats[type( - command)].format(**command.args) + yield self.environment_modifications_formats[type(command)].format(**command.args) except KeyError: - message = 'Cannot handle command of type {command} : skipping request' # NOQA: ignore=E501 - tty.warn(message.format(command=type(command))) - context = '{context} at {filename}:{lineno}' - tty.warn(context.format(**command.args)) + tty.warn('Cannot handle command of type {command} : skipping request'.format(command=type(command))) + tty.warn('{context} at {filename}:{lineno}'.format(**command.args)) @property def file_name(self): @@ -438,12 +408,9 @@ class EnvModule(object): if os.path.exists(mod_file): try: os.remove(mod_file) # Remove the module file - os.removedirs( - os.path.dirname(mod_file) - ) # Remove all the empty directories from the leaf up + os.removedirs(os.path.dirname(mod_file)) # Remove all the empty directories from the leaf up except OSError: - # removedirs throws OSError on first non-empty directory found - pass + pass # removedirs throws OSError on first non-empty directory found class Dotkit(EnvModule): @@ -457,12 +424,13 @@ class Dotkit(EnvModule): autoload_format = 'dk_op {module_file}\n' - default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' # NOQA: ignore=E501 + prerequisite_format = None # TODO : does something like prerequisite exist for dotkit? + + default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' @property def file_name(self): - return join_path(Dotkit.path, self.spec.architecture, - '%s.dk' % self.use_name) + return join_path(Dotkit.path, self.spec.architecture, '%s.dk' % self.use_name) @property def header(self): @@ -506,7 +474,7 @@ class TclModule(EnvModule): prerequisite_format = 'prereq {module_file}\n' - default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' # NOQA: ignore=E501 + default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' @property def file_name(self): @@ -514,10 +482,9 @@ class TclModule(EnvModule): @property def header(self): - timestamp = datetime.datetime.now() # TCL Modulefile header header = '#%Module1.0\n' - header += '## Module file created by spack (https://github.com/LLNL/spack) on %s\n' % timestamp # NOQA: ignore=E501 + header += '## Module file created by spack (https://github.com/LLNL/spack) on %s\n' % datetime.datetime.now() header += '##\n' header += '## %s\n' % self.spec.short_spec header += '##\n' @@ -542,19 +509,16 @@ class TclModule(EnvModule): f = string.Formatter() for item in conflict_format: line = 'conflict ' + item + '\n' - if len([x for x in f.parse(line) - ]) > 1: # We do have placeholder to substitute - for naming_dir, conflict_dir in zip( - self.naming_scheme.split('/'), item.split('/')): + if len([x for x in f.parse(line)]) > 1: # We do have placeholder to substitute + for naming_dir, conflict_dir in zip(self.naming_scheme.split('/'), item.split('/')): if naming_dir != conflict_dir: - message = 'conflict scheme does not match naming' - message += ' [{spec}]\n\n' + message = 'conflict scheme does not match naming scheme [{spec}]\n\n' message += 'naming scheme : "{nformat}"\n' message += 'conflict scheme : "{cformat}"\n\n' - message += '** You may want to check your `modules.yaml` configuration file **\n' # NOQA: ignore=E501 - tty.error(message.format(spec=self.spec, - nformat=self.naming_scheme, - cformat=item)) + message += '** You may want to check your `modules.yaml` configuration file **\n' + tty.error( + message.format(spec=self.spec, nformat=self.naming_scheme, cformat=item) + ) raise SystemExit('Module generation aborted.') line = line.format(**naming_tokens) yield line diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 395ca0c87a..05f58ab7b1 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -23,23 +23,53 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys - -import llnl.util.tty as tty +import unittest import nose -import spack + +from spack.test.tally_plugin import Tally from llnl.util.filesystem import join_path +import llnl.util.tty as tty from llnl.util.tty.colify import colify -from spack.test.tally_plugin import Tally + +import spack + """Names of tests to be included in Spack's test suite""" -test_names = ['versions', 'url_parse', 'url_substitution', 'packages', 'stage', - 'spec_syntax', 'spec_semantics', 'spec_dag', 'concretize', - 'multimethod', 'install', 'package_sanity', 'config', - 'directory_layout', 'pattern', 'python_version', 'git_fetch', - 'svn_fetch', 'hg_fetch', 'mirror', 'modules', 'url_extrapolate', - 'cc', 'link_tree', 'spec_yaml', 'optional_deps', - 'make_executable', 'configure_guess', 'lock', 'database', - 'namespace_trie', 'yaml', 'sbang', 'environment', - 'cmd.uninstall', 'cmd.test_install'] +test_names = ['versions', + 'url_parse', + 'url_substitution', + 'packages', + 'stage', + 'spec_syntax', + 'spec_semantics', + 'spec_dag', + 'concretize', + 'multimethod', + 'install', + 'package_sanity', + 'config', + 'directory_layout', + 'pattern', + 'python_version', + 'git_fetch', + 'svn_fetch', + 'hg_fetch', + 'mirror', + 'modules', + 'url_extrapolate', + 'cc', + 'link_tree', + 'spec_yaml', + 'optional_deps', + 'make_executable', + 'configure_guess', + 'lock', + 'database', + 'namespace_trie', + 'yaml', + 'sbang', + 'environment', + 'cmd.uninstall', + 'cmd.test_install'] def list_tests(): @@ -50,6 +80,7 @@ def list_tests(): def run(names, outputDir, verbose=False): """Run tests with the supplied names. Names should be a list. If it's empty, run ALL of Spack's tests.""" + verbosity = 1 if not verbose else 2 if not names: names = test_names @@ -64,7 +95,7 @@ def run(names, outputDir, verbose=False): tally = Tally() for test in names: module = 'spack.test.' + test - print(module) + print module tty.msg("Running test: %s" % test) @@ -74,13 +105,15 @@ def run(names, outputDir, verbose=False): xmlOutputFname = "unittests-{0}.xml".format(test) xmlOutputPath = join_path(outputDir, xmlOutputFname) runOpts += ["--with-xunit", - "--xunit-file={0}".format(xmlOutputPath)] + "--xunit-file={0}".format(xmlOutputPath)] argv = [""] + runOpts + [module] - nose.run(argv=argv, addplugins=[tally]) + result = nose.run(argv=argv, addplugins=[tally]) succeeded = not tally.failCount and not tally.errorCount - tty.msg("Tests Complete.", "%5d tests run" % tally.numberOfTestsRun, - "%5d failures" % tally.failCount, "%5d errors" % tally.errorCount) + tty.msg("Tests Complete.", + "%5d tests run" % tally.numberOfTestsRun, + "%5d failures" % tally.failCount, + "%5d errors" % tally.errorCount) if succeeded: tty.info("OK", format='g') diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index f0ff778a10..b8b0d6fc6a 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -2,18 +2,14 @@ import collections from contextlib import contextmanager import StringIO -import spack.modules -from spack.test.mock_packages_test import MockPackagesTest FILE_REGISTRY = collections.defaultdict(StringIO.StringIO) - # Monkey-patch open to write module files to a StringIO instance @contextmanager def mock_open(filename, mode): if not mode == 'w': - message = 'test.modules : unexpected opening mode [mock_open]' - raise RuntimeError(message) + raise RuntimeError('test.modules : unexpected opening mode for monkey-patched open') FILE_REGISTRY[filename] = StringIO.StringIO() @@ -24,6 +20,7 @@ def mock_open(filename, mode): FILE_REGISTRY[filename] = handle.getvalue() handle.close() +import spack.modules configuration_autoload_direct = { 'enable': ['tcl'], @@ -50,8 +47,7 @@ configuration_alter_environment = { 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']} }, '=x86-linux': { - 'environment': {'set': {'FOO': 'foo'}, - 'unset': ['BAR']} + 'environment': {'set': {'FOO': 'foo'}, 'unset': ['BAR']} } } } @@ -76,14 +72,15 @@ configuration_conflicts = { } } +from spack.test.mock_packages_test import MockPackagesTest + class TclTests(MockPackagesTest): def setUp(self): super(TclTests, self).setUp() self.configuration_obj = spack.modules.CONFIGURATION spack.modules.open = mock_open - # Make sure that a non-mocked configuration will trigger an error - spack.modules.CONFIGURATION = None + spack.modules.CONFIGURATION = None # Make sure that a non-mocked configuration will trigger an error def tearDown(self): del spack.modules.open @@ -101,7 +98,7 @@ class TclTests(MockPackagesTest): spack.modules.CONFIGURATION = configuration_autoload_direct spec = spack.spec.Spec('mpich@3.0.4=x86-linux') content = self.get_modulefile_content(spec) - self.assertTrue('module-whatis "mpich @3.0.4"' in content) + self.assertTrue('module-whatis "mpich @3.0.4"' in content ) def test_autoload(self): spack.modules.CONFIGURATION = configuration_autoload_direct @@ -120,22 +117,14 @@ class TclTests(MockPackagesTest): spack.modules.CONFIGURATION = configuration_alter_environment spec = spack.spec.Spec('mpileaks=x86-linux') content = self.get_modulefile_content(spec) - self.assertEqual( - len([x - for x in content - if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) - self.assertEqual( - len([x for x in content if 'setenv FOO "foo"' in x]), 1) + self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) + self.assertEqual(len([x for x in content if 'setenv FOO "foo"' in x]), 1) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 1) spec = spack.spec.Spec('libdwarf=x64-linux') content = self.get_modulefile_content(spec) - self.assertEqual( - len([x - for x in content - if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) - self.assertEqual( - len([x for x in content if 'setenv FOO "foo"' in x]), 0) + self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) + self.assertEqual(len([x for x in content if 'setenv FOO "foo"' in x]), 0) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 0) def test_blacklist(self): @@ -149,9 +138,6 @@ class TclTests(MockPackagesTest): spack.modules.CONFIGURATION = configuration_conflicts spec = spack.spec.Spec('mpileaks=x86-linux') content = self.get_modulefile_content(spec) - self.assertEqual( - len([x for x in content if x.startswith('conflict')]), 2) - self.assertEqual( - len([x for x in content if x == 'conflict mpileaks']), 1) - self.assertEqual( - len([x for x in content if x == 'conflict intel/14.0.1']), 1) + self.assertEqual(len([x for x in content if x.startswith('conflict')]), 2) + self.assertEqual(len([x for x in content if x == 'conflict mpileaks']), 1) + self.assertEqual(len([x for x in content if x == 'conflict intel/14.0.1']), 1) -- cgit v1.2.3-70-g09d2 From eba264fcd0701b2172dc39d2616cb2f90b4d8578 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 11 May 2016 15:45:57 +0200 Subject: fix formatting --- lib/spack/spack/package_test.py | 16 +++++----- .../repos/builtin/packages/openblas/package.py | 34 ++++++++++++---------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/lib/spack/spack/package_test.py b/lib/spack/spack/package_test.py index 4b021684ee..9c15e3f5d0 100644 --- a/lib/spack/spack/package_test.py +++ b/lib/spack/spack/package_test.py @@ -25,7 +25,8 @@ from spack import * import os -def compile_c_and_execute(source_file,include_flags, link_flags): + +def compile_c_and_execute(source_file, include_flags, link_flags): """Compile C @p source_file with @p include_flags and @p link_flags, run and return the output. """ @@ -46,14 +47,15 @@ def compare_output(current_output, blessed_output): if not (current_output == blessed_output): print "Produced output does not match expected output." print "Expected output:" - print '-'*80 + print '-' * 80 print blessed_output - print '-'*80 + print '-' * 80 print "Produced output:" - print '-'*80 + print '-' * 80 print current_output - print '-'*80 - raise RuntimeError("Ouput check failed. See spack_output.log for details") + print '-' * 80 + raise RuntimeError("Ouput check failed.", + "See spack_output.log for details") def compare_output_file(current_output, blessed_output_file): @@ -61,4 +63,4 @@ def compare_output_file(current_output, blessed_output_file): with open(blessed_output_file, 'r') as f: blessed_output = f.read() - compare_output(current_output,blessed_output) + compare_output(current_output, blessed_output) diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 47b30181a8..cd8e3755ce 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -1,8 +1,7 @@ from spack import * from spack.package_test import * -import sys import os -import shutil + class Openblas(Package): """OpenBLAS: An optimized BLAS library""" @@ -14,9 +13,9 @@ class Openblas(Package): version('0.2.16', 'fef46ab92463bdbb1479dcec594ef6dc') version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') - variant('shared', default=True, description="Build shared libraries as well as static libs.") + variant('shared', default=True, description="Build shared libraries as well as static libs.") # NOQA: ignore=E501 variant('openmp', default=False, description="Enable OpenMP support.") - variant('fpic', default=True, description="Build position independent code") + variant('fpic', default=True, description="Build position independent code") # NOQA: ignore=E501 # virtual dependency provides('blas') @@ -48,11 +47,11 @@ class Openblas(Package): # Add support for OpenMP if '+openmp' in spec: - # Note: Apple's most recent Clang 7.3.0 still does not support OpenMP. - # What is worse, Openblas (as of 0.2.18) hardcoded that OpenMP cannot + # Openblas (as of 0.2.18) hardcoded that OpenMP cannot # be used with any (!) compiler named clang, bummer. if spec.satisfies('%clang'): - raise InstallError('OpenBLAS does not support OpenMP with clang!') + raise InstallError('OpenBLAS does not support ', + 'OpenMP with clang!') make_defs += ['USE_OPENMP=1'] @@ -69,29 +68,34 @@ class Openblas(Package): symlink('libopenblas.a', 'blas.a') symlink('libopenblas.a', 'libblas.a') if '+shared' in spec: - symlink('libopenblas.%s' % dso_suffix, 'libblas.%s' % dso_suffix) + symlink('libopenblas.%s' % dso_suffix, + 'libblas.%s' % dso_suffix) # Lapack virtual package should provide liblapack.a with working_dir(prefix.lib): symlink('libopenblas.a', 'liblapack.a') if '+shared' in spec: - symlink('libopenblas.%s' % dso_suffix, 'liblapack.%s' % dso_suffix) + symlink('libopenblas.%s' % dso_suffix, + 'liblapack.%s' % dso_suffix) # Openblas may pass its own test but still fail to compile Lapack - # symbols. To make sure we get working Blas and Lapack, do a small test. + # symbols. To make sure we get working Blas and Lapack, do a small + # test. self.check_install(spec) - def setup_dependent_package(self, module, dspec): # This is WIP for a prototype interface for virtual packages. # We can update this as more builds start depending on BLAS/LAPACK. - libdir = find_library_path('libopenblas.a', self.prefix.lib64, self.prefix.lib) + libdir = find_library_path('libopenblas.a', + self.prefix.lib64, + self.prefix.lib) self.spec.blas_static_lib = join_path(libdir, 'libopenblas.a') self.spec.lapack_static_lib = self.spec.blas_static_lib if '+shared' in self.spec: - self.spec.blas_shared_lib = join_path(libdir, 'libopenblas.%s' % dso_suffix) + self.spec.blas_shared_lib = join_path(libdir, 'libopenblas.%s' % + dso_suffix) self.spec.lapack_shared_lib = self.spec.blas_shared_lib def check_install(self, spec): @@ -108,5 +112,5 @@ class Openblas(Package): if '+openmp' in spec: link_flags.extend([self.compiler.openmp_flag]) - output = compile_c_and_execute(source_file,include_flags,link_flags) - compare_output_file(output,blessed_file) + output = compile_c_and_execute(source_file, include_flags, link_flags) + compare_output_file(output, blessed_file) -- cgit v1.2.3-70-g09d2 From bb4b6c8ee24aa81af6b6ee3a62692a624d52d65a Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 11 May 2016 17:02:36 +0200 Subject: flake 8 : fixed checks --- lib/spack/spack/cmd/module.py | 16 +- lib/spack/spack/environment.py | 58 ++++--- lib/spack/spack/modules.py | 196 ++++++++++++---------- lib/spack/spack/package.py | 353 +++++++++++++++++++++------------------ lib/spack/spack/test/__init__.py | 70 ++------ lib/spack/spack/test/modules.py | 42 +++-- 6 files changed, 391 insertions(+), 344 deletions(-) diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index f996f4eb84..cfe59c8d98 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -32,18 +32,21 @@ from llnl.util.filesystem import mkdirp from spack.modules import module_types from spack.util.string import * -description ="Manipulate modules and dotkits." +description = "Manipulate modules and dotkits." def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='module_command') - refresh_parser = sp.add_parser('refresh', help='Regenerate all module files.') + sp.add_parser('refresh', help='Regenerate all module files.') find_parser = sp.add_parser('find', help='Find module files for packages.') - find_parser.add_argument( - 'module_type', help="Type of module to find file for. [" + '|'.join(module_types) + "]") - find_parser.add_argument('spec', nargs='+', help='spec to find a module file for.') + find_parser.add_argument('module_type', + help="Type of module to find file for. [" + + '|'.join(module_types) + "]") + find_parser.add_argument('spec', + nargs='+', + help='spec to find a module file for.') def module_find(mtype, spec_array): @@ -53,7 +56,8 @@ def module_find(mtype, spec_array): should type to use that package's module. """ if mtype not in module_types: - tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types))) + tty.die("Invalid module type: '%s'. Options are %s" % + (mtype, comma_or(module_types))) specs = spack.cmd.parse_specs(spec_array) if len(specs) > 1: diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 92ab4e6bea..3fbe2531c1 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -1,7 +1,7 @@ -import os -import os.path import collections import inspect +import os +import os.path class NameModifier(object): @@ -26,7 +26,8 @@ class SetEnv(NameValueModifier): class UnsetEnv(NameModifier): def execute(self): - os.environ.pop(self.name, None) # Avoid throwing if the variable was not set + # Avoid throwing if the variable was not set + os.environ.pop(self.name, None) class SetPath(NameValueModifier): @@ -55,7 +56,9 @@ class RemovePath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split(':') if environment_value else [] - directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.value)] + directories = [os.path.normpath(x) + for x in directories + if x != os.path.normpath(self.value)] os.environ[self.name] = ':'.join(directories) @@ -63,7 +66,8 @@ class EnvironmentModifications(object): """ Keeps track of requests to modify the current environment. - Each call to a method to modify the environment stores the extra information on the caller in the request: + Each call to a method to modify the environment stores the extra + information on the caller in the request: - 'filename' : filename of the module where the caller is defined - 'lineno': line number where the request occurred - 'context' : line of code that issued the request that failed @@ -71,10 +75,10 @@ class EnvironmentModifications(object): def __init__(self, other=None): """ - Initializes a new instance, copying commands from other if it is not None + Initializes a new instance, copying commands from other if not None Args: - other: another instance of EnvironmentModifications from which (optional) + other: another instance of EnvironmentModifications (optional) """ self.env_modifications = [] if other is not None: @@ -93,7 +97,8 @@ class EnvironmentModifications(object): @staticmethod def _check_other(other): if not isinstance(other, EnvironmentModifications): - raise TypeError('other must be an instance of EnvironmentModifications') + raise TypeError( + 'other must be an instance of EnvironmentModifications') def _get_outside_caller_attributes(self): stack = inspect.stack() @@ -101,12 +106,10 @@ class EnvironmentModifications(object): _, filename, lineno, _, context, index = stack[2] context = context[index].strip() except Exception: - filename, lineno, context = 'unknown file', 'unknown line', 'unknown context' - args = { - 'filename': filename, - 'lineno': lineno, - 'context': context - } + filename = 'unknown file' + lineno = 'unknown line' + context = 'unknown context' + args = {'filename': filename, 'lineno': lineno, 'context': context} return args def set(self, name, value, **kwargs): @@ -170,7 +173,8 @@ class EnvironmentModifications(object): def remove_path(self, name, path, **kwargs): """ - Stores in the current object a request to remove a path from a path list + Stores in the current object a request to remove a path from a path + list Args: name: name of the path list in the environment @@ -185,7 +189,8 @@ class EnvironmentModifications(object): Returns a dict of the modifications grouped by variable name Returns: - dict mapping the environment variable name to the modifications to be done on it + dict mapping the environment variable name to the modifications to + be done on it """ modifications = collections.defaultdict(list) for item in self: @@ -203,7 +208,7 @@ class EnvironmentModifications(object): Applies the modifications and clears the list """ modifications = self.group_by_name() - # Apply the modifications to the environment variables one variable at a time + # Apply modifications one variable at a time for name, actions in sorted(modifications.items()): for x in actions: x.execute() @@ -224,13 +229,19 @@ def concatenate_paths(paths): def set_or_unset_not_first(variable, changes, errstream): """ - Check if we are going to set or unset something after other modifications have already been requested + Check if we are going to set or unset something after other modifications + have already been requested """ - indexes = [ii for ii, item in enumerate(changes) if ii != 0 and type(item) in [SetEnv, UnsetEnv]] + indexes = [ii + for ii, item in enumerate(changes) + if ii != 0 and type(item) in [SetEnv, UnsetEnv]] if indexes: good = '\t \t{context} at {filename}:{lineno}' nogood = '\t--->\t{context} at {filename}:{lineno}' - errstream('Suspicious requests to set or unset the variable \'{var}\' found'.format(var=variable)) + message = 'Suspicious requests to set or unset the variable \'{var}\' found' # NOQA: ignore=E501 + errstream( + message.format( + var=variable)) for ii, item in enumerate(changes): print_format = nogood if ii in indexes else good errstream(print_format.format(**item.args)) @@ -238,8 +249,8 @@ def set_or_unset_not_first(variable, changes, errstream): def validate(env, errstream): """ - Validates the environment modifications to check for the presence of suspicious patterns. Prompts a warning for - everything that was found + Validates the environment modifications to check for the presence of + suspicious patterns. Prompts a warning for everything that was found Current checks: - set or unset variables after other changes on the same variable @@ -254,7 +265,8 @@ def validate(env, errstream): def filter_environment_blacklist(env, variables): """ - Generator that filters out any change to environment variables present in the input list + Generator that filters out any change to environment variables present in + the input list Args: env: list of environment modifications diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index ffed469b20..53f054094a 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -23,36 +23,35 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## """ -This module contains code for creating environment modules, which can include dotkits, tcl modules, lmod, and others. +This module contains code for creating environment modules, which can include +dotkits, tcl modules, lmod, and others. -The various types of modules are installed by post-install hooks and removed after an uninstall by post-uninstall hooks. -This class consolidates the logic for creating an abstract description of the information that module systems need. -Currently that includes a number of directories to be appended to paths in the user's environment: +The various types of modules are installed by post-install hooks and removed +after an uninstall by post-uninstall hooks. This class consolidates the logic +for creating an abstract description of the information that module systems +need. - * /bin directories to be appended to PATH - * /lib* directories for LD_LIBRARY_PATH - * /include directories for CPATH - * /man* and /share/man* directories for MANPATH - * the package prefix for CMAKE_PREFIX_PATH +This module also includes logic for coming up with unique names for the module +files so that they can be found by the various shell-support files in +$SPACK/share/spack/setup-env.*. -This module also includes logic for coming up with unique names for the module files so that they can be found by the -various shell-support files in $SPACK/share/spack/setup-env.*. - -Each hook in hooks/ implements the logic for writing its specific type of module file. +Each hook in hooks/ implements the logic for writing its specific type of +module file. """ import copy import datetime import os import os.path import re -import textwrap import string +import textwrap import llnl.util.tty as tty import spack import spack.config from llnl.util.filesystem import join_path, mkdirp -from spack.build_environment import parent_class_modules, set_module_variables_for_package +from spack.build_environment import parent_class_modules +from spack.build_environment import set_module_variables_for_package from spack.environment import * __all__ = ['EnvModule', 'Dotkit', 'TclModule'] @@ -67,30 +66,26 @@ def print_help(): """ For use by commands to tell user how to activate shell support. """ - tty.msg("This command requires spack's shell integration.", - "", + tty.msg("This command requires spack's shell integration.", "", "To initialize spack's shell commands, you must run one of", "the commands below. Choose the right command for your shell.", - "", - "For bash and zsh:", - " . %s/setup-env.sh" % spack.share_path, - "", - "For csh and tcsh:", - " setenv SPACK_ROOT %s" % spack.prefix, - " source %s/setup-env.csh" % spack.share_path, - "") + "", "For bash and zsh:", + " . %s/setup-env.sh" % spack.share_path, "", + "For csh and tcsh:", " setenv SPACK_ROOT %s" % spack.prefix, + " source %s/setup-env.csh" % spack.share_path, "") def inspect_path(prefix): """ - Inspects the prefix of an installation to search for common layouts. Issues a request to modify the environment - accordingly when an item is found. + Inspects the prefix of an installation to search for common layouts. Issues + a request to modify the environment accordingly when an item is found. Args: prefix: prefix of the installation Returns: - instance of EnvironmentModifications containing the requested modifications + instance of EnvironmentModifications containing the requested + modifications """ env = EnvironmentModifications() # Inspect the prefix to check for the existence of common directories @@ -105,18 +100,21 @@ def inspect_path(prefix): def dependencies(spec, request='all'): """ - Returns the list of dependent specs for a given spec, according to the given request + Returns the list of dependent specs for a given spec, according to the + given request Args: spec: target spec request: either 'none', 'direct' or 'all' Returns: - empty list if 'none', direct dependency list if 'direct', all dependencies if 'all' + empty list if 'none', direct dependency list if 'direct', all + dependencies if 'all' """ if request not in ('none', 'direct', 'all'): - raise tty.error("Wrong value for argument 'request' : should be one of ('none', 'direct', 'all') " - " [current value is '%s']" % request) + message = "Wrong value for argument 'request' : " + message += "should be one of ('none', 'direct', 'all')" + raise tty.error(message + " [current value is '%s']" % request) if request == 'none': return [] @@ -124,12 +122,19 @@ def dependencies(spec, request='all'): if request == 'direct': return [xx for _, xx in spec.dependencies.items()] - # FIXME : during module file creation nodes seem to be visited multiple times even if cover='nodes' - # FIXME : is given. This work around permits to get a unique list of spec anyhow. - # FIXME : Possibly we miss a merge step among nodes that refer to the same package. + # FIXME : during module file creation nodes seem to be visited multiple + # FIXME : times even if cover='nodes' is given. This work around permits + # FIXME : to get a unique list of spec anyhow. Do we miss a merge + # FIXME : step among nodes that refer to the same package? seen = set() seen_add = seen.add - l = [xx for xx in sorted(spec.traverse(order='post', depth=True, cover='nodes', root=False), reverse=True)] + l = [xx + for xx in sorted( + spec.traverse(order='post', + depth=True, + cover='nodes', + root=False), + reverse=True)] return [xx for ii, xx in l if not (xx in seen or seen_add(xx))] @@ -146,7 +151,8 @@ def update_dictionary_extending_lists(target, update): def parse_config_options(module_generator): """ - Parse the configuration file and returns a bunch of items that will be needed during module file generation + Parse the configuration file and returns a bunch of items that will be + needed during module file generation Args: module_generator: module generator for a given spec @@ -154,11 +160,14 @@ def parse_config_options(module_generator): Returns: autoloads: list of specs to be autoloaded prerequisites: list of specs to be marked as prerequisite - filters: list of environment variables whose modification is blacklisted in module files - env: list of custom environment modifications to be applied in the module file + filters: list of environment variables whose modification is + blacklisted in module files + env: list of custom environment modifications to be applied in the + module file """ # Get the configuration for this kind of generator - module_configuration = copy.deepcopy(CONFIGURATION.get(module_generator.name, {})) + module_configuration = copy.deepcopy(CONFIGURATION.get( + module_generator.name, {})) ##### # Merge all the rules @@ -179,9 +188,12 @@ def parse_config_options(module_generator): ##### # Automatic loading loads - module_file_actions['autoload'] = dependencies(module_generator.spec, module_file_actions.get('autoload', 'none')) + module_file_actions['autoload'] = dependencies( + module_generator.spec, module_file_actions.get('autoload', 'none')) # Prerequisites - module_file_actions['prerequisites'] = dependencies(module_generator.spec, module_file_actions.get('prerequisites', 'none')) + module_file_actions['prerequisites'] = dependencies( + module_generator.spec, module_file_actions.get('prerequisites', + 'none')) # Environment modifications environment_actions = module_file_actions.pop('environment', {}) env = EnvironmentModifications() @@ -189,7 +201,7 @@ def parse_config_options(module_generator): def process_arglist(arglist): if method == 'unset': for x in arglist: - yield (x,) + yield (x, ) else: for x in arglist.iteritems(): yield x @@ -198,19 +210,13 @@ def parse_config_options(module_generator): for args in process_arglist(arglist): getattr(env, method)(*args) - # for item in arglist: - # if method == 'unset': - # args = [item] - # else: - # args = item.split(',') - # getattr(env, method)(*args) - return module_file_actions, env def filter_blacklisted(specs, module_name): """ - Given a sequence of specs, filters the ones that are blacklisted in the module configuration file. + Given a sequence of specs, filters the ones that are blacklisted in the + module configuration file. Args: specs: sequence of spec instances @@ -233,7 +239,8 @@ class EnvModule(object): class __metaclass__(type): def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) - if cls.name != 'env_module' and cls.name in CONFIGURATION['enable']: + if cls.name != 'env_module' and cls.name in CONFIGURATION[ + 'enable']: module_types[cls.name] = cls def __init__(self, spec=None): @@ -249,7 +256,8 @@ class EnvModule(object): # long description is the docstring with reduced whitespace. self.long_description = None if self.spec.package.__doc__: - self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) + self.long_description = re.sub(r'\s+', ' ', + self.spec.package.__doc__) @property def naming_scheme(self): @@ -271,12 +279,14 @@ class EnvModule(object): @property def use_name(self): """ - Subclasses should implement this to return the name the module command uses to refer to the package. + Subclasses should implement this to return the name the module command + uses to refer to the package. """ naming_tokens = self.tokens naming_scheme = self.naming_scheme name = naming_scheme.format(**naming_tokens) - name += '-' + self.spec.dag_hash() # Always append the hash to make the module file unique + name += '-' + self.spec.dag_hash( + ) # Always append the hash to make the module file unique # Not everybody is working on linux... parts = name.split('/') name = join_path(*parts) @@ -296,8 +306,12 @@ class EnvModule(object): @property def blacklisted(self): configuration = CONFIGURATION.get(self.name, {}) - whitelist_matches = [x for x in configuration.get('whitelist', []) if self.spec.satisfies(x)] - blacklist_matches = [x for x in configuration.get('blacklist', []) if self.spec.satisfies(x)] + whitelist_matches = [x + for x in configuration.get('whitelist', []) + if self.spec.satisfies(x)] + blacklist_matches = [x + for x in configuration.get('blacklist', []) + if self.spec.satisfies(x)] if whitelist_matches: message = '\tWHITELIST : %s [matches : ' % self.spec.cshort_spec for rule in whitelist_matches: @@ -327,7 +341,8 @@ class EnvModule(object): """ if self.blacklisted: return - tty.debug("\tWRITE : %s [%s]" % (self.spec.cshort_spec, self.file_name)) + tty.debug("\tWRITE : %s [%s]" % + (self.spec.cshort_spec, self.file_name)) module_dir = os.path.dirname(self.file_name) if not os.path.exists(module_dir): @@ -337,11 +352,12 @@ class EnvModule(object): # installation prefix env = inspect_path(self.spec.prefix) - # Let the extendee/dependency modify their extensions/dependencies before asking for - # package-specific modifications + # Let the extendee/dependency modify their extensions/dependencies + # before asking for package-specific modifications spack_env = EnvironmentModifications() - # TODO : the code down below is quite similar to build_environment.setup_package and needs to be - # TODO : factored out to a single place + # TODO : the code down below is quite similar to + # TODO : build_environment.setup_package and needs to be factored out + # TODO : to a single place for item in dependencies(self.spec, 'all'): package = self.spec[item.name].package modules = parent_class_modules(package.__class__) @@ -358,14 +374,18 @@ class EnvModule(object): # Parse configuration file module_configuration, conf_env = parse_config_options(self) env.extend(conf_env) - filters = module_configuration.get('filter', {}).get('environment_blacklist',{}) + filters = module_configuration.get('filter', {}).get( + 'environment_blacklist', {}) # Build up the module file content module_file_content = self.header - for x in filter_blacklisted(module_configuration.pop('autoload', []), self.name): + for x in filter_blacklisted( + module_configuration.pop('autoload', []), self.name): module_file_content += self.autoload(x) - for x in filter_blacklisted(module_configuration.pop('prerequisites', []), self.name): + for x in filter_blacklisted( + module_configuration.pop('prerequisites', []), self.name): module_file_content += self.prerequisite(x) - for line in self.process_environment_command(filter_environment_blacklist(env, filters)): + for line in self.process_environment_command( + filter_environment_blacklist(env, filters)): module_file_content += line for line in self.module_specific_content(module_configuration): module_file_content += line @@ -392,10 +412,13 @@ class EnvModule(object): def process_environment_command(self, env): for command in env: try: - yield self.environment_modifications_formats[type(command)].format(**command.args) + yield self.environment_modifications_formats[type( + command)].format(**command.args) except KeyError: - tty.warn('Cannot handle command of type {command} : skipping request'.format(command=type(command))) - tty.warn('{context} at {filename}:{lineno}'.format(**command.args)) + message = 'Cannot handle command of type {command} : skipping request' # NOQA: ignore=E501 + details = '{context} at {filename}:{lineno}' + tty.warn(message.format(command=type(command))) + tty.warn(details.format(**command.args)) @property def file_name(self): @@ -408,9 +431,12 @@ class EnvModule(object): if os.path.exists(mod_file): try: os.remove(mod_file) # Remove the module file - os.removedirs(os.path.dirname(mod_file)) # Remove all the empty directories from the leaf up + os.removedirs( + os.path.dirname(mod_file) + ) # Remove all the empty directories from the leaf up except OSError: - pass # removedirs throws OSError on first non-empty directory found + # removedirs throws OSError on first non-empty directory found + pass class Dotkit(EnvModule): @@ -424,13 +450,12 @@ class Dotkit(EnvModule): autoload_format = 'dk_op {module_file}\n' - prerequisite_format = None # TODO : does something like prerequisite exist for dotkit? - - default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' + default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' # NOQA: ignore=E501 @property def file_name(self): - return join_path(Dotkit.path, self.spec.architecture, '%s.dk' % self.use_name) + return join_path(Dotkit.path, self.spec.architecture, + '%s.dk' % self.use_name) @property def header(self): @@ -474,7 +499,7 @@ class TclModule(EnvModule): prerequisite_format = 'prereq {module_file}\n' - default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' + default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' # NOQA: ignore=E501 @property def file_name(self): @@ -482,9 +507,10 @@ class TclModule(EnvModule): @property def header(self): + timestamp = datetime.datetime.now() # TCL Modulefile header header = '#%Module1.0\n' - header += '## Module file created by spack (https://github.com/LLNL/spack) on %s\n' % datetime.datetime.now() + header += '## Module file created by spack (https://github.com/LLNL/spack) on %s\n' % timestamp # NOQA: ignore=E501 header += '##\n' header += '## %s\n' % self.spec.short_spec header += '##\n' @@ -509,16 +535,18 @@ class TclModule(EnvModule): f = string.Formatter() for item in conflict_format: line = 'conflict ' + item + '\n' - if len([x for x in f.parse(line)]) > 1: # We do have placeholder to substitute - for naming_dir, conflict_dir in zip(self.naming_scheme.split('/'), item.split('/')): + if len([x for x in f.parse(line) + ]) > 1: # We do have placeholder to substitute + for naming_dir, conflict_dir in zip( + self.naming_scheme.split('/'), item.split('/')): if naming_dir != conflict_dir: - message = 'conflict scheme does not match naming scheme [{spec}]\n\n' + message = 'conflict scheme does not match naming scheme [{spec}]\n\n' # NOQA: ignore=E501 message += 'naming scheme : "{nformat}"\n' message += 'conflict scheme : "{cformat}"\n\n' - message += '** You may want to check your `modules.yaml` configuration file **\n' - tty.error( - message.format(spec=self.spec, nformat=self.naming_scheme, cformat=item) - ) + message += '** You may want to check your `modules.yaml` configuration file **\n' # NOQA: ignore=E501 + tty.error(message.format(spec=self.spec, + nformat=self.naming_scheme, + cformat=item)) raise SystemExit('Module generation aborted.') line = line.format(**naming_tokens) yield line diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 3626a574c8..8e6cf32954 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -37,7 +37,6 @@ import os import re import textwrap import time -import glob import llnl.util.tty as tty import spack @@ -62,7 +61,6 @@ from spack.util.environment import dump_environment from spack.util.executable import ProcessError from spack.version import * from urlparse import urlparse - """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] @@ -305,26 +303,21 @@ class Package(object): # """By default we build in parallel. Subclasses can override this.""" parallel = True - """# jobs to use for parallel make. If set, overrides default of ncpus.""" make_jobs = None - - """Most packages are NOT extendable. Set to True if you want extensions.""" + """Most packages are NOT extendable. Set to True if you want extensions.""" extendable = False - """List of prefix-relative file paths (or a single path). If these do not exist after install, or if they exist but are not files, sanity checks fail. """ sanity_check_is_file = [] - """List of prefix-relative directory paths (or a single path). If these do not exist after install, or if they exist but are not directories, sanity checks will fail. """ sanity_check_is_dir = [] - def __init__(self, spec): # this determines how the package should be built. self.spec = spec @@ -336,23 +329,24 @@ class Package(object): self.name = self.name[self.name.rindex('.') + 1:] # Allow custom staging paths for packages - self.path=None + self.path = None # Sanity check attributes required by Spack directives. spack.directives.ensure_dicts(type(self)) # Check versions in the versions dict. for v in self.versions: - assert(isinstance(v, Version)) + assert (isinstance(v, Version)) # Check version descriptors for v in sorted(self.versions): - assert(isinstance(self.versions[v], dict)) + assert (isinstance(self.versions[v], dict)) # Version-ize the keys in versions dict try: - self.versions = dict((Version(v), h) for v,h in self.versions.items()) - except ValueError, e: + self.versions = dict((Version(v), h) + for v, h in self.versions.items()) + except ValueError as e: raise ValueError("In package %s: %s" % (self.name, e.message)) # stage used to build this package. @@ -366,9 +360,9 @@ class Package(object): # This makes self.url behave sanely. if self.spec.versions.concrete: # TODO: this is a really roundabout way of determining the type - # TODO: of fetch to do. figure out a more sane fetch strategy/package - # TODO: init order (right now it's conflated with stage, package, and - # TODO: the tests make assumptions) + # TODO: of fetch to do. figure out a more sane fetch + # TODO: strategy/package init order (right now it's conflated with + # TODO: stage, package, and the tests make assumptions) f = fs.for_package_version(self, self.version) if isinstance(f, fs.URLFetchStrategy): self.url = self.url_for_version(self.spec.version) @@ -387,14 +381,12 @@ class Package(object): if self.is_extension: spack.repo.get(self.extendee_spec)._check_extendable() - @property def version(self): if not self.spec.versions.concrete: raise ValueError("Can only get of package with concrete version.") return self.spec.versions[0] - @memoized def version_urls(self): """Return a list of URLs for different versions of this @@ -407,7 +399,6 @@ class Package(object): version_urls[v] = args['url'] return version_urls - def nearest_url(self, version): """Finds the URL for the next lowest version with a URL. If there is no lower version with a URL, uses the @@ -424,10 +415,11 @@ class Package(object): url = version_urls[v] return url - # TODO: move this out of here and into some URL extrapolation module? def url_for_version(self, version): - """Returns a URL that you can download a new version of this package from.""" + """ + Returns a URL that you can download a new version of this package from. + """ if not isinstance(version, Version): version = Version(version) @@ -441,14 +433,17 @@ class Package(object): return version_urls[version] # If we have no idea, try to substitute the version. - return spack.url.substitute_version(self.nearest_url(version), - self.url_version(version)) + return spack.url.substitute_version( + self.nearest_url(version), self.url_version(version)) def _make_resource_stage(self, root_stage, fetcher, resource): resource_stage_folder = self._resource_stage(resource) resource_mirror = join_path(self.name, os.path.basename(fetcher.url)) - stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource, - name=resource_stage_folder, mirror_path=resource_mirror, + stage = ResourceStage(resource.fetcher, + root=root_stage, + resource=resource, + name=resource_stage_folder, + mirror_path=resource_mirror, path=self.path) return stage @@ -474,7 +469,8 @@ class Package(object): else: # Construct resource stage resource = resources[ii - 1] # ii == 0 is root! - stage = self._make_resource_stage(composite_stage[0], fetcher, resource) + stage = self._make_resource_stage(composite_stage[0], fetcher, + resource) # Append the item to the composite composite_stage.append(stage) @@ -492,13 +488,11 @@ class Package(object): self._stage = self._make_stage() return self._stage - @stage.setter def stage(self, stage): """Allow a stage object to be set to override the default.""" self._stage = stage - def _make_fetcher(self): # Construct a composite fetcher that always contains at least # one element (the root package). In case there are resources @@ -515,7 +509,8 @@ class Package(object): @property def fetcher(self): if not self.spec.versions.concrete: - raise ValueError("Can only get a fetcher for a package with concrete versions.") + raise ValueError( + "Can only get a fetcher for a package with concrete versions.") if not self._fetcher: self._fetcher = self._make_fetcher() return self._fetcher @@ -524,10 +519,11 @@ class Package(object): def fetcher(self, f): self._fetcher = f - @property def extendee_spec(self): - """Spec of the extendee of this package, or None if it is not an extension.""" + """ + Spec of the extendee of this package, or None if it is not an extension + """ if not self.extendees: return None @@ -549,10 +545,11 @@ class Package(object): spec, kwargs = self.extendees[name] return spec - @property def extendee_args(self): - """Spec of the extendee of this package, or None if it is not an extension.""" + """ + Spec of the extendee of this package, or None if it is not an extension + """ if not self.extendees: return None @@ -560,7 +557,6 @@ class Package(object): name = next(iter(self.extendees)) return self.extendees[name][1] - @property def is_extension(self): # if it is concrete, it's only an extension if it actually @@ -571,22 +567,20 @@ class Package(object): # If not, then it's an extension if it *could* be an extension return bool(self.extendees) - def extends(self, spec): - if not spec.name in self.extendees: + if spec.name not in self.extendees: return False s = self.extendee_spec return s and s.satisfies(spec) - @property def activated(self): if not self.is_extension: - raise ValueError("is_extension called on package that is not an extension.") + raise ValueError( + "is_extension called on package that is not an extension.") exts = spack.install_layout.extension_map(self.extendee_spec) return (self.name in exts) and (exts[self.name] == self.spec) - def preorder_traversal(self, visited=None, **kwargs): """This does a preorder traversal of the package's dependence DAG.""" virtual = kwargs.get("virtual", False) @@ -605,36 +599,35 @@ class Package(object): spec = self.dependencies[name] # currently, we do not descend into virtual dependencies, as this - # makes doing a sensible traversal much harder. We just assume that - # ANY of the virtual deps will work, which might not be true (due to - # conflicts or unsatisfiable specs). For now this is ok but we might - # want to reinvestigate if we start using a lot of complicated virtual - # dependencies + # makes doing a sensible traversal much harder. We just assume + # that ANY of the virtual deps will work, which might not be true + # (due to conflicts or unsatisfiable specs). For now this is ok + # but we might want to reinvestigate if we start using a lot of + # complicated virtual dependencies # TODO: reinvestigate this. if spec.virtual: if virtual: yield spec continue - for pkg in spack.repo.get(name).preorder_traversal(visited, **kwargs): + for pkg in spack.repo.get(name).preorder_traversal(visited, + **kwargs): yield pkg - def provides(self, vpkg_name): - """True if this package provides a virtual package with the specified name.""" + """ + True if this package provides a virtual package with the specified name + """ return any(s.name == vpkg_name for s in self.provided) - def virtual_dependencies(self, visited=None): for spec in sorted(set(self.preorder_traversal(virtual=True))): yield spec - @property def installed(self): return os.path.isdir(self.prefix) - @property def installed_dependents(self): """Return a list of the specs of all installed packages that depend @@ -651,60 +644,62 @@ class Package(object): dependents.append(spec) return dependents - @property def prefix(self): """Get the prefix into which this package should be installed.""" return self.spec.prefix - @property def compiler(self): - """Get the spack.compiler.Compiler object used to build this package.""" + """Get the spack.compiler.Compiler object used to build this package""" if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") return spack.compilers.compiler_for_spec(self.spec.compiler) - def url_version(self, version): - """Given a version, this returns a string that should be substituted into the - package's URL to download that version. - By default, this just returns the version string. Subclasses may need to - override this, e.g. for boost versions where you need to ensure that there - are _'s in the download URL. """ - return str(version) + Given a version, this returns a string that should be substituted + into the package's URL to download that version. + By default, this just returns the version string. Subclasses may need + to override this, e.g. for boost versions where you need to ensure that + there are _'s in the download URL. + """ + return str(version) def remove_prefix(self): - """Removes the prefix for a package along with any empty parent directories.""" + """ + Removes the prefix for a package along with any empty parent + directories + """ spack.install_layout.remove_install_directory(self.spec) - def do_fetch(self, mirror_only=False): - """Creates a stage directory and downloads the tarball for this package. - Working directory will be set to the stage directory. + """ + Creates a stage directory and downloads the tarball for this package. + Working directory will be set to the stage directory. """ if not self.spec.concrete: raise ValueError("Can only fetch concrete packages.") start_time = time.time() - if spack.do_checksum and not self.version in self.versions: - tty.warn("There is no checksum on file to fetch %s safely." - % self.spec.format('$_$@')) + if spack.do_checksum and self.version not in self.versions: + tty.warn("There is no checksum on file to fetch %s safely." % + self.spec.format('$_$@')) # Ask the user whether to skip the checksum if we're # interactive, but just fail if non-interactive. - checksum_msg = "Add a checksum or use --no-checksum to skip this check." + checksum_msg = "Add a checksum or use --no-checksum to skip this check." # NOQA: ignore=E501 ignore_checksum = False if sys.stdout.isatty(): - ignore_checksum = tty.get_yes_or_no(" Fetch anyway?", default=False) + ignore_checksum = tty.get_yes_or_no(" Fetch anyway?", + default=False) if ignore_checksum: tty.msg("Fetching with no checksum.", checksum_msg) if not ignore_checksum: - raise FetchError( - "Will not fetch %s" % self.spec.format('$_$@'), checksum_msg) + raise FetchError("Will not fetch %s" % + self.spec.format('$_$@'), checksum_msg) self.stage.fetch(mirror_only) @@ -723,7 +718,6 @@ class Package(object): self.stage.expand_archive() self.stage.chdir_to_source() - def do_patch(self): """Calls do_stage(), then applied patches to the expanded tarball if they haven't been applied already.""" @@ -743,10 +737,10 @@ class Package(object): # Construct paths to special files in the archive dir used to # keep track of whether patches were successfully applied. - archive_dir = self.stage.source_path - good_file = join_path(archive_dir, '.spack_patched') + archive_dir = self.stage.source_path + good_file = join_path(archive_dir, '.spack_patched') no_patches_file = join_path(archive_dir, '.spack_no_patches') - bad_file = join_path(archive_dir, '.spack_patch_failed') + bad_file = join_path(archive_dir, '.spack_patch_failed') # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. @@ -801,13 +795,11 @@ class Package(object): else: touch(no_patches_file) - @property def namespace(self): namespace, dot, module = self.__module__.rpartition('.') return namespace - def do_fake_install(self): """Make a fake install directory contaiing a 'fake' file in bin.""" mkdirp(self.prefix.bin) @@ -815,15 +807,15 @@ class Package(object): mkdirp(self.prefix.lib) mkdirp(self.prefix.man1) - def _get_needed_resources(self): resources = [] # Select the resources that are needed for this build for when_spec, resource_list in self.resources.items(): if when_spec in self.spec: resources.extend(resource_list) - # Sorts the resources by the length of the string representing their destination. Since any nested resource - # must contain another resource's name in its path, it seems that should work + # Sorts the resources by the length of the string representing their + # destination. Since any nested resource must contain another + # resource's name in its path, it seems that should work resources = sorted(resources, key=lambda res: len(res.destination)) return resources @@ -832,10 +824,14 @@ class Package(object): resource_stage_folder = '-'.join(pieces) return resource_stage_folder - def do_install(self, - keep_prefix=False, keep_stage=False, ignore_deps=False, - skip_patch=False, verbose=False, make_jobs=None, fake=False): + keep_prefix=False, + keep_stage=False, + ignore_deps=False, + skip_patch=False, + verbose=False, + make_jobs=None, + fake=False): """Called by commands to install a package and its dependencies. Package implementations should override install() to describe @@ -846,18 +842,20 @@ class Package(object): keep_stage -- By default, stage is destroyed only if there are no exceptions during build. Set to True to keep the stage even with exceptions. - ignore_deps -- Do not install dependencies before installing this package. + ignore_deps -- Don't install dependencies before installing this + package fake -- Don't really build -- install fake stub files instead. skip_patch -- Skip patch stage of build if True. verbose -- Display verbose build output (by default, suppresses it) - make_jobs -- Number of make jobs to use for install. Default is ncpus. + make_jobs -- Number of make jobs to use for install. Default is ncpus """ if not self.spec.concrete: raise ValueError("Can only install concrete packages.") # No installation needed if package is external if self.spec.external: - tty.msg("%s is externally installed in %s" % (self.name, self.spec.external)) + tty.msg("%s is externally installed in %s" % + (self.name, self.spec.external)) return # Ensure package is not already installed @@ -869,9 +867,13 @@ class Package(object): # First, install dependencies recursively. if not ignore_deps: - self.do_install_dependencies( - keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps, - fake=fake, skip_patch=skip_patch, verbose=verbose, make_jobs=make_jobs) + self.do_install_dependencies(keep_prefix=keep_prefix, + keep_stage=keep_stage, + ignore_deps=ignore_deps, + fake=fake, + skip_patch=skip_patch, + verbose=verbose, + make_jobs=make_jobs) # Set parallelism before starting build. self.make_jobs = make_jobs @@ -899,35 +901,41 @@ class Package(object): self.do_fake_install() else: # Do the real install in the source directory. - self.stage.chdir_to_source() + self.stage.chdir_to_source() - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') - try: - # Redirect I/O to a build log (and optionally to the terminal) + try: + # Redirect I/O to a build log (and optionally to + # the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), True): + with log_output(log_file, verbose, sys.stdout.isatty(), + True): dump_environment(env_path) self.install(self.spec, self.prefix) - except ProcessError as e: - # Annotate ProcessErrors with the location of the build log. - e.build_log = log_path - raise e + except ProcessError as e: + # Annotate ProcessErrors with the location of + # the build log + e.build_log = log_path + raise e - # Ensure that something was actually installed. - self.sanity_check_prefix() + # Ensure that something was actually installed. + self.sanity_check_prefix() - # Copy provenance into the install directory on success - log_install_path = spack.install_layout.build_log_path(self.spec) - env_install_path = spack.install_layout.build_env_path(self.spec) - packages_dir = spack.install_layout.build_packages_path(self.spec) + # Copy provenance into the install directory on success + log_install_path = spack.install_layout.build_log_path( + self.spec) + env_install_path = spack.install_layout.build_env_path( + self.spec) + packages_dir = spack.install_layout.build_packages_path( + self.spec) - install(log_path, log_install_path) - install(env_path, env_install_path) - dump_packages(self.spec, packages_dir) + install(log_path, log_install_path) + install(env_path, env_install_path) + dump_packages(self.spec, packages_dir) # Run post install hooks before build stage is removed. spack.hooks.post_install(self) @@ -937,8 +945,9 @@ class Package(object): build_time = self._total_time - self._fetch_time tty.msg("Successfully installed %s" % self.name, - "Fetch: %s. Build: %s. Total: %s." - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + "Fetch: %s. Build: %s. Total: %s." % + (_hms(self._fetch_time), _hms(build_time), + _hms(self._total_time))) print_pkg(self.prefix) try: @@ -953,16 +962,17 @@ class Package(object): tty.warn("Keeping install prefix in place despite error.", "Spack will think this package is installed. " + "Manually remove this directory to fix:", - self.prefix, wrap=True) + self.prefix, + wrap=True) raise # note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. spack.installed_db.add(self.spec, self.prefix) - def sanity_check_prefix(self): """This function checks whether install succeeded.""" + def check_paths(path_list, filetype, predicate): if isinstance(path_list, basestring): path_list = [path_list] @@ -970,8 +980,9 @@ class Package(object): for path in path_list: abs_path = os.path.join(self.prefix, path) if not predicate(abs_path): - raise InstallError("Install failed for %s. No such %s in prefix: %s" - % (self.name, filetype, path)) + raise InstallError( + "Install failed for %s. No such %s in prefix: %s" % + (self.name, filetype, path)) check_paths(self.sanity_check_is_file, 'file', os.path.isfile) check_paths(self.sanity_check_is_dir, 'directory', os.path.isdir) @@ -982,13 +993,11 @@ class Package(object): raise InstallError( "Install failed for %s. Nothing was installed!" % self.name) - def do_install_dependencies(self, **kwargs): # Pass along paths of dependencies here for dep in self.spec.dependencies.values(): dep.package.do_install(**kwargs) - @property def build_log_path(self): if self.installed: @@ -996,7 +1005,6 @@ class Package(object): else: return join_path(self.stage.source_path, 'spack-build.out') - @property def module(self): """Use this to add variables to the class's module's scope. @@ -1037,7 +1045,6 @@ class Package(object): """ pass - def setup_dependent_environment(self, spack_env, run_env, dependent_spec): """Set up the environment of packages that depend on this one. @@ -1077,7 +1084,6 @@ class Package(object): """ self.setup_environment(spack_env, run_env) - def setup_dependent_package(self, module, dependent_spec): """Set up Python module-scope variables for dependent packages. @@ -1123,8 +1129,11 @@ class Package(object): pass def install(self, spec, prefix): - """Package implementations override this with their own build configuration.""" - raise InstallError("Package %s provides no install method!" % self.name) + """ + Package implementations override this with their own configuration + """ + raise InstallError("Package %s provides no install method!" % + self.name) def do_uninstall(self, force=False): if not self.installed: @@ -1146,12 +1155,10 @@ class Package(object): # Once everything else is done, run post install hooks spack.hooks.post_uninstall(self) - def _check_extendable(self): if not self.extendable: raise ValueError("Package %s is not extendable!" % self.name) - def _sanity_check_extension(self): if not self.is_extension: raise ActivationError("This package is not an extension.") @@ -1160,12 +1167,13 @@ class Package(object): extendee_package._check_extendable() if not extendee_package.installed: - raise ActivationError("Can only (de)activate extensions for installed packages.") + raise ActivationError( + "Can only (de)activate extensions for installed packages.") if not self.installed: raise ActivationError("Extensions must first be installed.") - if not self.extendee_spec.name in self.extendees: - raise ActivationError("%s does not extend %s!" % (self.name, self.extendee.name)) - + if self.extendee_spec.name not in self.extendees: + raise ActivationError("%s does not extend %s!" % + (self.name, self.extendee.name)) def do_activate(self, force=False): """Called on an etension to invoke the extendee's activate method. @@ -1175,8 +1183,8 @@ class Package(object): """ self._sanity_check_extension() - spack.install_layout.check_extension_conflict( - self.extendee_spec, self.spec) + spack.install_layout.check_extension_conflict(self.extendee_spec, + self.spec) # Activate any package dependencies that are also extensions. if not force: @@ -1188,9 +1196,8 @@ class Package(object): self.extendee_spec.package.activate(self, **self.extendee_args) spack.install_layout.add_extension(self.extendee_spec, self.spec) - tty.msg("Activated extension %s for %s" - % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) - + tty.msg("Activated extension %s for %s" % + (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) def activate(self, extension, **kwargs): """Symlinks all files from the extension into extendee's install dir. @@ -1201,6 +1208,7 @@ class Package(object): always executed. """ + def ignore(filename): return (filename in spack.install_layout.hidden_file_paths or kwargs.get('ignore', lambda f: False)(filename)) @@ -1212,7 +1220,6 @@ class Package(object): tree.merge(self.prefix, ignore=ignore) - def do_deactivate(self, **kwargs): """Called on the extension to invoke extendee's deactivate() method.""" self._sanity_check_extension() @@ -1230,7 +1237,7 @@ class Package(object): for dep in aspec.traverse(): if self.spec == dep: raise ActivationError( - "Cannot deactivate %s beacuse %s is activated and depends on it." + "Cannot deactivate %s because %s is activated and depends on it." # NOQA: ignore=E501 % (self.spec.short_spec, aspec.short_spec)) self.extendee_spec.package.deactivate(self, **self.extendee_args) @@ -1238,11 +1245,11 @@ class Package(object): # redundant activation check -- makes SURE the spec is not # still activated even if something was wrong above. if self.activated: - spack.install_layout.remove_extension(self.extendee_spec, self.spec) - - tty.msg("Deactivated extension %s for %s" - % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) + spack.install_layout.remove_extension(self.extendee_spec, + self.spec) + tty.msg("Deactivated extension %s for %s" % + (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) def deactivate(self, extension, **kwargs): """Unlinks all files from extension out of this package's install dir. @@ -1253,6 +1260,7 @@ class Package(object): always executed. """ + def ignore(filename): return (filename in spack.install_layout.hidden_file_paths or kwargs.get('ignore', lambda f: False)(filename)) @@ -1260,17 +1268,14 @@ class Package(object): tree = LinkTree(extension.prefix) tree.unmerge(self.prefix, ignore=ignore) - def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() - def do_clean(self): """Removes the package's build stage and source tarball.""" self.stage.destroy() - def format_doc(self, **kwargs): """Wrap doc string at 72 characters and format nicely""" indent = kwargs.get('indent', 0) @@ -1285,7 +1290,6 @@ class Package(object): results.write((" " * indent) + line + "\n") return results.getvalue() - @property def all_urls(self): urls = [] @@ -1297,7 +1301,6 @@ class Package(object): urls.append(args['url']) return urls - def fetch_remote_versions(self): """Try to find remote versions of this package using the list_url and any other URLs described in the package file.""" @@ -1306,26 +1309,30 @@ class Package(object): try: return spack.util.web.find_versions_of_archive( - *self.all_urls, list_url=self.list_url, list_depth=self.list_depth) + *self.all_urls, + list_url=self.list_url, + list_depth=self.list_depth) except spack.error.NoNetworkConnectionError as e: - tty.die("Package.fetch_versions couldn't connect to:", - e.url, e.message) - + tty.die("Package.fetch_versions couldn't connect to:", e.url, + e.message) @property def rpath(self): """Get the rpath this package links with, as a list of paths.""" rpaths = [self.prefix.lib, self.prefix.lib64] - rpaths.extend(d.prefix.lib for d in self.spec.traverse(root=False) + rpaths.extend(d.prefix.lib + for d in self.spec.traverse(root=False) if os.path.isdir(d.prefix.lib)) - rpaths.extend(d.prefix.lib64 for d in self.spec.traverse(root=False) + rpaths.extend(d.prefix.lib64 + for d in self.spec.traverse(root=False) if os.path.isdir(d.prefix.lib64)) return rpaths - @property def rpath_args(self): - """Get the rpath args as a string, with -Wl,-rpath, for each element.""" + """ + Get the rpath args as a string, with -Wl,-rpath, for each element + """ return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) @@ -1333,6 +1340,7 @@ def install_dependency_symlinks(pkg, spec, prefix): """Execute a dummy install and flatten dependencies""" flatten_dependencies(spec, prefix) + def flatten_dependencies(spec, flat_dir): """Make each dependency of spec present in dir via symlink.""" for dep in spec.traverse(root=False): @@ -1341,13 +1349,13 @@ def flatten_dependencies(spec, flat_dir): dep_path = spack.install_layout.path_for_spec(dep) dep_files = LinkTree(dep_path) - os.mkdir(flat_dir+'/'+name) + os.mkdir(flat_dir + '/' + name) - conflict = dep_files.find_conflict(flat_dir+'/'+name) + conflict = dep_files.find_conflict(flat_dir + '/' + name) if conflict: raise DependencyConflictError(conflict) - dep_files.merge(flat_dir+'/'+name) + dep_files.merge(flat_dir + '/' + name) def validate_package_url(url_string): @@ -1388,9 +1396,11 @@ def dump_packages(spec, path): # Create a source repo and get the pkg directory out of it. try: source_repo = spack.repository.Repo(source_repo_root) - source_pkg_dir = source_repo.dirname_for_package_name(node.name) - except RepoError as e: - tty.warn("Warning: Couldn't copy in provenance for %s" % node.name) + source_pkg_dir = source_repo.dirname_for_package_name( + node.name) + except RepoError: + tty.warn("Warning: Couldn't copy in provenance for %s" % + node.name) # Create a destination repository dest_repo_root = join_path(path, node.namespace) @@ -1410,7 +1420,7 @@ def print_pkg(message): """Outputs a message with a package icon.""" from llnl.util.tty.color import cwrite cwrite('@*g{[+]} ') - print message + print(message) def _hms(seconds): @@ -1419,20 +1429,25 @@ def _hms(seconds): h, m = divmod(m, 60) parts = [] - if h: parts.append("%dh" % h) - if m: parts.append("%dm" % m) - if s: parts.append("%.2fs" % s) + if h: + parts.append("%dh" % h) + if m: + parts.append("%dm" % m) + if s: + parts.append("%.2fs" % s) return ' '.join(parts) class FetchError(spack.error.SpackError): """Raised when something goes wrong during fetch.""" + def __init__(self, message, long_msg=None): super(FetchError, self).__init__(message, long_msg) class InstallError(spack.error.SpackError): """Raised when something goes wrong during install or uninstall.""" + def __init__(self, message, long_msg=None): super(InstallError, self).__init__(message, long_msg) @@ -1443,21 +1458,24 @@ class ExternalPackageError(InstallError): class PackageStillNeededError(InstallError): """Raised when package is still needed by another on uninstall.""" + def __init__(self, spec, dependents): - super(PackageStillNeededError, self).__init__( - "Cannot uninstall %s" % spec) + super(PackageStillNeededError, self).__init__("Cannot uninstall %s" % + spec) self.spec = spec self.dependents = dependents class PackageError(spack.error.SpackError): """Raised when something is wrong with a package definition.""" + def __init__(self, message, long_msg=None): super(PackageError, self).__init__(message, long_msg) class PackageVersionError(PackageError): """Raised when a version URL cannot automatically be determined.""" + def __init__(self, version): super(PackageVersionError, self).__init__( "Cannot determine a URL automatically for version %s" % version, @@ -1466,6 +1484,7 @@ class PackageVersionError(PackageError): class VersionFetchError(PackageError): """Raised when a version URL cannot automatically be determined.""" + def __init__(self, cls): super(VersionFetchError, self).__init__( "Cannot fetch versions for package %s " % cls.__name__ + @@ -1474,12 +1493,15 @@ class VersionFetchError(PackageError): class NoURLError(PackageError): """Raised when someone tries to build a URL for a package with no URLs.""" + def __init__(self, cls): super(NoURLError, self).__init__( "Package %s has no version with a URL." % cls.__name__) -class ExtensionError(PackageError): pass +class ExtensionError(PackageError): + + pass class ExtensionConflictError(ExtensionError): @@ -1495,7 +1517,8 @@ class ActivationError(ExtensionError): class DependencyConflictError(spack.error.SpackError): """Raised when the dependencies cannot be flattened as asked for.""" + def __init__(self, conflict): super(DependencyConflictError, self).__init__( - "%s conflicts with another file in the flattened directory." %( + "%s conflicts with another file in the flattened directory." % ( conflict)) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 05f58ab7b1..10eaac1344 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -23,53 +23,23 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -import unittest -import nose -from spack.test.tally_plugin import Tally -from llnl.util.filesystem import join_path import llnl.util.tty as tty -from llnl.util.tty.colify import colify - +import nose import spack - +from llnl.util.filesystem import join_path +from llnl.util.tty.colify import colify +from spack.test.tally_plugin import Tally """Names of tests to be included in Spack's test suite""" -test_names = ['versions', - 'url_parse', - 'url_substitution', - 'packages', - 'stage', - 'spec_syntax', - 'spec_semantics', - 'spec_dag', - 'concretize', - 'multimethod', - 'install', - 'package_sanity', - 'config', - 'directory_layout', - 'pattern', - 'python_version', - 'git_fetch', - 'svn_fetch', - 'hg_fetch', - 'mirror', - 'modules', - 'url_extrapolate', - 'cc', - 'link_tree', - 'spec_yaml', - 'optional_deps', - 'make_executable', - 'configure_guess', - 'lock', - 'database', - 'namespace_trie', - 'yaml', - 'sbang', - 'environment', - 'cmd.uninstall', - 'cmd.test_install'] +test_names = ['versions', 'url_parse', 'url_substitution', 'packages', 'stage', + 'spec_syntax', 'spec_semantics', 'spec_dag', 'concretize', + 'multimethod', 'install', 'package_sanity', 'config', + 'directory_layout', 'pattern', 'python_version', 'git_fetch', + 'svn_fetch', 'hg_fetch', 'mirror', 'modules', 'url_extrapolate', + 'cc', 'link_tree', 'spec_yaml', 'optional_deps', + 'make_executable', 'configure_guess', 'lock', 'database', + 'namespace_trie', 'yaml', 'sbang', 'environment', + 'cmd.uninstall', 'cmd.test_install'] def list_tests(): @@ -80,8 +50,6 @@ def list_tests(): def run(names, outputDir, verbose=False): """Run tests with the supplied names. Names should be a list. If it's empty, run ALL of Spack's tests.""" - verbosity = 1 if not verbose else 2 - if not names: names = test_names else: @@ -95,7 +63,7 @@ def run(names, outputDir, verbose=False): tally = Tally() for test in names: module = 'spack.test.' + test - print module + print(module) tty.msg("Running test: %s" % test) @@ -105,15 +73,13 @@ def run(names, outputDir, verbose=False): xmlOutputFname = "unittests-{0}.xml".format(test) xmlOutputPath = join_path(outputDir, xmlOutputFname) runOpts += ["--with-xunit", - "--xunit-file={0}".format(xmlOutputPath)] + "--xunit-file={0}".format(xmlOutputPath)] argv = [""] + runOpts + [module] - result = nose.run(argv=argv, addplugins=[tally]) + nose.run(argv=argv, addplugins=[tally]) succeeded = not tally.failCount and not tally.errorCount - tty.msg("Tests Complete.", - "%5d tests run" % tally.numberOfTestsRun, - "%5d failures" % tally.failCount, - "%5d errors" % tally.errorCount) + tty.msg("Tests Complete.", "%5d tests run" % tally.numberOfTestsRun, + "%5d failures" % tally.failCount, "%5d errors" % tally.errorCount) if succeeded: tty.info("OK", format='g') diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index b8b0d6fc6a..c65d663250 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -2,14 +2,18 @@ import collections from contextlib import contextmanager import StringIO +import spack.modules +from spack.test.mock_packages_test import MockPackagesTest FILE_REGISTRY = collections.defaultdict(StringIO.StringIO) + # Monkey-patch open to write module files to a StringIO instance @contextmanager def mock_open(filename, mode): if not mode == 'w': - raise RuntimeError('test.modules : unexpected opening mode for monkey-patched open') + raise RuntimeError( + 'test.modules : unexpected opening mode for monkey-patched open') FILE_REGISTRY[filename] = StringIO.StringIO() @@ -20,7 +24,6 @@ def mock_open(filename, mode): FILE_REGISTRY[filename] = handle.getvalue() handle.close() -import spack.modules configuration_autoload_direct = { 'enable': ['tcl'], @@ -47,7 +50,8 @@ configuration_alter_environment = { 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']} }, '=x86-linux': { - 'environment': {'set': {'FOO': 'foo'}, 'unset': ['BAR']} + 'environment': {'set': {'FOO': 'foo'}, + 'unset': ['BAR']} } } } @@ -72,15 +76,14 @@ configuration_conflicts = { } } -from spack.test.mock_packages_test import MockPackagesTest - class TclTests(MockPackagesTest): def setUp(self): super(TclTests, self).setUp() self.configuration_obj = spack.modules.CONFIGURATION spack.modules.open = mock_open - spack.modules.CONFIGURATION = None # Make sure that a non-mocked configuration will trigger an error + # Make sure that a non-mocked configuration will trigger an error + spack.modules.CONFIGURATION = None def tearDown(self): del spack.modules.open @@ -98,7 +101,7 @@ class TclTests(MockPackagesTest): spack.modules.CONFIGURATION = configuration_autoload_direct spec = spack.spec.Spec('mpich@3.0.4=x86-linux') content = self.get_modulefile_content(spec) - self.assertTrue('module-whatis "mpich @3.0.4"' in content ) + self.assertTrue('module-whatis "mpich @3.0.4"' in content) def test_autoload(self): spack.modules.CONFIGURATION = configuration_autoload_direct @@ -117,14 +120,22 @@ class TclTests(MockPackagesTest): spack.modules.CONFIGURATION = configuration_alter_environment spec = spack.spec.Spec('mpileaks=x86-linux') content = self.get_modulefile_content(spec) - self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) - self.assertEqual(len([x for x in content if 'setenv FOO "foo"' in x]), 1) + self.assertEqual( + len([x + for x in content + if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) + self.assertEqual( + len([x for x in content if 'setenv FOO "foo"' in x]), 1) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 1) spec = spack.spec.Spec('libdwarf=x64-linux') content = self.get_modulefile_content(spec) - self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) - self.assertEqual(len([x for x in content if 'setenv FOO "foo"' in x]), 0) + self.assertEqual( + len([x + for x in content + if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) + self.assertEqual( + len([x for x in content if 'setenv FOO "foo"' in x]), 0) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 0) def test_blacklist(self): @@ -138,6 +149,9 @@ class TclTests(MockPackagesTest): spack.modules.CONFIGURATION = configuration_conflicts spec = spack.spec.Spec('mpileaks=x86-linux') content = self.get_modulefile_content(spec) - self.assertEqual(len([x for x in content if x.startswith('conflict')]), 2) - self.assertEqual(len([x for x in content if x == 'conflict mpileaks']), 1) - self.assertEqual(len([x for x in content if x == 'conflict intel/14.0.1']), 1) + self.assertEqual( + len([x for x in content if x.startswith('conflict')]), 2) + self.assertEqual( + len([x for x in content if x == 'conflict mpileaks']), 1) + self.assertEqual( + len([x for x in content if x == 'conflict intel/14.0.1']), 1) -- cgit v1.2.3-70-g09d2 From 0216fa2a00f3e438ec2cb3182c8b347f1038ae10 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 11 May 2016 09:05:24 -0700 Subject: Simplify /etc/spack/modules.yaml formatting. --- etc/spack/modules.yaml | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/etc/spack/modules.yaml b/etc/spack/modules.yaml index 8f8f88e908..99be5e7b6d 100644 --- a/etc/spack/modules.yaml +++ b/etc/spack/modules.yaml @@ -5,14 +5,25 @@ # although users can override these settings in their ~/.spack/modules.yaml. # ------------------------------------------------------------------------- modules: - prefix_inspections: { - bin: ['PATH'], - man: ['MANPATH'], - lib: ['LIBRARY_PATH', 'LD_LIBRARY_PATH'], - lib64: ['LIBRARY_PATH', 'LD_LIBRARY_PATH'], - include: ['CPATH'], - lib/pkgconfig: ['PKGCONFIG'], - lib64/pkgconfig: ['PKGCONFIG'], - '': ['CMAKE_PREFIX_PATH'] - } - enable: ['tcl', 'dotkit'] + enable: + - tcl + - dotkit + prefix_inspections: + bin: + - PATH + man: + - MANPATH + lib: + - LIBRARY_PATH + - LD_LIBRARY_PATH + lib64: + - LIBRARY_PATH + - LD_LIBRARY_PATH + include: + - CPATH + lib/pkgconfig: + - PKGCONFIG + lib64/pkgconfig: + - PKGCONFIG + '': + - CMAKE_PREFIX_PATH -- cgit v1.2.3-70-g09d2