From 33a11f32fdaea48192ecb13a85af11f85aa6b8bf Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 22 Jun 2014 12:26:36 -0700 Subject: Multi-compiler support feature-complete. Fix SPACK-3, SPACK-4, SPACK-12. - Fast compiler finding in path and for other directories - first time spack runs, it searches path. - user can add more compilers with 'spack compiler add' - Finds intel, gcc, clang, and pgi compilers with custom version args. - Builds can plug in alternate compilers with ease (e.g. %intel@12.1) --- lib/spack/spack/build_environment.py | 8 +- lib/spack/spack/cmd/compiler.py | 81 +++++++++++ lib/spack/spack/cmd/compilers.py | 12 +- lib/spack/spack/cmd/config.py | 61 +++++---- lib/spack/spack/compiler.py | 249 +++++++++++++++++++++++++--------- lib/spack/spack/compilers/__init__.py | 110 ++++++++++----- lib/spack/spack/compilers/clang.py | 15 +- lib/spack/spack/compilers/gcc.py | 15 +- lib/spack/spack/compilers/intel.py | 22 ++- lib/spack/spack/compilers/pgi.py | 51 +++++++ lib/spack/spack/util/multiproc.py | 45 ++++++ 11 files changed, 519 insertions(+), 150 deletions(-) create mode 100644 lib/spack/spack/cmd/compiler.py create mode 100644 lib/spack/spack/compilers/pgi.py create mode 100644 lib/spack/spack/util/multiproc.py diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 492e490320..b9d0c16353 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -94,13 +94,13 @@ def set_compiler_environment_variables(pkg): # Set SPACK compiler variables so that our wrapper knows what to call if compiler.cc: - os.environ['SPACK_CC'] = compiler.cc.command + os.environ['SPACK_CC'] = compiler.cc if compiler.cxx: - os.environ['SPACK_CXX'] = compiler.cxx.command + os.environ['SPACK_CXX'] = compiler.cxx if compiler.f77: - os.environ['SPACK_F77'] = compiler.f77.command + os.environ['SPACK_F77'] = compiler.f77 if compiler.fc: - os.environ['SPACK_FC'] = compiler.fc.command + os.environ['SPACK_FC'] = compiler.fc os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py new file mode 100644 index 0000000000..af69897ce9 --- /dev/null +++ b/lib/spack/spack/cmd/compiler.py @@ -0,0 +1,81 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import argparse + +from pprint import pprint + +import llnl.util.tty as tty +from llnl.util.tty.colify import colify +from llnl.util.lang import index_by + +import spack.compilers +import spack.spec +import spack.config +from spack.compilation import get_path + +description = "Manage compilers" + +def setup_parser(subparser): + sp = subparser.add_subparsers( + metavar='SUBCOMMAND', dest='compiler_command') + + update_parser = sp.add_parser( + 'add', help='Add compilers to the Spack configuration.') + update_parser.add_argument('add_paths', nargs=argparse.REMAINDER) + + remove_parser = sp.add_parser('remove', help='remove compiler') + remove_parser.add_argument('path') + + list_parser = sp.add_parser('list', help='list available compilers') + + +def compiler_add(args): + paths = args.add_paths + if not paths: + paths = get_path('PATH') + + compilers = spack.compilers.find_compilers(*args.add_paths) + spack.compilers.add_compilers_to_config('user', *compilers) + + +def compiler_remove(args): + pass + + +def compiler_list(args): + tty.msg("Available compilers") + + index = index_by(spack.compilers.all_compilers(), 'name') + for name, compilers in index.items(): + tty.hline(name, char='-', color=spack.spec.compiler_color) + colify(reversed(sorted(compilers)), indent=4) + + +def compiler(parser, args): + action = { 'add' : compiler_add, + 'remove' : compiler_remove, + 'list' : compiler_list } + action[args.compiler_command](args) + diff --git a/lib/spack/spack/cmd/compilers.py b/lib/spack/spack/cmd/compilers.py index c34118f033..8d046bfd7c 100644 --- a/lib/spack/spack/cmd/compilers.py +++ b/lib/spack/spack/cmd/compilers.py @@ -26,15 +26,9 @@ import llnl.util.tty as tty from llnl.util.tty.colify import colify from llnl.util.lang import index_by -import spack.compilers -import spack.spec +from spack.cmd.compiler import compiler_list -description = "List available compilers" +description = "List available compilers. Same as 'spack compiler list'." def compilers(parser, args): - tty.msg("Available compilers") - - index = index_by(spack.compilers.all_compilers(), 'name') - for name, compilers in index.items(): - tty.hline(name, char='-', color=spack.spec.compiler_color) - colify(compilers, indent=4) + compiler_list(args) diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index 25d302f94b..85f9642019 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -32,9 +32,8 @@ import spack.config description = "Get and set configuration options." def setup_parser(subparser): + # User can only choose one scope_group = subparser.add_mutually_exclusive_group() - - # File scope scope_group.add_argument( '--user', action='store_const', const='user', dest='scope', help="Use config file in user home directory (default).") @@ -42,36 +41,44 @@ def setup_parser(subparser): '--site', action='store_const', const='site', dest='scope', help="Use config file in spack prefix.") - # Get (vs. default set) - subparser.add_argument( - '--get', action='store_true', dest='get', - help="Get the value associated with a key.") + sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='config_command') - # positional arguments (value is only used on set) - subparser.add_argument( - 'key', help="Get the value associated with KEY") - subparser.add_argument( - 'value', nargs='?', default=None, - help="Value to associate with key") + set_parser = sp.add_parser('set', help='Set configuration values.') + set_parser.add_argument('key', help="Key to set value for.") + set_parser.add_argument('value', nargs='?', default=None, + help="Value to associate with key") + get_parser = sp.add_parser('get', help='Get configuration values.') + get_parser.add_argument('key', help="Key to get value for.") + + edit_parser = sp.add_parser('edit', help='Edit configuration file.') -def config(parser, args): - key, value = args.key, args.value - # If we're writing need to do a few checks. - if not args.get: - # Default scope for writing is user scope. - if not args.scope: - args.scope = 'user' +def config_set(args): + # default scope for writing is 'user' + if not args.scope: + args.scope = 'user' + + config = spack.config.get_config(args.scope) + config.set_value(args.key, args.value) + config.write() - if args.value is None: - tty.die("No value for '%s'. " % args.key - + "Spack config requires a key and a value.") +def config_get(args): config = spack.config.get_config(args.scope) + print config.get_value(args.key) + + +def config_edit(args): + if not args.scope: + args.scope = 'user' + config_file = spack.config.get_filename(args.scope) + spack.editor(config_file) + + +def config(parser, args): + action = { 'set' : config_set, + 'get' : config_get, + 'edit' : config_edit } + action[args.config_command](args) - if args.get: - print config.get_value(key) - else: - config.set_value(key, value) - config.write() diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index f6e173c359..b04034bbdd 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -1,26 +1,67 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## import os import re import itertools +from datetime import datetime +import llnl.util.tty as tty from llnl.util.lang import memoized from llnl.util.filesystem import join_path import spack.error import spack.spec +from spack.util.multiproc import parmap +from spack.util.executable import * from spack.version import Version -from spack.util.executable import Executable, which from spack.compilation import get_path -_default_order = [''] +__all__ = ['Compiler', 'get_compiler_version'] def _verify_executables(*paths): for path in paths: if not os.path.isfile(path) and os.access(path, os.X_OK): raise CompilerAccessError(path) -@memoized -def get_compiler_version(compiler, version_arg): - return compiler(version_arg, return_output=True) + +_version_cache = {} + +def get_compiler_version(compiler_path, version_arg, regex='(.*)'): + if not compiler_path in _version_cache: + compiler = Executable(compiler_path) + output = compiler(version_arg, return_output=True, error=None) + + match = re.search(regex, output) + _version_cache[compiler_path] = match.group(1) if match else None + + return _version_cache[compiler_path] + + +def dumpversion(compiler_path): + """Simple default dumpversion method -- this is what gcc does.""" + return get_compiler_version(compiler_path, '-dumpversion') class Compiler(object): @@ -48,37 +89,67 @@ class Compiler(object): # Optional suffix regexes for searching for this type of compiler. # Suffixes are used by some frameworks, e.g. macports uses an '-mp-X.Y' # version suffix for gcc. - suffixes = [] + suffixes = [r'-.*'] # Names of generic arguments used by this compiler - arg_version = '-dumpversion' arg_rpath = '-Wl,-rpath,%s' - def __init__(self, cc, cxx, f77, fc): - def make_exe(exe): + def __init__(self, cc, cxx, f77, fc, version=None): + def check(exe): if exe is None: return None _verify_executables(exe) - return Executable(exe) - - self.cc = make_exe(cc) - self.cxx = make_exe(cxx) - self.f77 = make_exe(f77) - self.fc = make_exe(fc) + return exe + self.cc = check(cc) + self.cxx = check(cxx) + self.f77 = check(f77) + self.fc = check(fc) - def _tuple(self): - return (self.cc, self.cxx, self.f77, self.fc) + # Allow versions to be memoized so we don't have to run + # compilers many times. Record them in the version cache if + # we get them in a constructor + # + # TODO: what to do if compilers have different versions? + # + self._version = version + self._cache_version() @property def version(self): - for comp in self._tuple(): - if comp is not None: - v = get_compiler_version(comp, self.arg_version) + if not self._version: + v = self.cc_version(self.cc) + if v is not None: + self._version = v + return Version(v) + + v = self.cxx_version(self.cxx) + if v is not None: + self._version = v return Version(v) - raise InvalidCompilerError() + + v = self.f77_version(self.f77) + if v is not None: + self._version = v + return Version(v) + + v = self.fc_version(self.fc) + if v is not None: + self._version = v + return Version(v) + + raise InvalidCompilerError() + + return Version(self._version) + + + def _cache_version(self): + _version_cache[self.cc] = self._version + _version_cache[self.cxx] = self._version + _version_cache[self.f77] = self._version + _version_cache[self.fc] = self._version @property @@ -87,66 +158,115 @@ class Compiler(object): @classmethod - def _find_matches_in_path(cls, compiler_names, *path): - """Try to find compilers with the supplied names in any of the suppled - paths. Searches for all combinations of each name with the - compiler's specified prefixes and suffixes. Compilers are - returned in a dict from (prefix, suffix) tuples to paths to - the compilers with those prefixes and suffixes. + def default_version(cls, cc): + """Override just this to override all compiler version functions.""" + return dumpversion(cc) + + @classmethod + def cc_version(cls, cc): + return cls.default_version(cc) + + @classmethod + def cxx_version(cls, cxx): + return cls.default_version(cxx) + + @classmethod + def f77_version(cls, f77): + return cls.default_version(f77) + + @classmethod + def fc_version(cls, fc): + return cls.default_version(fc) + + + @classmethod + def _find_matches_in_path(cls, compiler_names, detect_version, *path): + """Finds compilers in the paths supplied. + + Looks for all combinations of ``compiler_names`` with the + ``prefixes`` and ``suffixes`` defined for this compiler + class. If any compilers match the compiler_names, + prefixes, or suffixes, uses ``detect_version`` to figure + out what version the compiler is. + + This returns a dict with compilers grouped by (prefix, + suffix, version) tuples. This can be further organized by + find(). """ if not path: path = get_path('PATH') - matches = {} - ps_pairs = [p for p in itertools.product( - [''] + cls.prefixes, [''] + cls.suffixes)] + prefixes = [''] + cls.prefixes + suffixes = [''] + cls.suffixes + checks = [] for directory in path: files = os.listdir(directory) + for exe in files: + full_path = join_path(directory, exe) + + prod = itertools.product(prefixes, compiler_names, suffixes) + for pre, name, suf in prod: + regex = r'^(%s)%s(%s)$' % (pre, re.escape(name), suf) + + match = re.match(regex, exe) + if match: + key = (full_path,) + match.groups() + checks.append(key) + + def check(key): + try: + full_path, prefix, suffix = key + version = detect_version(full_path) + return (version, prefix, suffix, full_path) + except ProcessError, e: + tty.debug("Couldn't get version for compiler %s" % full_path, e) + return None - for pre_re, suf_re in ps_pairs: - for compiler_name in compiler_names: - regex = r'^(%s)%s(%s)$' % ( - pre_re, re.escape(compiler_name), suf_re) - - for exe in files: - match = re.match(regex, exe) - if match: - pair = match.groups() - if pair not in matches: - matches[pair] = join_path(directory, exe) - - return matches + successful = [key for key in parmap(check, checks) if key is not None] + return { (v, p, s) : path for v, p, s, path in successful } @classmethod def find(cls, *path): - """Try to find this type of compiler in the user's environment. For - each set of compilers found, this returns a 4-tuple with - the cc, cxx, f77, and fc paths. + """Try to find this type of compiler in the user's + environment. For each set of compilers found, this returns + compiler objects with the cc, cxx, f77, fc paths and the + version filled in. This will search for compilers with the names in cc_names, - cxx_names, etc. and it will group 4-tuples if they have - common prefixes and suffixes. e.g., gcc-mp-4.7 would be - grouped with g++-mp-4.7 and gfortran-mp-4.7. + cxx_names, etc. and it will group them if they have common + prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would + be grouped with g++-mp-4.7 and gfortran-mp-4.7. Example return values:: - [ ('/usr/bin/gcc', '/usr/bin/g++', - '/usr/bin/gfortran', '/usr/bin/gfortran'), - - ('/usr/bin/gcc-mp-4.5', '/usr/bin/g++-mp-4.5', - '/usr/bin/gfortran-mp-4.5', '/usr/bin/gfortran-mp-4.5') ] + [ gcc('/usr/bin/gcc', '/usr/bin/g++', + '/usr/bin/gfortran', '/usr/bin/gfortran', + Version('4.4.5')), + gcc('/usr/bin/gcc-mp-4.5', '/usr/bin/g++-mp-4.5', + '/usr/bin/gfortran-mp-4.5', '/usr/bin/gfortran-mp-4.5', + Version('4.7.2')) ] """ - pair_names = [cls._find_matches_in_path(names, *path) for names in ( - cls.cc_names, cls.cxx_names, cls.f77_names, cls.fc_names)] + dicts = parmap( + lambda t: cls._find_matches_in_path(*t), + [(cls.cc_names, cls.cc_version) + tuple(path), + (cls.cxx_names, cls.cxx_version) + tuple(path), + (cls.f77_names, cls.f77_version) + tuple(path), + (cls.fc_names, cls.fc_version) + tuple(path)]) + + all_keys = set() + for d in dicts: + all_keys.update(d) - keys = set() - for p in pair_names: - keys.update(p) + compilers = [] + for k in all_keys: + ver, pre, suf = k + paths = tuple(pn[k] if k in pn else None for pn in dicts) + args = paths + (ver,) + compilers.append(cls(*args)) - return [ tuple(pn[k] if k in pn else None for pn in pair_names) - for k in keys ] + return compilers def __repr__(self): @@ -156,11 +276,8 @@ class Compiler(object): def __str__(self): """Return a string represntation of the compiler toolchain.""" - def p(path): - return ' '.join(path.exe) if path else None - return "%s(%s, %s, %s, %s)" % ( - self.name, - p(self.cc), p(self.cxx), p(self.f77), p(self.fc)) + return "%s(%s)" % ( + self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) class CompilerAccessError(spack.error.SpackError): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 4b21dd6f7e..36d995b6bd 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -22,7 +22,11 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +"""This module contains functions related to finding compilers on the +system and configuring Spack to use multiple compilers. +""" import imp +import os from llnl.util.lang import memoized, list_modules from llnl.util.filesystem import join_path @@ -32,13 +36,16 @@ import spack.error import spack.spec import spack.config +from spack.util.multiproc import parmap from spack.compiler import Compiler from spack.util.executable import which from spack.util.naming import mod_to_class +from spack.compilation import get_path _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] +_default_order = ['gcc', 'intel', 'pgi', 'clang'] def _auto_compiler_spec(function): def converter(cspec_like): @@ -59,23 +66,72 @@ def _get_config(): if existing: return config - user_config = spack.config.get_config('user') - - compilers = find_default_compilers() - for name, clist in compilers.items(): - for compiler in clist: - if compiler.spec not in existing: - add_compiler(user_config, compiler) - user_config.write() + compilers = find_compilers(*get_path('PATH')) + new_compilers = [ + c for c in compilers if c.spec not in existing] + add_compilers_to_config('user', *new_compilers) # After writing compilers to the user config, return a full config # from all files. - return spack.config.get_config() + return spack.config.get_config(refresh=True) + + +@memoized +def default_compiler(): + versions = [] + for name in _default_order: # TODO: customize order. + versions = find(name) + if versions: break + + if not versions: + raise NoCompilersError() + + return sorted(versions)[-1] + + +def find_compilers(*path): + """Return a list of compilers found in the suppied paths. + This invokes the find() method for each Compiler class, + and appends the compilers detected to a list. + """ + # Make sure path elements exist, and include /bin directories + # under prefixes. + filtered_path = [] + for p in path: + # Eliminate symlinks and just take the real directories. + p = os.path.realpath(p) + if not os.path.isdir(p): + continue + filtered_path.append(p) + + # Check for a bin directory, add it if it exists + bin = join_path(p, 'bin') + if os.path.isdir(bin): + filtered_path.append(os.path.realpath(bin)) + + # Once the paths are cleaned up, do a search for each type of + # compiler. We can spawn a bunch of parallel searches to reduce + # the overhead of spelunking all these directories. + types = all_compiler_types() + compiler_lists = parmap(lambda cls: cls.find(*filtered_path), types) + + # ensure all the version calls we made are cached in the parent + # process, as well. This speeds up Spack a lot. + clist = reduce(lambda x,y: x+y, compiler_lists) + for c in clist: c._cache_version() + return clist + + +def add_compilers_to_config(scope, *compilers): + config = spack.config.get_config(scope) + for compiler in compilers: + add_compiler(config, compiler) + config.write() def add_compiler(config, compiler): def setup_field(cspec, name, exe): - path = ' '.join(exe.exe) if exe else "None" + path = exe if exe else "None" config.set_value('compiler', cspec, name, path) for c in _required_instance_vars: @@ -134,7 +190,9 @@ def compilers_for_spec(compiler_spec): compiler_paths.append(compiler_path) else: compiler_paths.append(None) - return cls(*compiler_paths) + + args = tuple(compiler_paths) + (compiler_spec.version,) + return cls(*args) matches = find(compiler_spec) return [get_compiler(cspec) for cspec in matches] @@ -168,34 +226,14 @@ def all_compiler_types(): return [class_for_compiler_name(c) for c in supported_compilers()] -def find_default_compilers(): - """Search the user's environment to get default compilers. Each - compiler class can have its own find() class method that can be - customized to locate that type of compiler. - """ - # Compiler name is inserted on load by class_for_compiler_name - return { - Compiler.name : [Compiler(*c) for c in Compiler.find()] - for Compiler in all_compiler_types() } - - -@memoized -def default_compiler(): - """Get the spec for the default compiler on the system. - Currently just returns the system's default gcc. - - TODO: provide a more sensible default. e.g. on Intel systems - we probably want icc. On Mac OS, clang. Probably need - to inspect the system and figure this out. - """ - gcc = which('gcc', required=True) - version = gcc('-dumpversion', return_output=True) - return spack.spec.CompilerSpec('gcc', version) - - class InvalidCompilerConfigurationError(spack.error.SpackError): def __init__(self, compiler_spec): super(InvalidCompilerConfigurationError, self).__init__( "Invalid configuration for [compiler \"%s\"]: " % compiler_spec, "Compiler configuration must contain entries for all compilers: %s" % _required_instance_vars) + + +class NoCompilersError(spack.error.SpackError): + def __init__(self): + super(NoCompilersError, self).__init__("Spack could not find any compilers!") diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 1616fcaf08..73c462b49f 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -37,5 +37,16 @@ class Clang(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = [] - def __init__(self, cc, cxx, f77, fc): - super(Clang, self).__init__(cc, cxx, f77, fc) + + @classmethod + def default_version(self, comp): + """The '--version' option works for clang compilers. + Output looks like this:: + + clang version 3.1 (trunk 149096) + Target: x86_64-unknown-linux-gnu + Thread model: posix + """ + return get_compiler_version( + comp, '--version', r'clang version ([^ ]+)') + diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index 05dae96e65..cc3c52ca61 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from spack.compiler import Compiler +from spack.compiler import * class Gcc(Compiler): # Subclasses use possible names of C compiler @@ -40,5 +40,14 @@ class Gcc(Compiler): # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. suffixes = [r'-mp-\d\.\d'] - def __init__(self, cc, cxx, f77, fc): - super(Gcc, self).__init__(cc, cxx, f77, fc) + @classmethod + def fc_version(cls, fc): + return get_compiler_version( + fc, '-dumpversion', + # older gfortran versions don't have simple dumpversion output. + r'(?:GNU Fortran \(GCC\))?(\d+\.\d+\.\d+)') + + + @classmethod + def f77_version(cls, f77): + return cls.fc_version(f77) diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index fe2aabd864..02e3b96b19 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from spack.compiler import Compiler +from spack.compiler import * class Intel(Compiler): # Subclasses use possible names of C compiler @@ -37,5 +37,21 @@ class Intel(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['ifort'] - def __init__(self, cc, cxx, f77, fc): - super(Intel, self).__init__(cc, cxx, f77, fc) + + @classmethod + def default_version(cls, comp): + """The '--version' option seems to be the most consistent one + for intel compilers. Output looks like this:: + + icpc (ICC) 12.1.5 20120612 + Copyright (C) 1985-2012 Intel Corporation. All rights reserved. + + or:: + + ifort (IFORT) 12.1.5 20120612 + Copyright (C) 1985-2012 Intel Corporation. All rights reserved. + """ + return get_compiler_version( + comp, '--version', r'\((?:IFORT|ICC)\) ([^ ]+)') + + diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py new file mode 100644 index 0000000000..d97f24c12e --- /dev/null +++ b/lib/spack/spack/compilers/pgi.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack.compiler import * + +class Pgi(Compiler): + # Subclasses use possible names of C compiler + cc_names = ['pgcc'] + + # Subclasses use possible names of C++ compiler + cxx_names = ['pgCC'] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['pgf77'] + + # Subclasses use possible names of Fortran 90 compiler + fc_names = ['pgf95', 'pgf90'] + + @classmethod + def default_version(cls, comp): + """The '-V' option works for all the PGI compilers. + Output looks like this:: + + pgf95 10.2-0 64-bit target on x86-64 Linux -tp nehalem-64 + Copyright 1989-2000, The Portland Group, Inc. All Rights Reserved. + Copyright 2000-2010, STMicroelectronics, Inc. All Rights Reserved. + """ + return get_compiler_version( + comp, '-V', r'pg[^ ]* ([^ ]+) \d\d\d?-bit target') + diff --git a/lib/spack/spack/util/multiproc.py b/lib/spack/spack/util/multiproc.py new file mode 100644 index 0000000000..9e045a090f --- /dev/null +++ b/lib/spack/spack/util/multiproc.py @@ -0,0 +1,45 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +""" +This implements a parallel map operation but it can accept more values +than multiprocessing.Pool.apply() can. For example, apply() will fail +to pickle functions if they're passed indirectly as parameters. +""" +from multiprocessing import Process, Pipe +from itertools import izip + +def spawn(f): + def fun(pipe,x): + pipe.send(f(x)) + pipe.close() + return fun + +def parmap(f,X): + pipe=[Pipe() for x in X] + proc=[Process(target=spawn(f),args=(c,x)) for x,(p,c) in izip(X,pipe)] + [p.start() for p in proc] + [p.join() for p in proc] + return [p.recv() for (p,c) in pipe] + -- cgit v1.2.3-70-g09d2