summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorTodd Gamblin <tgamblin@llnl.gov>2014-06-19 10:34:21 -0700
committerTodd Gamblin <tgamblin@llnl.gov>2014-06-22 12:50:34 -0700
commit3653cfe6f08acc28ef9b65d8a81596119168b316 (patch)
tree5e6324a54ef25a4b0e9407fb6d05e862053074ae /lib
parentb6740cf1d1fe847ae0064f7d7a5be1b24dcf4779 (diff)
downloadspack-3653cfe6f08acc28ef9b65d8a81596119168b316.tar.gz
spack-3653cfe6f08acc28ef9b65d8a81596119168b316.tar.bz2
spack-3653cfe6f08acc28ef9b65d8a81596119168b316.tar.xz
spack-3653cfe6f08acc28ef9b65d8a81596119168b316.zip
Compiler support now uses configuration files.
- no more need for compiler python files. - Default compilers are found in user's environment and added to ~/.spackconfig automatically - User can add new compilers by editing configuration file
Diffstat (limited to 'lib')
-rwxr-xr-xlib/spack/env/cc17
-rw-r--r--lib/spack/spack/__init__.py5
-rw-r--r--lib/spack/spack/build_environment.py14
-rw-r--r--lib/spack/spack/cmd/uninstall.py13
-rw-r--r--lib/spack/spack/compiler.py145
-rw-r--r--lib/spack/spack/compilers/__init__.py107
-rw-r--r--lib/spack/spack/compilers/gcc.py3
-rw-r--r--lib/spack/spack/concretize.py8
-rw-r--r--lib/spack/spack/config.py39
-rw-r--r--lib/spack/spack/packages.py1
-rw-r--r--lib/spack/spack/spec.py2
11 files changed, 286 insertions, 68 deletions
diff --git a/lib/spack/env/cc b/lib/spack/env/cc
index 09abf3a31d..4478747844 100755
--- a/lib/spack/env/cc
+++ b/lib/spack/env/cc
@@ -27,10 +27,11 @@ spack_env_path = get_path("SPACK_ENV_PATH")
spack_debug_log_dir = get_env_var("SPACK_DEBUG_LOG_DIR")
spack_spec = get_env_var("SPACK_SPEC")
-spack_cc = get_env_var("SPACK_CC")
-spack_cxx = get_env_var("SPACK_CXX")
-spack_f77 = get_env_var("SPACK_F77")
-spack_fc = get_env_var("SPACK_FC")
+compiler_spec = get_env_var("SPACK_COMPILER_SPEC")
+spack_cc = get_env_var("SPACK_CC", required=False)
+spack_cxx = get_env_var("SPACK_CXX", required=False)
+spack_f77 = get_env_var("SPACK_F77", required=False)
+spack_fc = get_env_var("SPACK_FC", required=False)
# Figure out what type of operation we're doing
command = os.path.basename(sys.argv[0])
@@ -51,17 +52,25 @@ else:
if command in ('cc', 'gcc', 'c89', 'c99', 'clang'):
command = spack_cc
+ language = "C"
elif command in ('c++', 'CC', 'g++', 'clang++'):
command = spack_cxx
+ language = "C++"
elif command in ('f77'):
command = spack_f77
+ language = "Fortran 77"
elif command in ('fc'):
command = spack_fc
+ language = "Fortran 90"
elif command in ('ld', 'cpp'):
pass # leave it the same. TODO: what's the right thing?
else:
raise Exception("Unknown compiler: %s" % command)
+if command is None:
+ print "ERROR: Compiler '%s' does not support compiling %s programs." % (
+ compiler_spec, language)
+ sys.exit(1)
version_args = ['-V', '-v', '--version', '-dumpversion']
if any(arg in sys.argv for arg in version_args):
diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index 42769f0bdb..475fec1e1b 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -53,11 +53,6 @@ stage_path = join_path(var_path, "stage")
install_path = join_path(prefix, "opt")
#
-# Place to look for usable compiler versions.
-#
-compiler_version_path = join_path(var_path, "compilers")
-
-#
# Set up the packages database.
#
from spack.packages import PackageDB
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index 36dae74e84..492e490320 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -93,10 +93,16 @@ def set_compiler_environment_variables(pkg):
os.environ['FC'] = 'fc'
# Set SPACK compiler variables so that our wrapper knows what to call
- os.environ['SPACK_CC'] = compiler.cc.command
- os.environ['SPACK_CXX'] = compiler.cxx.command
- os.environ['SPACK_F77'] = compiler.f77.command
- os.environ['SPACK_FC'] = compiler.fc.command
+ if compiler.cc:
+ os.environ['SPACK_CC'] = compiler.cc.command
+ if compiler.cxx:
+ os.environ['SPACK_CXX'] = compiler.cxx.command
+ if compiler.f77:
+ os.environ['SPACK_F77'] = compiler.f77.command
+ if compiler.fc:
+ os.environ['SPACK_FC'] = compiler.fc.command
+
+ os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler)
def set_build_environment_variables(pkg):
diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py
index 044a9b2960..28a2f659ae 100644
--- a/lib/spack/spack/cmd/uninstall.py
+++ b/lib/spack/spack/cmd/uninstall.py
@@ -56,11 +56,14 @@ def uninstall(parser, args):
for spec in specs:
matching_specs = spack.db.get_installed(spec)
if not args.all and len(matching_specs) > 1:
- tty.die("%s matches multiple packages." % spec,
- "You can either:",
- " a) Use spack uninstall -a to uninstall ALL matching specs, or",
- " b) use a more specific spec.",
- "Matching packages:", *(" " + str(s) for s in matching_specs))
+ args = ["%s matches multiple packages." % spec,
+ "Matching packages:"]
+ args += [" " + str(s) for s in matching_specs]
+ args += ["You can either:",
+ " a) Use spack uninstall -a to uninstall ALL matching specs, or",
+ " b) use a more specific spec."]
+ tty.die(*args)
+
if len(matching_specs) == 0:
tty.die("%s does not match any installed packages." % spec)
diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py
index 3d097b6180..f6e173c359 100644
--- a/lib/spack/spack/compiler.py
+++ b/lib/spack/spack/compiler.py
@@ -1,17 +1,26 @@
import os
+import re
+import itertools
+
from llnl.util.lang import memoized
+from llnl.util.filesystem import join_path
import spack.error
+import spack.spec
from spack.version import Version
-from spack.util.executable import Executable
-
-
+from spack.util.executable import Executable, which
+from spack.compilation import get_path
+_default_order = ['']
def _verify_executables(*paths):
for path in paths:
if not os.path.isfile(path) and os.access(path, os.X_OK):
- raise InvalidCompilerPathError(path)
+ raise CompilerAccessError(path)
+
+@memoized
+def get_compiler_version(compiler, version_arg):
+ return compiler(version_arg, return_output=True)
class Compiler(object):
@@ -32,27 +41,135 @@ class Compiler(object):
# Subclasses use possible names of Fortran 90 compiler
fc_names = []
+ # Optional prefix regexes for searching for this type of compiler.
+ # Prefixes are sometimes used for toolchains, e.g. 'powerpc-bgq-linux-'
+ prefixes = []
+
+ # Optional suffix regexes for searching for this type of compiler.
+ # Suffixes are used by some frameworks, e.g. macports uses an '-mp-X.Y'
+ # version suffix for gcc.
+ suffixes = []
+
# Names of generic arguments used by this compiler
arg_version = '-dumpversion'
arg_rpath = '-Wl,-rpath,%s'
def __init__(self, cc, cxx, f77, fc):
- _verify_executables(cc, cxx, f77, fc)
+ def make_exe(exe):
+ if exe is None:
+ return None
+ _verify_executables(exe)
+ return Executable(exe)
+
+ self.cc = make_exe(cc)
+ self.cxx = make_exe(cxx)
+ self.f77 = make_exe(f77)
+ self.fc = make_exe(fc)
+
+
+ def _tuple(self):
+ return (self.cc, self.cxx, self.f77, self.fc)
- self.cc = Executable(cc)
- self.cxx = Executable(cxx)
- self.f77 = Executable(f77)
- self.fc = Executable(fc)
@property
- @memoized
def version(self):
- v = self.cc(arg_version)
- return Version(v)
+ for comp in self._tuple():
+ if comp is not None:
+ v = get_compiler_version(comp, self.arg_version)
+ return Version(v)
+ raise InvalidCompilerError()
+
+
+ @property
+ def spec(self):
+ return spack.spec.CompilerSpec(self.name, self.version)
+
+
+ @classmethod
+ def _find_matches_in_path(cls, compiler_names, *path):
+ """Try to find compilers with the supplied names in any of the suppled
+ paths. Searches for all combinations of each name with the
+ compiler's specified prefixes and suffixes. Compilers are
+ returned in a dict from (prefix, suffix) tuples to paths to
+ the compilers with those prefixes and suffixes.
+ """
+ if not path:
+ path = get_path('PATH')
+
+ matches = {}
+ ps_pairs = [p for p in itertools.product(
+ [''] + cls.prefixes, [''] + cls.suffixes)]
+
+ for directory in path:
+ files = os.listdir(directory)
+
+ for pre_re, suf_re in ps_pairs:
+ for compiler_name in compiler_names:
+ regex = r'^(%s)%s(%s)$' % (
+ pre_re, re.escape(compiler_name), suf_re)
+ for exe in files:
+ match = re.match(regex, exe)
+ if match:
+ pair = match.groups()
+ if pair not in matches:
+ matches[pair] = join_path(directory, exe)
-class InvalidCompilerPathError(spack.error.SpackError):
+ return matches
+
+ @classmethod
+ def find(cls, *path):
+ """Try to find this type of compiler in the user's environment. For
+ each set of compilers found, this returns a 4-tuple with
+ the cc, cxx, f77, and fc paths.
+
+ This will search for compilers with the names in cc_names,
+ cxx_names, etc. and it will group 4-tuples if they have
+ common prefixes and suffixes. e.g., gcc-mp-4.7 would be
+ grouped with g++-mp-4.7 and gfortran-mp-4.7.
+
+ Example return values::
+
+ [ ('/usr/bin/gcc', '/usr/bin/g++',
+ '/usr/bin/gfortran', '/usr/bin/gfortran'),
+
+ ('/usr/bin/gcc-mp-4.5', '/usr/bin/g++-mp-4.5',
+ '/usr/bin/gfortran-mp-4.5', '/usr/bin/gfortran-mp-4.5') ]
+
+ """
+ pair_names = [cls._find_matches_in_path(names, *path) for names in (
+ cls.cc_names, cls.cxx_names, cls.f77_names, cls.fc_names)]
+
+ keys = set()
+ for p in pair_names:
+ keys.update(p)
+
+ return [ tuple(pn[k] if k in pn else None for pn in pair_names)
+ for k in keys ]
+
+
+ def __repr__(self):
+ """Return a string represntation of the compiler toolchain."""
+ return self.__str__()
+
+
+ def __str__(self):
+ """Return a string represntation of the compiler toolchain."""
+ def p(path):
+ return ' '.join(path.exe) if path else None
+ return "%s(%s, %s, %s, %s)" % (
+ self.name,
+ p(self.cc), p(self.cxx), p(self.f77), p(self.fc))
+
+
+class CompilerAccessError(spack.error.SpackError):
def __init__(self, path):
- super(InvalidCompilerPathError, self).__init__(
+ super(CompilerAccessError, self).__init__(
"'%s' is not a valid compiler." % path)
+
+
+class InvalidCompilerError(spack.error.SpackError):
+ def __init__(self):
+ super(InvalidCompilerError, self).__init__(
+ "Compiler has no executables.")
diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py
index a36ea618bc..4b21dd6f7e 100644
--- a/lib/spack/spack/compilers/__init__.py
+++ b/lib/spack/spack/compilers/__init__.py
@@ -22,9 +22,6 @@
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
-#
-# This needs to be expanded for full compiler support.
-#
import imp
from llnl.util.lang import memoized, list_modules
@@ -33,12 +30,14 @@ from llnl.util.filesystem import join_path
import spack
import spack.error
import spack.spec
+import spack.config
+
from spack.compiler import Compiler
from spack.util.executable import which
from spack.util.naming import mod_to_class
-_imported_compilers_module = 'spack.compiler.versions'
-_imported_versions_module = 'spack.compilers'
+_imported_compilers_module = 'spack.compilers'
+_required_instance_vars = ['cc', 'cxx', 'f77', 'fc']
def _auto_compiler_spec(function):
@@ -49,7 +48,40 @@ def _auto_compiler_spec(function):
return converter
-@memoized
+def _get_config():
+ """Get a Spack config, but make sure it has compiler configuration
+ first."""
+ # If any configuration file has compilers, just stick with the
+ # ones already configured.
+ config = spack.config.get_config()
+ existing = [spack.spec.CompilerSpec(s)
+ for s in config.get_section_names('compiler')]
+ if existing:
+ return config
+
+ user_config = spack.config.get_config('user')
+
+ compilers = find_default_compilers()
+ for name, clist in compilers.items():
+ for compiler in clist:
+ if compiler.spec not in existing:
+ add_compiler(user_config, compiler)
+ user_config.write()
+
+ # After writing compilers to the user config, return a full config
+ # from all files.
+ return spack.config.get_config()
+
+
+def add_compiler(config, compiler):
+ def setup_field(cspec, name, exe):
+ path = ' '.join(exe.exe) if exe else "None"
+ config.set_value('compiler', cspec, name, path)
+
+ for c in _required_instance_vars:
+ setup_field(compiler.spec, c, getattr(compiler, c))
+
+
def supported_compilers():
"""Return a set of names of compilers supported by Spack.
@@ -65,13 +97,13 @@ def supported(compiler_spec):
return compiler_spec.name in supported_compilers()
-@memoized
def all_compilers():
"""Return a set of specs for all the compiler versions currently
available to build with. These are instances of CompilerSpec.
"""
- return set(spack.spec.CompilerSpec(c)
- for c in list_modules(spack.compiler_version_path))
+ configuration = _get_config()
+ return [spack.spec.CompilerSpec(s)
+ for s in configuration.get_section_names('compiler')]
@_auto_compiler_spec
@@ -86,20 +118,32 @@ def compilers_for_spec(compiler_spec):
"""This gets all compilers that satisfy the supplied CompilerSpec.
Returns an empty list if none are found.
"""
- matches = find(compiler_spec)
+ config = _get_config()
+
+ def get_compiler(cspec):
+ items = { k:v for k,v in config.items('compiler "%s"' % cspec) }
+
+ if not all(n in items for n in _required_instance_vars):
+ raise InvalidCompilerConfigurationError(cspec)
- compilers = []
- for cspec in matches:
- path = join_path(spack.compiler_version_path, "%s.py" % cspec)
- mod = imp.load_source(_imported_versions_module, path)
cls = class_for_compiler_name(cspec.name)
- compilers.append(cls(mod.cc, mod.cxx, mod.f77, mod.fc))
+ compiler_paths = []
+ for c in _required_instance_vars:
+ compiler_path = items[c]
+ if compiler_path != "None":
+ compiler_paths.append(compiler_path)
+ else:
+ compiler_paths.append(None)
+ return cls(*compiler_paths)
- return compilers
+ matches = find(compiler_spec)
+ return [get_compiler(cspec) for cspec in matches]
@_auto_compiler_spec
def compiler_for_spec(compiler_spec):
+ """Get the compiler that satisfies compiler_spec. compiler_spec must
+ be concrete."""
assert(compiler_spec.concrete)
compilers = compilers_for_spec(compiler_spec)
assert(len(compilers) == 1)
@@ -107,11 +151,32 @@ def compiler_for_spec(compiler_spec):
def class_for_compiler_name(compiler_name):
+ """Given a compiler module name, get the corresponding Compiler class."""
assert(supported(compiler_name))
file_path = join_path(spack.compilers_path, compiler_name + ".py")
compiler_mod = imp.load_source(_imported_compilers_module, file_path)
- return getattr(compiler_mod, mod_to_class(compiler_name))
+ cls = getattr(compiler_mod, mod_to_class(compiler_name))
+
+ # make a note of the name in the module so we can get to it easily.
+ cls.name = compiler_name
+
+ return cls
+
+
+def all_compiler_types():
+ return [class_for_compiler_name(c) for c in supported_compilers()]
+
+
+def find_default_compilers():
+ """Search the user's environment to get default compilers. Each
+ compiler class can have its own find() class method that can be
+ customized to locate that type of compiler.
+ """
+ # Compiler name is inserted on load by class_for_compiler_name
+ return {
+ Compiler.name : [Compiler(*c) for c in Compiler.find()]
+ for Compiler in all_compiler_types() }
@memoized
@@ -126,3 +191,11 @@ def default_compiler():
gcc = which('gcc', required=True)
version = gcc('-dumpversion', return_output=True)
return spack.spec.CompilerSpec('gcc', version)
+
+
+class InvalidCompilerConfigurationError(spack.error.SpackError):
+ def __init__(self, compiler_spec):
+ super(InvalidCompilerConfigurationError, self).__init__(
+ "Invalid configuration for [compiler \"%s\"]: " % compiler_spec,
+ "Compiler configuration must contain entries for all compilers: %s"
+ % _required_instance_vars)
diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py
index f73cb08c63..05dae96e65 100644
--- a/lib/spack/spack/compilers/gcc.py
+++ b/lib/spack/spack/compilers/gcc.py
@@ -37,5 +37,8 @@ class Gcc(Compiler):
# Subclasses use possible names of Fortran 90 compiler
fc_names = ['gfortran']
+ # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes.
+ suffixes = [r'-mp-\d\.\d']
+
def __init__(self, cc, cxx, f77, fc):
super(Gcc, self).__init__(cc, cxx, f77, fc)
diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index d4c7c25dca..f5775ef1bf 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -110,14 +110,18 @@ class DefaultConcretizer(object):
build with the compiler that will be used by libraries that
link to this one, to maximize compatibility.
"""
- if spec.compiler and spec.compiler.concrete:
+ all_compilers = spack.compilers.all_compilers()
+
+ if (spec.compiler and
+ spec.compiler.concrete and
+ spec.compiler in all_compilers):
return
try:
nearest = next(p for p in spec.preorder_traversal(direction='parents')
if p.compiler is not None).compiler
- if not nearest.concrete:
+ if not nearest in all_compilers:
# Take the newest compiler that saisfies the spec
matches = sorted(spack.compilers.find(nearest))
if not matches:
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index b36b83bfaa..9081a96b7b 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -88,7 +88,6 @@ import ConfigParser as cp
from llnl.util.lang import memoized
-import spack
import spack.error
__all__ = [
@@ -196,7 +195,7 @@ def _autokey(fun):
class SpackConfigParser(cp.RawConfigParser):
"""Slightly modified from Python's raw config file parser to accept
- leading whitespace.
+ leading whitespace and preserve comments.
"""
# Slightly modified Python option expression. This one allows
# leading whitespace.
@@ -305,6 +304,7 @@ class SpackConfigParser(cp.RawConfigParser):
cursect = None # None, or a dictionary
optname = None
lineno = 0
+ comment = 0
e = None # None, or an exception
while True:
line = fp.readline()
@@ -312,10 +312,10 @@ class SpackConfigParser(cp.RawConfigParser):
break
lineno = lineno + 1
# comment or blank line?
- if line.strip() == '' or line[0] in '#;':
- continue
- if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
- # no leading whitespace
+ if ((line.strip() == '' or line[0] in '#;') or
+ (line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR")):
+ self._sections["comment-%d" % comment] = line
+ comment += 1
continue
# a section header or option header?
else:
@@ -375,6 +375,10 @@ class SpackConfigParser(cp.RawConfigParser):
all_sections = [self._defaults]
all_sections.extend(self._sections.values())
for options in all_sections:
+ # skip comments
+ if isinstance(options, basestring):
+ continue
+
for name, val in options.items():
if isinstance(val, list):
options[name] = '\n'.join(val)
@@ -391,17 +395,22 @@ class SpackConfigParser(cp.RawConfigParser):
for (key, value) in self._defaults.items():
fp.write(" %s = %s\n" % (key, str(value).replace('\n', '\n\t')))
fp.write("\n")
+
for section in self._sections:
- # Allow leading whitespace
- fp.write("[%s]\n" % section)
- for (key, value) in self._sections[section].items():
- if key == "__name__":
- continue
- if (value is not None) or (self._optcre == self.OPTCRE):
- key = " = ".join((key, str(value).replace('\n', '\n\t')))
- fp.write(" %s\n" % (key))
- fp.write("\n")
+ # Handles comments and blank lines.
+ if isinstance(self._sections[section], basestring):
+ fp.write(self._sections[section])
+ continue
+ else:
+ # Allow leading whitespace
+ fp.write("[%s]\n" % section)
+ for (key, value) in self._sections[section].items():
+ if key == "__name__":
+ continue
+ if (value is not None) or (self._optcre == self.OPTCRE):
+ key = " = ".join((key, str(value).replace('\n', '\n\t')))
+ fp.write(" %s\n" % (key))
class SpackConfigurationError(spack.error.SpackError):
diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py
index 5a31f1fbb9..00834c95d5 100644
--- a/lib/spack/spack/packages.py
+++ b/lib/spack/spack/packages.py
@@ -32,7 +32,6 @@ import llnl.util.tty as tty
from llnl.util.filesystem import join_path
from llnl.util.lang import memoized
-import spack
import spack.error
import spack.spec
from spack.virtual import ProviderIndex
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 5848ac2000..1e2da10dcc 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -233,7 +233,7 @@ class CompilerSpec(object):
def concrete(self):
"""A CompilerSpec is concrete if its versions are concrete and there
is an available compiler with the right version."""
- return self.versions.concrete and self in compilers.all_compilers()
+ return self.versions.concrete
@property