From 3f8613ae42b67007c8000a97371322c14cdaf5db Mon Sep 17 00:00:00 2001 From: becker33 Date: Thu, 8 Dec 2016 22:40:26 -0800 Subject: Make targets an attribute of compilers (#2500) * Make targets an attribute of compilers, similar to OS. Allows users to use `spack compiler find` to add compilers for the same platform/os but different targets when spack is installed on a file system mounted on machines with different targets. * Changed get_compilers() method to treat old compilers with no target as target='any' * flake8 changes * Address change requests for comments/code clarity --- lib/spack/spack/architecture.py | 3 ++- lib/spack/spack/cmd/compiler.py | 25 +++++++++++++++++-------- lib/spack/spack/compiler.py | 3 ++- lib/spack/spack/compilers/__init__.py | 16 ++++++++++++++-- lib/spack/spack/concretize.py | 31 +++++++++++++++++++------------ lib/spack/spack/operating_systems/cnl.py | 3 ++- 6 files changed, 56 insertions(+), 25 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index a77832fcb1..c86804deda 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -77,6 +77,7 @@ will be responsible for compiler detection. """ import os import inspect +import platform as py_platform from llnl.util.lang import memoized, list_modules, key_ordering from llnl.util.filesystem import join_path @@ -334,7 +335,7 @@ class OperatingSystem(object): if newcount <= prevcount: continue - compilers[ver] = cmp_cls(spec, self, paths) + compilers[ver] = cmp_cls(spec, self, py_platform.machine(), paths) return list(compilers.values()) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 80a3c71d61..609210f77e 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -32,7 +32,7 @@ import spack.spec from llnl.util.lang import index_by from llnl.util.tty.colify import colify from llnl.util.tty.color import colorize -from spack.spec import CompilerSpec +from spack.spec import CompilerSpec, ArchSpec from spack.util.environment import get_path description = "Manage compilers" @@ -91,17 +91,26 @@ def compiler_find(args): # Just let compiler_find do the # entire process and return an empty config from all_compilers # Default for any other process is init_config=True - compilers = [c for c in spack.compilers.find_compilers(*paths) - if c.spec not in spack.compilers.all_compilers( - scope=args.scope, init_config=False)] - if compilers: - spack.compilers.add_compilers_to_config(compilers, scope=args.scope, + compilers = [c for c in spack.compilers.find_compilers(*paths)] + new_compilers = [] + for c in compilers: + arch_spec = ArchSpec(None, c.operating_system, c.target) + same_specs = spack.compilers.compilers_for_spec(c.spec, + arch_spec, + args.scope) + + if not same_specs: + new_compilers.append(c) + + if new_compilers: + spack.compilers.add_compilers_to_config(new_compilers, + scope=args.scope, init_config=False) - n = len(compilers) + n = len(new_compilers) s = 's' if n > 1 else '' filename = spack.config.get_config_filename(args.scope, 'compilers') tty.msg("Added %d new compiler%s to %s" % (n, s, filename)) - colify(reversed(sorted(c.spec for c in compilers)), indent=4) + colify(reversed(sorted(c.spec for c in new_compilers)), indent=4) else: tty.msg("Found no new compilers") diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 4a7ee614d3..0d9727c69e 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -112,11 +112,12 @@ class Compiler(object): # Name of module used to switch versions of this compiler PrgEnv_compiler = None - def __init__(self, cspec, operating_system, + def __init__(self, cspec, operating_system, target, paths, modules=[], alias=None, environment=None, extra_rpaths=None, **kwargs): self.spec = cspec self.operating_system = str(operating_system) + self.target = str(target) self.modules = modules self.alias = alias diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 9a9036da83..6e65f50269 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -61,6 +61,7 @@ def _to_dict(compiler): for attr in _path_instance_vars) d['flags'] = dict((fname, fvals) for fname, fvals in compiler.flags) d['operating_system'] = str(compiler.operating_system) + d['target'] = str(compiler.target) d['modules'] = compiler.modules if compiler.modules else [] d['environment'] = compiler.environment if compiler.environment else {} d['extra_rpaths'] = compiler.extra_rpaths if compiler.extra_rpaths else [] @@ -216,10 +217,21 @@ def compilers_for_spec(compiler_spec, arch_spec=None, scope=None): if items['spec'] != str(cspec): continue + # If an arch spec is given, confirm that this compiler + # is for the given operating system os = items.get('operating_system', None) if arch_spec and os != arch_spec.platform_os: continue + # If an arch spec is given, confirm that this compiler + # is for the given target. If the target is 'any', match + # any given arch spec. If the compiler has no assigned + # target this is an old compiler config file, skip this logic. + target = items.get('target', None) + if arch_spec and target and (target != arch_spec.target and + target != 'any'): + continue + if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)): raise InvalidCompilerConfigurationError(cspec) @@ -244,8 +256,8 @@ def compilers_for_spec(compiler_spec, arch_spec=None, scope=None): extra_rpaths = items.get('extra_rpaths', []) compilers.append( - cls(cspec, os, compiler_paths, mods, alias, environment, - extra_rpaths, **compiler_flags)) + cls(cspec, os, target, compiler_paths, mods, alias, + environment, extra_rpaths, **compiler_flags)) return compilers diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 51d3c777fc..7eb05c49c9 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -305,10 +305,12 @@ class DefaultConcretizer(object): build with the compiler that will be used by libraries that link to this one, to maximize compatibility. """ - # Pass on concretizing the compiler if the target is not yet determined - if not spec.architecture.platform_os: - # Although this usually means changed, this means awaiting other - # changes + # Pass on concretizing the compiler if the target or operating system + # is not yet determined + if not (spec.architecture.platform_os and spec.architecture.target): + # We haven't changed, but other changes need to happen before we + # continue. `return True` here to force concretization to keep + # running. return True # Only use a matching compiler if it is of the proper style @@ -356,7 +358,8 @@ class DefaultConcretizer(object): if not matches: arch = spec.architecture raise UnavailableCompilerVersionError(other_compiler, - arch.platform_os) + arch.platform_os, + arch.target) # copy concrete version into other_compiler try: @@ -365,7 +368,8 @@ class DefaultConcretizer(object): if _proper_compiler_style(c, spec.architecture)).copy() except StopIteration: raise UnavailableCompilerVersionError( - spec.compiler, spec.architecture.platform_os + spec.compiler, spec.architecture.platform_os, + spec.architecture.target ) assert(spec.compiler.concrete) @@ -377,10 +381,12 @@ class DefaultConcretizer(object): compiler is used, defaulting to no compiler flags in the spec. Default specs set at the compiler level will still be added later. """ - - if not spec.architecture.platform_os: - # Although this usually means changed, this means awaiting other - # changes + # Pass on concretizing the compiler flags if the target or operating + # system is not set. + if not (spec.architecture.platform_os and spec.architecture.target): + # We haven't changed, but other changes need to happen before we + # continue. `return True` here to force concretization to keep + # running. return True def compiler_match(spec1, spec2): @@ -515,10 +521,11 @@ class UnavailableCompilerVersionError(spack.error.SpackError): """Raised when there is no available compiler that satisfies a compiler spec.""" - def __init__(self, compiler_spec, operating_system): + def __init__(self, compiler_spec, operating_system, target): super(UnavailableCompilerVersionError, self).__init__( "No available compiler version matches '%s' on operating_system %s" - % (compiler_spec, operating_system), + "for target %s" + % (compiler_spec, operating_system, target), "Run 'spack compilers' to see available compiler Options.") diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py index 78807865b3..bd128040ae 100644 --- a/lib/spack/spack/operating_systems/cnl.py +++ b/lib/spack/spack/operating_systems/cnl.py @@ -57,7 +57,8 @@ class Cnl(OperatingSystem): for name, version in matches: v = version comp = cmp_cls( - spack.spec.CompilerSpec(name + '@' + v), self, + spack.spec.CompilerSpec(name + '@' + v), + self, any, ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name + '/' + v]) compilers.append(comp) -- cgit v1.2.3-70-g09d2