From 552b4eae9e633e380e9ebe323222dc6a805aaaa2 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Sat, 3 Dec 2016 15:38:31 -0800 Subject: Fixes to Handling Multiple Architectures (#2261) * Added some notes about how multiarch detection could be fixed. * Implemented a preliminary version of the "spack.spec.ArchSpec" class. * Updated the "spack.spec.Spec" class to use "ArchSpec" instead of "Arch". * Fixed a number of small bugs in the "spack.spec.ArchSpec" class. * Fixed the 'Concretizer.concretize_architecture' method so that it uses the new architecture specs. * Updated the package class to properly use arch specs. Removed a number of unused architecture functions. * Fixed up a number of bugs that were causing the regression tests to fail. Added a couple of additional regression tests related to architecture parsing/specification. Fixed a few bugs with setting reserved os/target values on "ArchSpec" objects. Removed a number of unnecessary functions in the "spack.architecture" and "spack.concretize" modules. * Fixed a few bugs with reading architecture information from specs. Updated the tests to use a uniform architecture to improve reliability. Fixed a few minor style issues. * Adapted the compiler component of Spack to use arch specs. * Implemented more test cases for the extended architecture spec features. Improved error detection for multiple arch components in a spec. * Fix for backwards compatibility with v0.8 and prior * Changed os to unknown for compatibility specs * Use `spack09` instead of `spackcompat` for the platform of old specs. --- lib/spack/spack/architecture.py | 116 ++++------- lib/spack/spack/build_environment.py | 4 +- lib/spack/spack/compiler.py | 2 +- lib/spack/spack/compilers/__init__.py | 26 ++- lib/spack/spack/concretize.py | 75 ++------ lib/spack/spack/package.py | 9 +- lib/spack/spack/platforms/test.py | 14 +- lib/spack/spack/spec.py | 338 +++++++++++++++++++++++---------- lib/spack/spack/test/__init__.py | 11 ++ lib/spack/spack/test/architecture.py | 29 +-- lib/spack/spack/test/concretize.py | 2 +- lib/spack/spack/test/spec_semantics.py | 49 ++++- lib/spack/spack/test/spec_syntax.py | 70 +++++++ 13 files changed, 473 insertions(+), 272 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index b14de35109..6ea7bafb6a 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -126,8 +126,7 @@ class Target(object): @key_ordering class Platform(object): """ Abstract class that each type of Platform will subclass. - Will return a instance of it once it - is returned + Will return a instance of it once it is returned. """ priority = None # Subclass sets number. Controls detection order @@ -139,6 +138,9 @@ class Platform(object): back_os = None default_os = None + reserved_targets = ['default_target', 'frontend', 'fe', 'backend', 'be'] + reserved_oss = ['default_os', 'frontend', 'fe', 'backend', 'be'] + def __init__(self, name): self.targets = {} self.operating_sys = {} @@ -149,7 +151,7 @@ class Platform(object): Raises an error if the platform specifies a name that is reserved by spack as an alias. """ - if name in ['frontend', 'fe', 'backend', 'be', 'default_target']: + if name in Platform.reserved_targets: raise ValueError( "%s is a spack reserved alias " "and cannot be the name of a target" % name) @@ -174,7 +176,7 @@ class Platform(object): """ Add the operating_system class object into the platform.operating_sys dictionary """ - if name in ['frontend', 'fe', 'backend', 'be', 'default_os']: + if name in Platform.reserved_oss: raise ValueError( "%s is a spack reserved alias " "and cannot be the name of an OS" % name) @@ -241,7 +243,7 @@ class OperatingSystem(object): self.version = version def __str__(self): - return self.name + self.version + return "%s%s" % (self.name, self.version) def __repr__(self): return self.__str__() @@ -409,86 +411,52 @@ class Arch(object): return (platform, platform_os, target) def to_dict(self): - return syaml_dict(( - ('platform', - str(self.platform) if self.platform else None), - ('platform_os', - str(self.platform_os) if self.platform_os else None), - ('target', - str(self.target) if self.target else None))) - - -def _target_from_dict(target_name, plat=None): - """ Creates new instance of target and assigns all the attributes of - that target from the dictionary - """ - if not plat: - plat = platform() - return plat.target(target_name) + str_or_none = lambda v: str(v) if v else None + d = syaml_dict([ + ('platform', str_or_none(self.platform)), + ('platform_os', str_or_none(self.platform_os)), + ('target', str_or_none(self.target))]) + return syaml_dict([('arch', d)]) + @staticmethod + def from_dict(d): + spec = spack.spec.ArchSpec.from_dict(d) + return arch_for_spec(spec) -def _operating_system_from_dict(os_name, plat=None): - """ uses platform's operating system method to grab the constructed - operating systems that are valid on the platform. - """ - if not plat: - plat = platform() - if isinstance(os_name, dict): - name = os_name['name'] - version = os_name['version'] - return plat.operating_system(name + version) - else: - return plat.operating_system(os_name) - - -def _platform_from_dict(platform_name): - """ Constructs a platform from a dictionary. """ + +def get_platform(platform_name): + """Returns a platform object that corresponds to the given name.""" platform_list = all_platforms() for p in platform_list: if platform_name.replace("_", "").lower() == p.__name__.lower(): return p() -def arch_from_dict(d): - """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict - helper methods to recreate the arch tuple from the dictionary read from - a yaml file +def verify_platform(platform_name): + """ Determines whether or not the platform with the given name is supported + in Spack. For more information, see the 'spack.platforms' submodule. """ - arch = Arch() - - if isinstance(d, basestring): - # We have an old spec using a string for the architecture - arch.platform = Platform('spack_compatibility') - arch.platform_os = OperatingSystem('unknown', '') - arch.target = Target(d) - - arch.os_string = None - arch.target_string = None - else: - if d is None: - return None - platform_name = d['platform'] - os_name = d['platform_os'] - target_name = d['target'] - - if platform_name: - arch.platform = _platform_from_dict(platform_name) - else: - arch.platform = None - if target_name: - arch.target = _target_from_dict(target_name, arch.platform) - else: - arch.target = None - if os_name: - arch.platform_os = _operating_system_from_dict(os_name, - arch.platform) - else: - arch.platform_os = None + platform_name = platform_name.replace("_", "").lower() + platform_names = [p.__name__.lower() for p in all_platforms()] + + if platform_name not in platform_names: + tty.die("%s is not a supported platform; supported platforms are %s" % + (platform_name, platform_names)) + + +def arch_for_spec(arch_spec): + """Transforms the given architecture spec into an architecture objct.""" + arch_spec = spack.spec.ArchSpec(arch_spec) + assert(arch_spec.concrete) - arch.os_string = None - arch.target_string = None + arch_plat = get_platform(arch_spec.platform) + if not (arch_plat.operating_system(arch_spec.platform_os) and + arch_plat.target(arch_spec.target)): + raise ValueError( + "Can't recreate arch for spec %s on current arch %s; " + "spec architecture is too different" % (arch_spec, sys_type())) - return arch + return Arch(arch_plat, arch_spec.platform_os, arch_spec.target) @memoized diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 49df7a90b4..5e99d4552b 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -339,7 +339,7 @@ def set_build_environment_variables(pkg, env, dirty=False): if os.path.isdir(pcdir): env.prepend_path('PKG_CONFIG_PATH', pcdir) - if pkg.spec.architecture.target.module_name: + if pkg.architecture.target.module_name: load_module(pkg.spec.architecture.target.module_name) return env @@ -492,7 +492,7 @@ def setup_package(pkg, dirty=False): set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env, dirty) - pkg.spec.architecture.platform.setup_platform_environment(pkg, spack_env) + pkg.architecture.platform.setup_platform_environment(pkg, spack_env) load_external_modules(pkg) # traverse in postorder so package can use vars from its dependencies spec = pkg.spec diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 5170872cf7..4a7ee614d3 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -115,8 +115,8 @@ class Compiler(object): def __init__(self, cspec, operating_system, paths, modules=[], alias=None, environment=None, extra_rpaths=None, **kwargs): - self.operating_system = operating_system self.spec = cspec + self.operating_system = str(operating_system) self.modules = modules self.alias = alias diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 5b049367cc..9a9036da83 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -202,20 +202,23 @@ def find(compiler_spec, scope=None): @_auto_compiler_spec -def compilers_for_spec(compiler_spec, scope=None, **kwargs): +def compilers_for_spec(compiler_spec, arch_spec=None, scope=None): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ - platform = kwargs.get('platform', None) config = all_compilers_config(scope) def get_compilers(cspec): compilers = [] for items in config: - if items['compiler']['spec'] != str(cspec): - continue items = items['compiler'] + if items['spec'] != str(cspec): + continue + + os = items.get('operating_system', None) + if arch_spec and os != arch_spec.platform_os: + continue if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)): @@ -235,11 +238,6 @@ def compilers_for_spec(compiler_spec, scope=None, **kwargs): if mods == 'None': mods = [] - os = None - if 'operating_system' in items: - os = spack.architecture._operating_system_from_dict( - items['operating_system'], platform) - alias = items.get('alias', None) compiler_flags = items.get('flags', {}) environment = items.get('environment', {}) @@ -259,17 +257,15 @@ def compilers_for_spec(compiler_spec, scope=None, **kwargs): @_auto_compiler_spec -def compiler_for_spec(compiler_spec, arch): +def compiler_for_spec(compiler_spec, arch_spec): """Get the compiler that satisfies compiler_spec. compiler_spec must be concrete.""" - operating_system = arch.platform_os assert(compiler_spec.concrete) + assert(arch_spec.concrete) - compilers = [ - c for c in compilers_for_spec(compiler_spec, platform=arch.platform) - if c.operating_system == operating_system] + compilers = compilers_for_spec(compiler_spec, arch_spec=arch_spec) if len(compilers) < 1: - raise NoCompilerForSpecError(compiler_spec, operating_system) + raise NoCompilerForSpecError(compiler_spec, arch_spec.platform_os) if len(compilers) > 1: raise CompilerSpecInsufficientlySpecificError(compiler_spec) return compilers[0] diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index dcea147814..1bc933e543 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -240,47 +240,6 @@ class DefaultConcretizer(object): return True # Things changed - def _concretize_operating_system(self, spec): - if spec.architecture.platform_os is not None and isinstance( - spec.architecture.platform_os, - spack.architecture.OperatingSystem): - return False - - if spec.root.architecture and spec.root.architecture.platform_os: - if isinstance(spec.root.architecture.platform_os, - spack.architecture.OperatingSystem): - spec.architecture.platform_os = \ - spec.root.architecture.platform_os - else: - spec.architecture.platform_os = \ - spec.architecture.platform.operating_system('default_os') - return True # changed - - def _concretize_target(self, spec): - if spec.architecture.target is not None and isinstance( - spec.architecture.target, spack.architecture.Target): - return False - if spec.root.architecture and spec.root.architecture.target: - if isinstance(spec.root.architecture.target, - spack.architecture.Target): - spec.architecture.target = spec.root.architecture.target - else: - spec.architecture.target = spec.architecture.platform.target( - 'default_target') - return True # changed - - def _concretize_platform(self, spec): - if spec.architecture.platform is not None and isinstance( - spec.architecture.platform, spack.architecture.Platform): - return False - if spec.root.architecture and spec.root.architecture.platform: - if isinstance(spec.root.architecture.platform, - spack.architecture.Platform): - spec.architecture.platform = spec.root.architecture.platform - else: - spec.architecture.platform = spack.architecture.platform() - return True # changed? - def concretize_architecture(self, spec): """If the spec is empty provide the defaults of the platform. If the architecture is not a basestring, then check if either the platform, @@ -292,16 +251,25 @@ class DefaultConcretizer(object): DAG has an architecture, then use the root otherwise use the defaults on the platform. """ + root_arch = spec.root.architecture + sys_arch = spack.spec.ArchSpec(spack.architecture.sys_type()) + spec_changed = False + if spec.architecture is None: - # Set the architecture to all defaults - spec.architecture = spack.architecture.Arch() - return True + spec.architecture = spack.spec.ArchSpec(sys_arch) + spec_changed = True - # Concretize the operating_system and target based of the spec - ret = any((self._concretize_platform(spec), - self._concretize_operating_system(spec), - self._concretize_target(spec))) - return ret + default_archs = [root_arch, sys_arch] + while not spec.architecture.concrete and default_archs: + arch = default_archs.pop(0) + + replacement_fields = [k for k, v in arch.to_cmp_dict().iteritems() + if v and not getattr(spec.architecture, k)] + for field in replacement_fields: + setattr(spec.architecture, field, getattr(arch, field)) + spec_changed = True + + return spec_changed def concretize_variants(self, spec): """If the spec already has variants filled in, return. Otherwise, add @@ -343,13 +311,8 @@ class DefaultConcretizer(object): # Takes advantage of the proper logic already existing in # compiler_for_spec Should think whether this can be more # efficient - def _proper_compiler_style(cspec, arch): - platform = arch.platform - compilers = spack.compilers.compilers_for_spec(cspec, - platform=platform) - return filter(lambda c: c.operating_system == - arch.platform_os, compilers) - # return compilers + def _proper_compiler_style(cspec, aspec): + return spack.compilers.compilers_for_spec(cspec, arch_spec=aspec) all_compilers = spack.compilers.all_compilers() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8bb19042cc..9c946844c6 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -898,7 +898,14 @@ class PackageBase(object): return self.spec.prefix @property - # TODO: Change this to architecture + def architecture(self): + """Get the spack.architecture.Arch object that represents the + environment in which this package will be built.""" + if not self.spec.concrete: + raise ValueError("Can only get the arch for concrete package.") + return spack.architecture.arch_for_spec(self.spec.architecture) + + @property def compiler(self): """Get the spack.compiler.Compiler object used to build this package""" if not self.spec.concrete: diff --git a/lib/spack/spack/platforms/test.py b/lib/spack/spack/platforms/test.py index c918211555..a40e1f3b44 100644 --- a/lib/spack/spack/platforms/test.py +++ b/lib/spack/spack/platforms/test.py @@ -23,8 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack.architecture import Platform, Target -from spack.operating_systems.linux_distro import LinuxDistro -from spack.operating_systems.cnl import Cnl +from spack.architecture import OperatingSystem as OS class Test(Platform): @@ -33,18 +32,17 @@ class Test(Platform): back_end = 'x86_64' default = 'x86_64' - back_os = 'CNL10' - default_os = 'CNL10' + front_os = 'redhat6' + back_os = 'debian6' + default_os = 'debian6' def __init__(self): super(Test, self).__init__('test') self.add_target(self.default, Target(self.default)) self.add_target(self.front_end, Target(self.front_end)) - self.add_operating_system(self.default_os, Cnl()) - linux_dist = LinuxDistro() - self.front_os = linux_dist.name - self.add_operating_system(self.front_os, linux_dist) + self.add_operating_system(self.default_os, OS('debian', 6)) + self.add_operating_system(self.front_os, OS('redhat', 6)) @classmethod def detect(self): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 951e9ae652..b95f43e901 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -97,7 +97,6 @@ expansion when it is the first character in an id typed on the command line. """ import base64 import hashlib -import imp import ctypes from StringIO import StringIO from operator import attrgetter @@ -105,7 +104,6 @@ from operator import attrgetter from yaml.error import MarkedYAMLError import llnl.util.tty as tty -from llnl.util.filesystem import join_path from llnl.util.lang import * from llnl.util.tty.color import * @@ -116,7 +114,6 @@ import spack.compilers as compilers import spack.error import spack.parse from spack.build_environment import get_path_from_module, load_module -from spack.util.naming import mod_to_class from spack.util.prefix import Prefix from spack.util.string import * import spack.util.spack_yaml as syaml @@ -255,6 +252,204 @@ def colorize_spec(spec): return colorize(re.sub(_separators, insert_color(), str(spec)) + '@.') +@key_ordering +class ArchSpec(object): + """ The ArchSpec class represents an abstract architecture specification + that a package should be built with. At its core, each ArchSpec is + comprised of three elements: a platform (e.g. Linux), an OS (e.g. + RHEL6), and a target (e.g. x86_64). + """ + + # TODO: Formalize the specifications for architectures and then use + # the appropriate parser here to read these specifications. + def __init__(self, *args): + to_attr_string = lambda s: str(s) if s and s != "None" else None + + self.platform, self.platform_os, self.target = (None, None, None) + + if len(args) == 1: + spec_like = args[0] + if isinstance(spec_like, ArchSpec): + self._dup(spec_like) + elif isinstance(spec_like, basestring): + spec_fields = spec_like.split("-") + + if len(spec_fields) == 3: + self.platform, self.platform_os, self.target = tuple( + to_attr_string(f) for f in spec_fields) + else: + raise ValueError("%s is an invalid arch spec" % spec_like) + elif len(args) == 3: + self.platform = to_attr_string(args[0]) + self.platform_os = to_attr_string(args[1]) + self.target = to_attr_string(args[2]) + elif len(args) != 0: + raise TypeError("Can't make arch spec from %s" % args) + + def _autospec(self, spec_like): + if isinstance(spec_like, ArchSpec): + return spec_like + return ArchSpec(spec_like) + + def _cmp_key(self): + return (self.platform, self.platform_os, self.target) + + def _dup(self, other): + self.platform = other.platform + self.platform_os = other.platform_os + self.target = other.target + + @property + def platform(self): + return self._platform + + @platform.setter + def platform(self, value): + """ The platform of the architecture spec will be verified as a + supported Spack platform before it's set to ensure all specs + refer to valid platforms. + """ + value = str(value) if value is not None else None + self._platform = value + + @property + def platform_os(self): + return self._platform_os + + @platform_os.setter + def platform_os(self, value): + """ The OS of the architecture spec will update the platform field + if the OS is set to one of the reserved OS types so that the + default OS type can be resolved. Since the reserved OS + information is only available for the host machine, the platform + will assumed to be the host machine's platform. + """ + value = str(value) if value is not None else None + + if value in spack.architecture.Platform.reserved_oss: + curr_platform = str(spack.architecture.platform()) + self.platform = self.platform or curr_platform + + if self.platform != curr_platform: + raise ValueError( + "Can't set arch spec OS to reserved value '%s' when the " + "arch platform (%s) isn't the current platform (%s)" % + (value, self.platform, curr_platform)) + + spec_platform = spack.architecture.get_platform(self.platform) + value = str(spec_platform.operating_system(value)) + + self._platform_os = value + + @property + def target(self): + return self._target + + @target.setter + def target(self, value): + """ The target of the architecture spec will update the platform field + if the target is set to one of the reserved target types so that + the default target type can be resolved. Since the reserved target + information is only available for the host machine, the platform + will assumed to be the host machine's platform. + """ + value = str(value) if value is not None else None + + if value in spack.architecture.Platform.reserved_targets: + curr_platform = str(spack.architecture.platform()) + self.platform = self.platform or curr_platform + + if self.platform != curr_platform: + raise ValueError( + "Can't set arch spec target to reserved value '%s' when " + "the arch platform (%s) isn't the current platform (%s)" % + (value, self.platform, curr_platform)) + + spec_platform = spack.architecture.get_platform(self.platform) + value = str(spec_platform.target(value)) + + self._target = value + + def satisfies(self, other, strict=False): + other = self._autospec(other) + sdict, odict = self.to_cmp_dict(), other.to_cmp_dict() + + if strict or self.concrete: + return all(getattr(self, attr) == getattr(other, attr) + for attr in odict if odict[attr]) + else: + return all(getattr(self, attr) == getattr(other, attr) + for attr in odict if sdict[attr] and odict[attr]) + + def constrain(self, other): + """ Projects all architecture fields that are specified in the given + spec onto the instance spec if they're missing from the instance + spec. This will only work if the two specs are compatible. + """ + other = self._autospec(other) + + if not self.satisfies(other): + raise UnsatisfiableArchitectureSpecError(self, other) + + constrained = False + for attr, svalue in self.to_cmp_dict().iteritems(): + ovalue = getattr(other, attr) + if svalue is None and ovalue is not None: + setattr(self, attr, ovalue) + constrained = True + + return constrained + + def copy(self): + clone = ArchSpec.__new__(ArchSpec) + clone._dup(self) + return clone + + @property + def concrete(self): + return all(v for k, v in self.to_cmp_dict().iteritems()) + + def to_cmp_dict(self): + """Returns a dictionary that can be used for field comparison.""" + return dict([ + ('platform', self.platform), + ('platform_os', self.platform_os), + ('target', self.target)]) + + def to_dict(self): + d = syaml_dict([ + ('platform', self.platform), + ('platform_os', self.platform_os), + ('target', self.target)]) + return syaml_dict([('arch', d)]) + + @staticmethod + def from_dict(d): + """Import an ArchSpec from raw YAML/JSON data. + + This routine implements a measure of compatibility with older + versions of Spack. Spack releases before 0.10 used a single + string with no OS or platform identifiers. We import old Spack + architectures with platform ``spack09``, OS ``unknown``, and the + old arch string as the target. + + Specs from `0.10` or later have a more fleshed out architecture + descriptor with a platform, an OS, and a target. + + """ + if not isinstance(d['arch'], dict): + return ArchSpec('spack09', 'unknown', d['arch']) + + d = d['arch'] + return ArchSpec(d['platform'], d['platform_os'], d['target']) + + def __str__(self): + return "%s-%s-%s" % (self.platform, self.platform_os, self.target) + + def __repr__(self): + return str(self) + + @key_ordering class CompilerSpec(object): """The CompilerSpec field represents the compiler or range of compiler @@ -664,38 +859,42 @@ class Spec(object): """ valid_flags = FlagMap.valid_compiler_flags() if name == 'arch' or name == 'architecture': - parts = value.split('-') - if len(parts) == 3: - platform, op_sys, target = parts - else: - platform, op_sys, target = None, None, value - - assert(self.architecture.platform is None) - assert(self.architecture.platform_os is None) - assert(self.architecture.target is None) - assert(self.architecture.os_string is None) - assert(self.architecture.target_string is None) - self._set_platform(platform) - self._set_os(op_sys) - self._set_target(target) + parts = tuple(value.split('-')) + plat, os, tgt = parts if len(parts) == 3 else (None, None, value) + self._set_architecture(platform=plat, platform_os=os, target=tgt) elif name == 'platform': - self._set_platform(value) + self._set_architecture(platform=value) elif name == 'os' or name == 'operating_system': - if self.architecture.platform: - self._set_os(value) - else: - self.architecture.os_string = value + self._set_architecture(platform_os=value) elif name == 'target': - if self.architecture.platform: - self._set_target(value) - else: - self.architecture.target_string = value + self._set_architecture(target=value) elif name in valid_flags: assert(self.compiler_flags is not None) self.compiler_flags[name] = value.split() else: self._add_variant(name, value) + def _set_architecture(self, **kwargs): + """Called by the parser to set the architecture.""" + arch_attrs = ['platform', 'platform_os', 'target'] + if self.architecture and self.architecture.concrete: + raise DuplicateArchitectureError( + "Spec for '%s' cannot have two architectures." % self.name) + + if not self.architecture: + new_vals = tuple(kwargs.get(arg, None) for arg in arch_attrs) + self.architecture = ArchSpec(*new_vals) + else: + new_attrvals = [(a, v) for a, v in kwargs.iteritems() + if a in arch_attrs] + for new_attr, new_value in new_attrvals: + if getattr(self.architecture, new_attr): + raise DuplicateArchitectureError( + "Spec for '%s' cannot have two '%s' specified " + "for its architecture" % (self.name, new_attr)) + else: + setattr(self.architecture, new_attr, new_value) + def _set_compiler(self, compiler): """Called by the parser to set the compiler.""" if self.compiler: @@ -703,53 +902,6 @@ class Spec(object): "Spec for '%s' cannot have two compilers." % self.name) self.compiler = compiler - def _set_platform(self, value): - """Called by the parser to set the architecture platform""" - if isinstance(value, basestring): - mod_path = spack.platform_path - mod_string = 'spack.platformss' - names = list_modules(mod_path) - if value in names: - # Create a platform object from the name - mod_name = mod_string + value - path = join_path(mod_path, value) + '.py' - mod = imp.load_source(mod_name, path) - class_name = mod_to_class(value) - if not hasattr(mod, class_name): - tty.die( - 'No class %s defined in %s' % (class_name, mod_name)) - cls = getattr(mod, class_name) - if not inspect.isclass(cls): - tty.die('%s.%s is not a class' % (mod_name, class_name)) - platform = cls() - else: - tty.die("No platform class %s defined." % value) - else: - # The value is a platform - platform = value - - self.architecture.platform = platform - - # Set os and target if we previously got strings for them - if self.architecture.os_string: - self._set_os(self.architecture.os_string) - self.architecture.os_string = None - if self.architecture.target_string: - self._set_target(self.architecture.target_string) - self.architecture.target_string = None - - def _set_os(self, value): - """Called by the parser to set the architecture operating system""" - arch = self.architecture - if arch.platform: - arch.platform_os = arch.platform.operating_system(value) - - def _set_target(self, value): - """Called by the parser to set the architecture target""" - arch = self.architecture - if arch.platform: - arch.target = arch.platform.target(value) - def _add_dependency(self, spec, deptypes): """Called by the parser to add another spec as a dependency.""" if spec.name in self._dependencies: @@ -990,6 +1142,9 @@ class Spec(object): if self.versions: d.update(self.versions.to_dict()) + if self.architecture: + d.update(self.architecture.to_dict()) + if self.compiler: d.update(self.compiler.to_dict()) @@ -1002,9 +1157,6 @@ class Spec(object): if params: d['parameters'] = params - if self.architecture: - d['arch'] = self.architecture.to_dict() - # TODO: restore build dependencies here once we have less picky # TODO: concretization. deps = self.dependencies_dict(deptype=('link', 'run')) @@ -1042,7 +1194,7 @@ class Spec(object): spec.versions = VersionList.from_dict(node) if 'arch' in node: - spec.architecture = spack.architecture.arch_from_dict(node['arch']) + spec.architecture = ArchSpec.from_dict(node) if 'compiler' in node: spec.compiler = CompilerSpec.from_dict(node) @@ -1861,25 +2013,10 @@ class Spec(object): # Architecture satisfaction is currently just string equality. # If not strict, None means unconstrained. - sarch, oarch = self.architecture, other.architecture - if sarch and oarch: - if ((sarch.platform and - oarch.platform and - sarch.platform != oarch.platform) or - - (sarch.platform_os and - oarch.platform_os and - sarch.platform_os != oarch.platform_os) or - - (sarch.target and - oarch.target and - sarch.target != oarch.target)): + if self.architecture and other.architecture: + if not self.architecture.satisfies(other.architecture, strict): return False - - elif strict and ((oarch and not sarch) or - (oarch.platform and not sarch.platform) or - (oarch.platform_os and not sarch.platform_os) or - (oarch.target and not sarch.target)): + elif strict and (other.architecture and not self.architecture): return False if not self.compiler_flags.satisfies( @@ -1975,7 +2112,8 @@ class Spec(object): # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() - self.architecture = other.architecture + self.architecture = other.architecture.copy() if other.architecture \ + else None self.compiler = other.compiler.copy() if other.compiler else None if cleardeps: self._dependents = DependencyMap() @@ -2540,10 +2678,12 @@ class SpecParser(spack.parse.Parser): # If the spec has an os or a target and no platform, give it # the default platform + platform_default = spack.architecture.platform().name for spec in specs: for s in spec.traverse(): - if s.architecture.os_string or s.architecture.target_string: - s._set_platform(spack.architecture.platform()) + if s.architecture and not s.architecture.platform and \ + (s.architecture.platform_os or s.architecture.target): + s._set_architecture(platform=platform_default) return specs def parse_compiler(self, text): @@ -2585,7 +2725,7 @@ class SpecParser(spack.parse.Parser): spec.name = spec_name spec.versions = VersionList() spec.variants = VariantMap(spec) - spec.architecture = spack.architecture.Arch() + spec.architecture = None spec.compiler = None spec.external = None spec.external_module = None diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index c0a4c7354f..79122cc1de 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -28,9 +28,11 @@ import os import llnl.util.tty as tty import nose import spack +import spack.architecture from llnl.util.filesystem import join_path from llnl.util.tty.colify import colify from spack.test.tally_plugin import Tally +from spack.platforms.test import Test as TestPlatform """Names of tests to be included in Spack's test suite""" # All the tests Spack knows about. @@ -84,6 +86,13 @@ test_names = [ ] +def setup_tests(): + """Prepare the environment for the Spack tests to be run.""" + test_platform = TestPlatform() + spack.architecture.real_platform = spack.architecture.platform + spack.architecture.platform = lambda: test_platform + + def list_tests(): """Return names of all tests that can be run for Spack.""" return test_names @@ -117,6 +126,8 @@ def run(names, outputDir, verbose=False): runOpts += ["--with-xunit", "--xunit-file={0}".format(xmlOutputPath)] argv = [""] + runOpts + modules + + setup_tests() nose.run(argv=argv, addplugins=[tally]) succeeded = not tally.failCount and not tally.errorCount diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 22ddd4c97e..0ce583c6ea 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -54,10 +54,7 @@ class ArchitectureTest(MockPackagesTest): arch.platform_os = arch.platform.operating_system('default_os') arch.target = arch.platform.target('default_target') - d = arch.to_dict() - - new_arch = spack.architecture.arch_from_dict(d) - + new_arch = spack.architecture.Arch.from_dict(arch.to_dict()) self.assertEqual(arch, new_arch) self.assertTrue(isinstance(arch, spack.architecture.Arch)) @@ -75,7 +72,7 @@ class ArchitectureTest(MockPackagesTest): spack.architecture.Target)) def test_platform(self): - output_platform_class = spack.architecture.platform() + output_platform_class = spack.architecture.real_platform() if os.path.exists('/opt/cray/craype'): my_platform_class = Cray() elif os.path.exists('/bgsys'): @@ -114,10 +111,12 @@ class ArchitectureTest(MockPackagesTest): """Test when user inputs just frontend that both the frontend target and frontend operating system match """ - frontend_os = self.platform.operating_system("frontend") - frontend_target = self.platform.target("frontend") + frontend_os = str(self.platform.operating_system("frontend")) + frontend_target = str(self.platform.target("frontend")) + frontend_spec = Spec("libelf os=frontend target=frontend") frontend_spec.concretize() + self.assertEqual(frontend_os, frontend_spec.architecture.platform_os) self.assertEqual(frontend_target, frontend_spec.architecture.target) @@ -125,19 +124,22 @@ class ArchitectureTest(MockPackagesTest): """Test when user inputs backend that both the backend target and backend operating system match """ - backend_os = self.platform.operating_system("backend") - backend_target = self.platform.target("backend") + backend_os = str(self.platform.operating_system("backend")) + backend_target = str(self.platform.target("backend")) + backend_spec = Spec("libelf os=backend target=backend") backend_spec.concretize() + self.assertEqual(backend_os, backend_spec.architecture.platform_os) self.assertEqual(backend_target, backend_spec.architecture.target) def test_user_defaults(self): - default_os = self.platform.operating_system("default_os") - default_target = self.platform.target("default_target") + default_os = str(self.platform.operating_system("default_os")) + default_target = str(self.platform.target("default_target")) default_spec = Spec("libelf") # default is no args default_spec.concretize() + self.assertEqual(default_os, default_spec.architecture.platform_os) self.assertEqual(default_target, default_spec.architecture.target) @@ -156,8 +158,9 @@ class ArchitectureTest(MockPackagesTest): spec = Spec("libelf os=%s target=%s" % (o, t)) spec.concretize() results.append(spec.architecture.platform_os == - self.platform.operating_system(o)) - results.append(spec.architecture.target == self.platform.target(t)) + str(self.platform.operating_system(o))) + results.append(spec.architecture.target == + str(self.platform.target(t))) res = all(results) self.assertTrue(res) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 8ecbddbda2..154e0eb68e 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -250,7 +250,7 @@ class ConcretizeTest(MockPackagesTest): def test_external_package_module(self): # No tcl modules on darwin/linux machines # TODO: improved way to check for this. - platform = spack.architecture.platform().name + platform = spack.architecture.real_platform().name if (platform == 'darwin' or platform == 'linux'): return diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 79ffc99298..c165934948 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -132,15 +132,60 @@ class SpecSematicsTest(MockPackagesTest): self.check_unsatisfiable('foo %gcc@4.7', '%gcc@4.7.3') def test_satisfies_architecture(self): + self.check_satisfies( + 'foo platform=test', + 'platform=test') + self.check_satisfies( + 'foo platform=linux', + 'platform=linux') + self.check_satisfies( + 'foo platform=test', + 'platform=test target=frontend') + self.check_satisfies( + 'foo platform=test', + 'platform=test os=frontend target=frontend') + self.check_satisfies( + 'foo platform=test os=frontend target=frontend', + 'platform=test') + + self.check_unsatisfiable( + 'foo platform=linux', + 'platform=test os=redhat6 target=x86_32') + self.check_unsatisfiable( + 'foo os=redhat6', + 'platform=test os=debian6 target=x86_64') + self.check_unsatisfiable( + 'foo target=x86_64', + 'platform=test os=redhat6 target=x86_32') + + self.check_satisfies( + 'foo arch=test-None-None', + 'platform=test') + self.check_satisfies( + 'foo arch=test-None-frontend', + 'platform=test target=frontend') + self.check_satisfies( + 'foo arch=test-frontend-frontend', + 'platform=test os=frontend target=frontend') + self.check_satisfies( + 'foo arch=test-frontend-frontend', + 'platform=test') + self.check_unsatisfiable( + 'foo arch=test-frontend-frontend', + 'platform=test os=frontend target=backend') + self.check_satisfies( 'foo platform=test target=frontend os=frontend', 'platform=test target=frontend os=frontend') self.check_satisfies( 'foo platform=test target=backend os=backend', - 'platform=test target=backend', 'platform=test os=backend') + 'platform=test target=backend os=backend') self.check_satisfies( 'foo platform=test target=default_target os=default_os', - 'platform=test target=default_target os=default_os') + 'platform=test os=default_os') + self.check_unsatisfiable( + 'foo platform=test target=x86_32 os=redhat6', + 'platform=linux target=x86_32 os=redhat6') def test_satisfies_dependencies(self): self.check_satisfies('mpileaks^mpich', '^mpich') diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index d4eb9e057f..1e072fe970 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -120,6 +120,10 @@ class SpecSyntaxTest(unittest.TestCase): 'mvapich_foo' '^_openmpi@1.2:1.4,1.6%intel@12.1 cppflags="-O3"+debug~qt_4' '^stackwalker@8.1_1e') + self.check_parse( + "mvapich_foo" + "^_openmpi@1.2:1.4,1.6%intel@12.1 debug=2~qt_4" + "^stackwalker@8.1_1e arch=test-redhat6-x86_32") def test_canonicalize(self): self.check_parse( @@ -144,6 +148,22 @@ class SpecSyntaxTest(unittest.TestCase): "x^y@1,2:3,4%intel@1,2,3,4+a~b+c~d+e~f", "x ^y~f+e~d+c~b+a@4,2:3,1%intel@4,3,2,1") + self.check_parse( + "x arch=test-redhat6-None" + "^y arch=test-None-x86_64" + "^z arch=linux-None-None", + + "x os=fe" + "^y target=be" + "^z platform=linux") + + self.check_parse( + "x arch=test-debian6-x86_64" + "^y arch=test-debian6-x86_64", + + "x os=default_os target=default_target" + "^y os=default_os target=default_target") + self.check_parse("x^y", "x@: ^y@:") def test_parse_errors(self): @@ -169,10 +189,12 @@ class SpecSyntaxTest(unittest.TestCase): def test_duplicate_compiler(self): self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x%intel%intel") + self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x%intel%gcc") self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x%gcc%intel") + self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x ^y%intel%intel") self.assertRaises(DuplicateCompilerSpecError, @@ -180,6 +202,54 @@ class SpecSyntaxTest(unittest.TestCase): self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x ^y%gcc%intel") + def test_duplicate_architecture(self): + self.assertRaises( + DuplicateArchitectureError, self.check_parse, + "x arch=linux-rhel7-x86_64 arch=linux-rhel7-x86_64") + + self.assertRaises( + DuplicateArchitectureError, self.check_parse, + "x arch=linux-rhel7-x86_64 arch=linux-rhel7-ppc64le") + self.assertRaises( + DuplicateArchitectureError, self.check_parse, + "x arch=linux-rhel7-ppc64le arch=linux-rhel7-x86_64") + + self.assertRaises( + DuplicateArchitectureError, self.check_parse, + "y ^x arch=linux-rhel7-x86_64 arch=linux-rhel7-x86_64") + self.assertRaises( + DuplicateArchitectureError, self.check_parse, + "y ^x arch=linux-rhel7-x86_64 arch=linux-rhel7-ppc64le") + + def test_duplicate_architecture_component(self): + self.assertRaises( + DuplicateArchitectureError, self.check_parse, + "x os=fe os=fe") + self.assertRaises( + DuplicateArchitectureError, self.check_parse, + "x os=fe os=be") + + self.assertRaises( + DuplicateArchitectureError, self.check_parse, + "x target=fe target=fe") + self.assertRaises( + DuplicateArchitectureError, self.check_parse, + "x target=fe target=be") + + self.assertRaises( + DuplicateArchitectureError, self.check_parse, + "x platform=test platform=test") + self.assertRaises( + DuplicateArchitectureError, self.check_parse, + "x platform=test platform=test") + + self.assertRaises( + DuplicateArchitectureError, self.check_parse, + "x os=fe platform=test target=fe os=fe") + self.assertRaises( + DuplicateArchitectureError, self.check_parse, + "x target=be platform=test os=be os=fe") + # ======================================================================== # Lex checks # ======================================================================== -- cgit v1.2.3-60-g2f50