diff options
author | Gregory Becker <becker33@llnl.gov> | 2016-03-22 15:22:51 -0700 |
---|---|---|
committer | Gregory Becker <becker33@llnl.gov> | 2016-03-22 15:22:51 -0700 |
commit | 7c729d4c3c09848c6632795a752e8fecbc2e1efe (patch) | |
tree | 7eef7d767ddb33e674b3394dd28acd2a4cd67ba4 | |
parent | 670024cf7762044c300b874607ede940eef47556 (diff) | |
parent | 4f9a309de850f339ed43cd788be45d208bebf6c6 (diff) | |
download | spack-7c729d4c3c09848c6632795a752e8fecbc2e1efe.tar.gz spack-7c729d4c3c09848c6632795a752e8fecbc2e1efe.tar.bz2 spack-7c729d4c3c09848c6632795a752e8fecbc2e1efe.tar.xz spack-7c729d4c3c09848c6632795a752e8fecbc2e1efe.zip |
Merged newarch into merge
42 files changed, 1319 insertions, 184 deletions
diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 3051d3f742..f556b83924 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -39,7 +39,9 @@ spack_file = join_path(spack_root, "bin", "spack") lib_path = join_path(spack_root, "lib", "spack") build_env_path = join_path(lib_path, "env") module_path = join_path(lib_path, "spack") +platform_path = join_path(module_path, 'platforms') compilers_path = join_path(module_path, "compilers") +operating_system_path = join_path(module_path, 'operating_systems') test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") var_path = join_path(spack_root, "var", "spack") diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index 7e565bcbf9..d872952040 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -35,8 +35,9 @@ class ABI(object): The current implementation is rather rough and could be improved.""" def architecture_compatible(self, parent, child): - """Returns true iff the parent and child specs have ABI compatible architectures.""" - return not parent.architecture or not child.architecture or parent.architecture == child.architecture + """Returns true iff the parent and child specs have ABI compatible targets.""" + return not parent.architecture or not child.architecture \ + or parent.architecture == child.architecture @memoized diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 2701fab90c..aef38b8a55 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,13 +22,17 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os -import re -import platform +from collections import namedtuple +import imp +import platform as py_platform +import inspect -from llnl.util.lang import memoized +from llnl.util.lang import memoized, list_modules, key_ordering +from llnl.util.filesystem import join_path +import llnl.util.tty as tty import spack +from spack.util.naming import mod_to_class import spack.error as serr @@ -44,46 +48,302 @@ class NoSysTypeError(serr.SpackError): "Could not determine sys_type for this machine.") -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" - if not hasattr(spack, "sys_type"): - return None - elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() - else: - return spack.sys_type +@key_ordering +class Target(object): + """ Target is the processor of the host machine. + The host machine may have different front-end and back-end targets, + especially if it is a Cray machine. The target will have a name and + also the module_name (e.g craype-compiler). Targets will also + recognize which platform they came from using the set_platform method. + Targets will have compiler finding strategies + """ + + def __init__(self, name, module_name=None): + self.name = name # case of cray "ivybridge" but if it's x86_64 + self.module_name = module_name # craype-ivybridge + + # Sets only the platform name to avoid recursiveness + + def _cmp_key(self): + return (self.name, self.module_name) + + def __repr__(self): + return self.__str__() + + def __str__(self): + return self.name + + def to_dict(self): + d = {} + d['name'] = self.name + d['module_name'] = self.module_name + + return d + +@key_ordering +class Platform(object): + """ Abstract class that each type of Platform will subclass. + Will return a instance of it once it + is returned + """ + + priority = None # Subclass needs to set this number. This controls order in which platform is detected. + front_end = None + back_end = None + default = None # The default back end target. On cray ivybridge + + front_os = None + back_os = None + default_os = None + + def __init__(self, name): + self.targets = {} + self.operating_sys = {} + self.name = name + + def add_target(self, name, target): + """Used by the platform specific subclass to list available targets. + Raises an error if the platform specifies a name + that is reserved by spack as an alias. + """ + if name in ['front_end', 'fe', 'back_end', 'be', 'default']: + raise ValueError( + "%s is a spack reserved alias " + "and cannot be the name of a target" % name) + self.targets[name] = target + + def target(self, name): + """This is a getter method for the target dictionary + that handles defaulting based on the values provided by default, + front-end, and back-end. This can be overwritten + by a subclass for which we want to provide further aliasing options. + """ + if name == 'default': + name = self.default + elif name == 'front_end' or name == 'fe': + name = self.front_end + elif name == 'back_end' or name == 'be': + name = self.back_end + + return self.targets[name] + + def add_operating_system(self, name, os_class): + """ Add the operating_system class object into the + platform.operating_sys dictionary + """ + self.operating_sys[name] = os_class + + def operating_system(self, name): + if name == 'default_os': + name = self.default_os + if name == 'front_os': + name = self.front_os + if name == 'back_os': + name = self.back_os + + return self.operating_sys[name] + + @classmethod + def detect(self): + """ Subclass is responsible for implementing this method. + Returns True if the Platform class detects that + it is the current platform + and False if it's not. + """ + raise NotImplementedError() + + def __repr__(self): + return self.__str__() + + def __str__(self): + return self.name + + def _cmp_key(self): + return (self.name, (_cmp_key(t) for t in self.targets.values()), + (_cmp_key(o) for o in self.operating_sys.values())) + +@key_ordering +class OperatingSystem(object): + """ Operating System will be like a class similar to platform extended + by subclasses for the specifics. Operating System will contain the + compiler finding logic. Instead of calling two separate methods to + find compilers we call find_compilers method for each operating system + """ + + def __init__(self, name, version, compiler_strategy="PATH"): + self.name = name + self.version = version + self.compiler_strategy = compiler_strategy + + def __str__(self): + return self.name + self.version + + def __repr__(self): + return self.__str__() + + def _cmp_key(self): + return (self.name, self.version, self.compiler_strategy) + def to_dict(self): + d = {} + d['name'] = self.name + d['version'] = self.version + d['compiler_strategy'] = self.compiler_strategy -def get_sys_type_from_environment(): - """Return $SYS_TYPE or None if it's not defined.""" - return os.environ.get('SYS_TYPE') + return d +#NOTE: Key error caused because Architecture has no comparison method +@key_ordering +class Arch(object): + "Architecture is now a class to help with setting attributes" -def get_sys_type_from_platform(): - """Return the architecture from Python's platform module.""" - sys_type = platform.system() + '-' + platform.machine() - sys_type = re.sub(r'[^\w-]', '_', sys_type) - return sys_type.lower() + def __init__(self, platform_os=None, target=None): + self.platform = sys_type() + self.platform_os = platform_os + self.target = target + def __str__(self): + return (str(self.platform) +"-"+ + str(self.platform_os) + "-" + str(self.target) ) + + def _cmp_key(self): + platform = self.platform.name + os = self.platform_os.name if isinstance(self.platform_os, OperatingSystem) else self.platform_os + target = self.target.name if isinstance(self.target, Target) else self.target + return (platform, os, target) + + def to_dict(self): + d = {} + platform = self.platform + platform_os = self.platform_os + target = self.target + + d['platform'] = self.platform.name + d['platform_os'] = self.platform_os.to_dict() + d['target'] = self.target.to_dict() + + return d + +#def _helper_to_dict(arch_field_dict, arch_field_name, *args): +# """ General method to turn each class in architecture into a +# dictionary. Takes as argument the class dictionary, the field name +# (platform, platform_os, target) and then any attribute args +# """ +# d = {} +# d[arch_field_name] = {} +# for items in args: +# d[arch_field_name][items] = arch_field_dict[items] +# return d +# + +#def to_dict(arch): +# """ Convert the Arch tuple into a dictionary for yaml dumping. This +# uses the _helper_to_dict method to create the dictionary from the +# provided architecture field. Can assign the architecture +# field name (either platform, platform_os or target) and any +# attributes that make up that architecture field, +# """ +# d = {} +# +# platform = arch.platform.__dict__ +# platform_os = arch.platform_os.__dict__ +# target = arch.target.__dict__ +# +# platform_dict = _helper_to_dict(platform,'platform','name') +# os_dict = _helper_to_dict(platform_os, 'platform_os', 'name','version', +# 'compiler_strategy') +# target_dict = _helper_to_dict(target,'target', 'name', +# 'module_name','platform_name') +# +# d.update(platform_dict) +# d.update(os_dict) +# d.update(target_dict) +# +# return d + +#def _platform_from_dict(platform): +# """Creates all the platform class module names into a dictionary of +# name : <class_mod> key-value pairs. From there we can construct the +# platform subclass +# """ +# platform_list = all_platforms() +# platform_names = {plat.__name__.lower():plat for plat in platform_list} +# return platform_names[platform['name']]() + + +def _target_from_dict(target_dict): + """ Creates new instance of target and assigns all the attributes of + that target from the dictionary + """ + target = Target.__new__(Target) + target.name = target_dict['name'] + #target.compiler_strategy = target_dict['compiler_strategy'] + target.module_name = target_dict['module_name'] + if 'platform_name' in target_dict: + target.platform_name = target_dict['platform_name'] + return target + +def _operating_system_from_dict(os_dict, platform_class): + """ uses platform's operating system method to grab the constructed + operating systems that are valid on the platform. + """ +# NOTE: Might need a better way to create operating system objects + name = os_dict['name'] + return platform_class.operating_system(name) + + +def arch_from_dict(d): + """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict + helper methods to recreate the arch tuple from the dictionary read from + a yaml file + """ + arch = Arch() + + if d is None: + return None + os_dict = d['platform_os'] + target_dict = d['target'] + + target = _target_from_dict(target_dict) + platform_os = _operating_system_from_dict(os_dict, arch.platform) + arch.target =target + arch.platform_os = platform_os + + return arch @memoized -def sys_type(): - """Returns a SysType for the current machine.""" - methods = [get_sys_type_from_spack_globals, - get_sys_type_from_environment, - get_sys_type_from_platform] +def all_platforms(): + modules = [] + + mod_path = spack.platform_path + mod_string = "spack.platformss" - # search for a method that doesn't return None - sys_type = None - for method in methods: - sys_type = method() - if sys_type: break + for name in list_modules(mod_path): + mod_name = mod_string + name + path = join_path(mod_path, name) + ".py" + mod = imp.load_source(mod_name, path) + class_name = mod_to_class(name) + if not hasattr(mod, class_name): + tty.die('No class %s defined in %s' % (class_name, mod_name)) + cls = getattr(mod, class_name) + if not inspect.isclass(cls): + tty.die('%s.%s is not a class' % (mod_name, class_name)) - # Couldn't determine the sys_type for this machine. - if sys_type is None: - return "unknown_arch" + modules.append(cls) - if not isinstance(sys_type, basestring): - raise InvalidSysTypeError(sys_type) + return modules + +@memoized +def sys_type(): + """ Gather a list of all available subclasses of platforms. + Sorts the list according to their priority looking. Priority is + an arbitrarily set number. Detects platform either using uname or + a file path (/opt/cray...) + """ + # Try to create a Platform object using the config file FIRST + platform_list = all_platforms() + platform_list.sort(key=lambda a: a.priority) - return sys_type + for platform in platform_list: + if platform.detect(): + return platform() diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 87fc310b5a..a64ce8aeb4 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -32,6 +32,8 @@ import sys import shutil import multiprocessing import platform +import re + from llnl.util.filesystem import * import spack @@ -57,6 +59,9 @@ SPACK_DEBUG = 'SPACK_DEBUG' SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC' SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR' +SPACK_CRAYPE = 'SPACK_CRAYPE' +SPACK_COMP_MODULE = 'SPACK_COMP_MODULE' + class MakeExecutable(Executable): """Special callable executable object for make so the user can @@ -83,6 +88,68 @@ class MakeExecutable(Executable): return super(MakeExecutable, self).__call__(*args, **kwargs) +def load_module(mod): + """Takes a module name and removes modules until it is possible to + load that module. It then loads the provided module. Depends on the + modulecmd implementation of modules used in cray and lmod. + """ + #Create an executable of the module command that will output python code + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module and remove any conflicting modules + # We do this without checking that they are already installed + # for ease of programming because unloading a module that is not + # loaded does nothing. + text = modulecmd('show', mod, return_oe=True).split() + for i, word in enumerate(text): + if word == 'conflict': + exec(compile(modulecmd('unload', text[i+1], return_oe=True), '<string>', 'exec')) + # Load the module now that there are no conflicts + load = modulecmd('load', mod, return_oe=True) + exec(compile(load, '<string>', 'exec')) + + +def get_path_from_module(mod): + """Inspects a TCL module for entries that indicate the absolute path + at which the library supported by said module can be found. + """ + # Create a modulecmd executable + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module + text = modulecmd('show', mod, return_oe=True).split('\n') + + # If it lists its package directory, return that + for line in text: + if line.find(mod.upper()+'_DIR') >= 0: + words = line.split() + return words[2] + + # If it lists a -rpath instruction, use that + for line in text: + rpath = line.find('-rpath/') + if rpath >= 0: + return line[rpath+6:line.find('/lib')] + + # If it lists a -L instruction, use that + for line in text: + L = line.find('-L/') + if L >= 0: + return line[L+2:line.find('/lib')] + + # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that + for line in text: + if line.find('LD_LIBRARY_PATH') >= 0: + words = line.split() + path = words[2] + return path[:path.find('/lib')] + + # Unable to find module path + return None + + def set_compiler_environment_variables(pkg): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -109,6 +176,10 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) + if compiler.strategy == 'MODULES': + for mod in compiler.modules: + load_module(mod) + def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. @@ -172,6 +243,8 @@ def set_build_environment_variables(pkg): pkg_config_dirs.append(pcdir) path_set("PKG_CONFIG_PATH", pkg_config_dirs) + if pkg.spec.architecture.target.module_name: + load_module(pkg.spec.architecture.target.module_name) def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. @@ -241,10 +314,21 @@ def set_module_variables_for_package(pkg, m): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" + + # First load all modules for external packages and update the external + # packages' paths to reflect what is found in the modules so that we can + # rpath through the modules when possible, but if not possible they are + # already loaded. + for spec in pkg.spec.traverse(root=False): + if spec.external_module: + load_module(spec.external_module) + spec.external = get_path_from_module(spec.external_module) + + # Construct rpaths from the paths of each dep rpaths = [pkg.prefix.lib, pkg.prefix.lib64] - rpaths.extend(d.prefix.lib for d in pkg.spec.dependencies.values() + rpaths.extend(d.prefix.lib for d in pkg.spec.traverse(root=False) if os.path.isdir(d.prefix.lib)) - rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() + rpaths.extend(d.prefix.lib64 for d in pkg.spec.traverse(root=False) if os.path.isdir(d.prefix.lib64)) return rpaths diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 3e58e82184..bed5db3d47 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -30,6 +30,8 @@ from llnl.util.tty.color import colorize from llnl.util.tty.colify import colify from llnl.util.lang import index_by +import spack.architecture +import spack.compiler import spack.compilers import spack.spec import spack.config @@ -39,8 +41,7 @@ from spack.spec import CompilerSpec description = "Manage compilers" def setup_parser(subparser): - sp = subparser.add_subparsers( - metavar='SUBCOMMAND', dest='compiler_command') + sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='compiler_command') scopes = spack.config.config_scopes @@ -71,9 +72,11 @@ def setup_parser(subparser): def compiler_add(args): - """Search either $PATH or a list of paths for compilers and add them + """Search either $PATH or a list of paths OR MODULES for compilers and add them to Spack's configuration.""" - paths = args.add_paths + + + paths = args.add_paths # This might be a parser method. Parsing method to add_paths if not paths: paths = get_path('PATH') diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 714f1d514b..4223c53b06 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -89,19 +89,19 @@ def display_specs(specs, **kwargs): hashes = True hlen = None - # Make a dict with specs keyed by architecture and compiler. - index = index_by(specs, ('architecture', 'compiler')) + # Make a dict with specs keyed by target and compiler. + index = index_by(specs, ('target', 'compiler')) # Traverse the index and print out each package - for i, (architecture, compiler) in enumerate(sorted(index)): + for i, (target, compiler) in enumerate(sorted(index)): if i > 0: print header = "%s{%s} / %s{%s}" % ( - spack.spec.architecture_color, architecture, + spack.spec.target_color, target, spack.spec.compiler_color, compiler) tty.hline(colorize(header), char='-') - specs = index[(architecture,compiler)] + specs = index[(target,compiler)] specs.sort() nfmt = '.' if namespace else '_' diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index d38c0b00b1..e3ea918aca 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -33,6 +33,7 @@ from llnl.util.filesystem import join_path import spack.error import spack.spec +import spack.architecture from spack.util.multiproc import parmap from spack.util.executable import * from spack.util.environment import get_path @@ -98,19 +99,33 @@ class Compiler(object): cxx11_flag = "-std=c++11" - def __init__(self, cspec, cc, cxx, f77, fc): + # Cray PrgEnv name that can be used to load this compiler + PrgEnv = None + + # Name of module used to switch versions of this compiler + PrgEnv_compiler = None + + + def __init__(self, cspec, strategy, paths, modules=None): def check(exe): if exe is None: return None _verify_executables(exe) return exe - self.cc = check(cc) - self.cxx = check(cxx) - self.f77 = check(f77) - self.fc = check(fc) + self.strategy = strategy + + self.cc = check(paths[0]) + self.cxx = check(paths[1]) + if len(paths) > 2: + self.f77 = check(paths[2]) + if len(paths) == 3: + self.fc = self.f77 + else: + self.fc = check(paths[3]) self.spec = cspec + self.modules = modules @property @@ -206,6 +221,18 @@ class Compiler(object): @classmethod def find(cls, *path): + compilers = [] + platform = spack.architecture.sys_type() + strategies = [o.compiler_strategy for o in platform.operating_sys.values()] + if 'PATH' in strategies: + compilers.extend(cls.find_in_path(*path)) + if 'MODULES' in strategies: + compilers.extend(cls.find_in_modules()) + return compilers + + + @classmethod + def find_in_path(cls, *path): """Try to find this type of compiler in the user's environment. For each set of compilers found, this returns compiler objects with the cc, cxx, f77, fc paths and the @@ -250,20 +277,47 @@ class Compiler(object): if newcount <= prevcount: continue - compilers[ver] = cls(spec, *paths) + compilers[ver] = cls(spec, 'PATH', paths) return list(compilers.values()) + @classmethod + def find_in_modules(cls): + compilers = [] + + if cls.PrgEnv: + if not cls.PrgEnv_compiler: + tty.die('Must supply PrgEnv_compiler with PrgEnv') + + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + output = modulecmd('avail', cls.PrgEnv_compiler, return_oe=True) + matches = re.findall(r'(%s)/([\d\.]+[\d])' % cls.PrgEnv_compiler, output) + + for name, version in matches: + v = version + comp = cls(spack.spec.CompilerSpec(name + '@' + v), 'MODULES', + ['cc', 'CC', 'ftn'], [cls.PrgEnv, name +'/' + v]) + + compilers.append(comp) + + return compilers + + def __repr__(self): """Return a string representation of the compiler toolchain.""" return self.__str__() def __str__(self): - """Return a string representation of the compiler toolchain.""" - return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) + """Return a string represntation of the compiler toolchain.""" + if self.strategy is 'MODULES': + return "%s(%s)" % ( + self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc, self.modules)))) + else: + return "%s(%s)" % ( + self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc)))) class CompilerAccessError(spack.error.SpackError): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 3a04bc2ebc..93c5172dde 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -46,7 +46,9 @@ from spack.util.environment import get_path _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] +_optional_instance_vars = ['modules'] +_default_order = [] # TODO: customize order in config file if platform.system() == 'Darwin': _default_order = ['clang', 'gcc', 'intel'] @@ -121,7 +123,7 @@ def add_compilers_to_config(compilers, arch=None, scope=None): for compiler in compilers: compiler_config[str(compiler.spec)] = dict( (c, getattr(compiler, c, "None")) - for c in _required_instance_vars) + for c in _required_instance_vars + ['strategy'] + _optional_instance_vars) update = { arch : compiler_config } spack.config.update_config('compilers', update, scope) @@ -247,6 +249,11 @@ def compilers_for_spec(compiler_spec, arch=None, scope=None): raise InvalidCompilerConfigurationError(cspec) cls = class_for_compiler_name(cspec.name) + + strategy = items['strategy'] + if not strategy: + raise InvalidCompilerConfigurationError(cspec) + compiler_paths = [] for c in _required_instance_vars: compiler_path = items[c] @@ -255,19 +262,28 @@ def compilers_for_spec(compiler_spec, arch=None, scope=None): else: compiler_paths.append(None) - return cls(cspec, *compiler_paths) + for m in _optional_instance_vars: + if m not in items: + items[m] = None + mods = items[m] + + return cls(cspec, strategy, compiler_paths, mods) matches = find(compiler_spec, arch, scope) return [get_compiler(cspec) for cspec in matches] @_auto_compiler_spec -def compiler_for_spec(compiler_spec): +def compiler_for_spec(compiler_spec, operating_system): """Get the compiler that satisfies compiler_spec. compiler_spec must be concrete.""" assert(compiler_spec.concrete) - compilers = compilers_for_spec(compiler_spec) - assert(len(compilers) == 1) + compilers = [c for c in compilers_for_spec(compiler_spec) + if c.strategy == operating_system.compiler_strategy] + if len(compilers) < 1: + raise NoCompilerForSpecError(compiler_spec, operating_system) + if len(compilers) > 1: + raise CompilerSpecInsufficientlySpecificError(compiler_spec) return compilers[0] @@ -286,6 +302,7 @@ def class_for_compiler_name(compiler_name): def all_compiler_types(): +# return [class_for_compiler_name(c) for c in ['gcc']] return [class_for_compiler_name(c) for c in supported_compilers()] @@ -300,3 +317,13 @@ class InvalidCompilerConfigurationError(spack.error.SpackError): class NoCompilersError(spack.error.SpackError): def __init__(self): super(NoCompilersError, self).__init__("Spack could not find any compilers!") + +class NoCompilerForSpecError(spack.error.SpackError): + def __init__(self, compiler_spec, target): + super(NoCompilerForSpecError, self).__init__("No compilers for target %s satisfy spec %s" % ( + target, compiler_spec)) + +class CompilerSpecInsufficientlySpecificError(spack.error.SpackError): + def __init__(self, compiler_spec): + super(CompilerSpecInsufficientlySpecificError, self).__init__("Multiple compilers satisfy spec %s", + compiler_spec) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index e406d86a24..8226f62eb2 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -48,7 +48,7 @@ class Clang(Compiler): 'fc' : 'f90' } @classmethod - def default_version(self, comp): + def default_version(cls, comp): """The '--version' option works for clang compilers. On most platforms, output looks like this:: diff --git a/lib/spack/spack/compilers/craype.py b/lib/spack/spack/compilers/craype.py new file mode 100644 index 0000000000..e8ae284f5b --- /dev/null +++ b/lib/spack/spack/compilers/craype.py @@ -0,0 +1,57 @@ +##############################################################################} +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import llnl.util.tty as tty + +#from spack.build_environment import load_module +from spack.compiler import * +#from spack.version import ver + +class Craype(Compiler): + # Subclasses use possible names of C compiler + cc_names = ['cc'] + + # Subclasses use possible names of C++ compiler + cxx_names = ['CC'] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['ftn'] + + # Subclasses use possible names of Fortran 90 compiler + fc_names = ['ftn'] + + # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. + suffixes = [r'-mp-\d\.\d'] + + PrgEnv = 'PrgEnv-cray' + PrgEnv_compiler = 'craype' + +# @property +# def cxx11_flag(self): +# return "-hstd=c++11" + + @classmethod + def default_version(cls, comp): + return get_compiler_version(comp, r'([Vv]ersion).*(\d+(\.\d+)+)') + diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index 495b638a3a..c1fe7496c7 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -48,6 +48,9 @@ class Gcc(Compiler): 'f77' : 'gcc/gfortran', 'fc' : 'gcc/gfortran' } + PrgEnv = 'PrgEnv-gnu' + PrgEnv_compiler = 'gcc' + @property def cxx11_flag(self): if self.version < ver('4.3'): @@ -62,9 +65,9 @@ class Gcc(Compiler): return get_compiler_version( fc, '-dumpversion', # older gfortran versions don't have simple dumpversion output. - r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)') + r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)', module) @classmethod def f77_version(cls, f77): - return cls.fc_version(f77) + return cls.fc_version(f77, module) diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 69e9764790..80de0867fc 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -43,6 +43,9 @@ class Intel(Compiler): 'f77' : 'intel/ifort', 'fc' : 'intel/ifort' } + PrgEnv = 'PrgEnv-intel' + PrgEnv_compiler = 'intel' + @property def cxx11_flag(self): if self.version < ver('11.1'): diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index c6a1078bd9..9c155c1ec4 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -43,6 +43,9 @@ class Pgi(Compiler): 'f77' : 'pgi/pgfortran', 'fc' : 'pgi/pgfortran' } + PrgEnv = 'PrgEnv-pgi' + PrgEnv_compiler = 'pgi' + @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index c1d55109a3..3041da9047 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -51,8 +51,9 @@ class Xl(Compiler): else: return "-qlanglvl=extended0x" + @classmethod - def default_version(self, comp): + def default_version(cls, comp): """The '-qversion' is the standard option fo XL compilers. Output looks like this:: @@ -78,6 +79,7 @@ class Xl(Compiler): return get_compiler_version( comp, '-qversion',r'([0-9]?[0-9]\.[0-9])') + @classmethod def fc_version(cls, fc): """The fortran and C/C++ versions of the XL compiler are always two units apart. diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index bad67c34e3..330ecdf509 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -33,11 +33,14 @@ or user preferences. TODO: make this customizable and allow users to configure concretization policies. """ +import collections +from llnl.util.filesystem import join_path import spack import spack.spec import spack.compilers import spack.architecture import spack.error +from spack.util.naming import mod_to_class from spack.version import * from functools import partial from spec import DependencyMap @@ -80,6 +83,11 @@ class DefaultConcretizer(object): for ext in externals: if ext[0].satisfies(spec): result.append(ext) +# if externals: +# sorted_externals = sorted(externals, cmp=lambda a,b: a[0].__cmp__(b[0])) +# for external in sorted_externals: +# if external[0].satisfies(spec): +# result.append(external) if not result: raise NoBuildError(spec) @@ -120,8 +128,10 @@ class DefaultConcretizer(object): if not candidate: # No ABI matches. Pick the top choice based on the orignal preferences. candidate = candidates[0] - candidate_spec = candidate[0] + + external_module = candidate[2] external = candidate[1] + candidate_spec = candidate[0] changed = False # If we're external then trim the dependencies @@ -148,7 +158,13 @@ class DefaultConcretizer(object): changed = True if spec._dup(candidate_spec, deps=False, cleardeps=False): changed = True - spec.external = external + + if not spec.external and external: + spec.external = external + changed = True + if not spec.external_module and external_module: + spec.external_module = external_module + changed = True return changed @@ -173,7 +189,7 @@ class DefaultConcretizer(object): # If there are known available versions, return the most recent # version that satisfies the spec - pkg = spec.package + pkg = spec.package # Gives error here with dynist cmp_versions = partial(spack.pkgsort.version_compare, spec.name) valid_versions = sorted( [v for v in pkg.versions @@ -203,31 +219,114 @@ class DefaultConcretizer(object): return True # Things changed + def _concretize_operating_system(self, spec): + if spec.architecture.platform_os is not None: + if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): + return False + else: + spec.add_operating_system_from_string(spec.architecture.platform_os) + return True #changed + if spec.root.architecture and spec.root.architecture.platform_os: + if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): + spec.architecture.platform_os = spec.root.architecture.platform_os + else: + spec.add_operating_system_from_string(spec.root.architecture.platform_os) + else: + spec.architecture.platform_os = spec.architecture.platform.operating_system('default_os') + + return True #changed + +# """ Future method for concretizing operating system """ +# if isinstance(arch.platform_os, spack.architecture.OperatingSystem): +# return False +# else: +# arch.arch_os = platform.operating_system('default_os') +# return True - def concretize_architecture(self, spec): - """If the spec already had an architecture, return. Otherwise if - the root of the DAG has an architecture, then use that. - Otherwise take the system's default architecture. - - Intuition: Architectures won't be set a lot, and generally you - want the host system's architecture. When architectures are - mised in a spec, it is likely because the tool requries a - cross-compiled component, e.g. for tools that run on BlueGene - or Cray machines. These constraints will likely come directly - from packages, so require the user to be explicit if they want - to mess with the architecture, and revert to the default when - they're not explicit. - """ - if spec.architecture is not None: - return False - if spec.root.architecture: - spec.architecture = spec.root.architecture + def _concretize_target(self, spec): + if spec.architecture.target is not None: + if isinstance(spec.architecture.target,spack.architecture.Target): + return False + else: + spec.add_target_from_string(spec.architecture.target) + return True #changed + + if spec.root.architecture and spec.root.architecture.target: + if isinstance(spec.root.architecture.target,spack.architecture.Target): + spec.architecture.target = spec.root.architecture.target + else: + spec.add_target_from_string(spec.root.architecture.target) else: - spec.architecture = spack.architecture.sys_type() + spec.architecture.target = spec.architecture.platform.target('default') + + return True #changed - assert(spec.architecture is not None) - return True # changed +# if isinstance(arch.target, spack.architecture.Target): +# return False +# else: +# arch.target = platform.target('default') +# return True + + def concretize_architecture(self, spec): + """If the spec is empty provide the defaults of the platform. If the + architecture is not a basestring, then check if either the platform, + target or operating system are concretized. If any of the fields are + changed then return True. If everything is concretized (i.e the + architecture attribute is a namedtuple of classes) then return False. + If the target is a string type, then convert the string into a + concretized architecture. If it has no architecture and the root of the + DAG has an architecture, then use the root otherwise use the defaults + on the platform. + """ + if spec.architecture is None: + # Set the architecture to all defaults + spec.architecture = spack.architecture.Arch() + return True + #If there is a target and it is a tuple and has both filled return + #False +# if isinstance(spec.architecture, basestring): +# spec.split_architecture_string(spec.architecture) + + ret = any(( + self._concretize_operating_system(spec), + self._concretize_target(spec))) + + + # Does not look pretty at all!!! +# if spec.root.architecture and \ +# not isinstance(spec.root.architecture, basestring): +# bool_flag = any(( +# self._concretize_platform(spec.root.architecture, platform), +# self._concretize_operating_system(spec.root.architecture, +# platform), +# self._concretize_target(spec.root.target, platform))) +# spec.architecture =spec.root.architecture +# return bool_flag +# else: +# spec.add_architecture_from_string(spec.root.architecture) + + return ret + + # if there is no target specified used the defaults + + #if spec.target is not None: + # if isinstance(spec.target,spack.architecture.Target): + # return False + # else: + # spec.add_target_from_string(spec.target) + # return True #changed + + #if spec.root.target: + # if isinstance(spec.root.target,spack.architecture.Target): + # spec.target = spec.root.target + # else: + # spec.add_target_from_string(spec.root.target) + #else: + # platform = spack.architecture.sys_type() + # spec.target = platform.target('default') + + #return True #changed def concretize_variants(self, spec): @@ -254,6 +353,24 @@ class DefaultConcretizer(object): build with the compiler that will be used by libraries that link to this one, to maximize compatibility. """ + # Pass on concretizing the compiler if the target is not yet determined + if not spec.architecture.target: + #Although this usually means changed, this means awaiting other changes + return True + + # Only use a matching compiler if it is of the proper style + # Takes advantage of the proper logic already existing in compiler_for_spec + # Should think whether this can be more efficient + def _proper_compiler_style(cspec, architecture): + compilers = spack.compilers.compilers_for_spec(cspec) + filter(lambda c: c.strategy == architecture.platform_os.compiler_strategy, compilers) +#if architecture.platform_os.compiler_strategy == 'PATH': + # filter(lambda c: not c.modules, compilers) + #if architecture.platform_os.compiler_strategy == 'MODULES': + # filter(lambda c: c.modules, compilers) + return compilers + + all_compilers = spack.compilers.all_compilers() if (spec.compiler and @@ -281,7 +398,12 @@ class DefaultConcretizer(object): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - spec.compiler = matches[0].copy() + index = len(matches)-1 + while not _proper_compiler_style(matches[index], spec.architecture): + index -= 1 + if index == 0: + raise NoValidVersionError(spec) + spec.compiler = matches[index].copy() assert(spec.compiler.concrete) return True # things changed. diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3a785fe692..182c174baa 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -136,7 +136,7 @@ from spack.error import SpackError # Hacked yaml for configuration files preserves line numbers. import spack.util.spack_yaml as syaml - +from spack.build_environment import get_path_from_module """Dict from section names -> schema for that section.""" section_schemas = { @@ -551,9 +551,13 @@ def spec_externals(spec): for pkg,path in pkg_paths.iteritems(): if not spec.satisfies(pkg): continue + + module = allpkgs.get(pkg, {}).get('module', None) if not path: - continue - spec_locations.append( (spack.spec.Spec(pkg), path) ) + if not module: + continue + path = get_path_from_module(module) + spec_locations.append( (spack.spec.Spec(pkg), path, module) ) return spec_locations diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 61cd303012..f0691765ba 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -259,7 +259,7 @@ def variant(pkg, name, default=False, description=""): """Define a variant for the package. Packager can specify a default value (on or off) as well as a text description.""" - default = bool(default) + default = default description = str(description).strip() if not re.match(spack.spec.identifier_re, name): diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 39ee4e203d..96a49554ae 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -165,7 +165,7 @@ class DirectoryLayout(object): class YamlDirectoryLayout(DirectoryLayout): """Lays out installation directories like this:: <install root>/ - <architecture>/ + <target>/ <compiler>-<compiler version>/ <name>-<version>-<variants>-<hash> @@ -207,8 +207,7 @@ class YamlDirectoryLayout(DirectoryLayout): spec.version, spec.dag_hash(self.hash_len)) - path = join_path( - spec.architecture, + path = join_path(spec.architecture, "%s-%s" % (spec.compiler.name, spec.compiler.version), dir_name) diff --git a/lib/spack/spack/operating_systems/__init__.py b/lib/spack/spack/operating_systems/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/spack/spack/operating_systems/__init__.py diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py new file mode 100644 index 0000000000..9a0bf6c194 --- /dev/null +++ b/lib/spack/spack/operating_systems/cnl.py @@ -0,0 +1,13 @@ +from spack.architecture import OperatingSystem + +class Cnl(OperatingSystem): + """ Compute Node Linux (CNL) is the operating system used for the Cray XC + series super computers. It is a very stripped down version of GNU/Linux. + Any compilers found through this operating system will be used with + modules. If updated, user must make sure that version and name are + updated to indicate that OS has been upgraded (or downgraded) + """ + def __init__(self): + name = 'CNL' + version = '10' + super(Cnl, self).__init__(name, version, "MODULES") diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py new file mode 100644 index 0000000000..81c7a86430 --- /dev/null +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -0,0 +1,15 @@ +import platform as py_platform +from spack.architecture import OperatingSystem + +class LinuxDistro(OperatingSystem): + """ This class will represent the autodetected operating system + for a Linux System. Since there are many different flavors of + Linux, this class will attempt to encompass them all through + autodetection using the python module platform and the method + platform.dist() + """ + def __init__(self): + name = py_platform.dist()[0] + version = py_platform.dist()[1] + + super(LinuxDistro, self).__init__(name, version) diff --git a/lib/spack/spack/operating_systems/mac_osx.py b/lib/spack/spack/operating_systems/mac_osx.py new file mode 100644 index 0000000000..a9de03d2cc --- /dev/null +++ b/lib/spack/spack/operating_systems/mac_osx.py @@ -0,0 +1,29 @@ +import platform as py_platform +from spack.architecture import OperatingSystem + +class MacOsx(OperatingSystem): + """ This class represents the MAC_OSX operating system. This will be auto + detected using the python platform.mac_ver. The MAC_OSX platform + will be represented using the major version operating system name, i.e + el capitan, yosemite...etc. + """ + + def __init__(self): + """ Autodetects the mac version from a dictionary. Goes back as + far as 10.6 snowleopard. If the user has an older mac then + the version will just be a generic mac_os. + """ + mac_releases = {'10.6': "snowleopard", + "10.7": "lion", + "10.8": "mountainlion", + "10.9": "mavericks", + "10.10": "yosemite", + "10.11": "elcapitan"} + + mac_ver = py_platform.mac_ver()[0][:-2] + try: + name = mac_releases[mac_ver] + except KeyError: + name = "mac_os" + + super(MacOsx, self).__init__(name, mac_ver) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 696adaf896..1da9af753c 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -649,11 +649,13 @@ class Package(object): @property + #TODO: Change this to architecture def compiler(self): """Get the spack.compiler.Compiler object used to build this package.""" if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") - return spack.compilers.compiler_for_spec(self.spec.compiler) + return spack.compilers.compiler_for_spec(self.spec.compiler, + self.spec.architecture.platform_os) def url_version(self, version): diff --git a/lib/spack/spack/platforms/__init__.py b/lib/spack/spack/platforms/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/spack/spack/platforms/__init__.py diff --git a/lib/spack/spack/platforms/bgq.py b/lib/spack/spack/platforms/bgq.py new file mode 100644 index 0000000000..e0eb76f336 --- /dev/null +++ b/lib/spack/spack/platforms/bgq.py @@ -0,0 +1,18 @@ +import os +from spack.architecture import Platform, Target + +class Bgq(Platform): + priority = 30 + front_end = 'power7' + back_end = 'powerpc' + default = 'powerpc' + + def __init__(self): + super(Bgq, self).__init__('bgq') + self.add_target(self.front_end, Target(self.front_end)) + self.add_target(self.back_end, Target(self.back_end,)) + + @classmethod + def detect(self): + return os.path.exists('/bgsys') + diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py new file mode 100644 index 0000000000..4843a47c62 --- /dev/null +++ b/lib/spack/spack/platforms/cray_xc.py @@ -0,0 +1,46 @@ +import os +from spack.architecture import Platform, Target +from spack.operating_systems.linux_distro import LinuxDistro +from spack.operating_systems.cnl import Cnl + +class CrayXc(Platform): + priority = 20 + front_end = 'sandybridge' + back_end = 'ivybridge' + default = 'ivybridge' + + front_os = "SuSE" + back_os = "CNL" + default_os = "CNL" + + def __init__(self): + ''' Since cori doesn't have ivybridge as a front end it's better + if we use CRAY_CPU_TARGET as the default. This will ensure + that if we're on a XC-40 or XC-30 then we can detect the target + ''' + super(CrayXc, self).__init__('crayxc') + + # Handle the default here so we can check for a key error + if 'CRAY_CPU_TARGET' in os.environ: + self.default = os.environ['CRAY_CPU_TARGET'] + + # Change the defaults to haswell if we're on an XC40 + if self.default == 'haswell': + self.front_end = self.default + self.back_end = self.default + + # Could switch to use modules and fe targets for front end + # Currently using compilers by path for front end. + self.add_target('sandybridge', Target('sandybridge')) + self.add_target('ivybridge', + Target('ivybridge', 'craype-ivybridge')) + self.add_target('haswell', + Target('haswell','craype-haswell')) + + self.add_operating_system('SuSE', LinuxDistro()) + self.add_operating_system('CNL', Cnl()) + + @classmethod + def detect(self): + return os.path.exists('/opt/cray/craype') + diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py new file mode 100644 index 0000000000..4c3d38851f --- /dev/null +++ b/lib/spack/spack/platforms/darwin.py @@ -0,0 +1,22 @@ +import subprocess +from spack.architecture import Platform, Target +from spack.operating_systems.mac_osx import MacOsx + +class Darwin(Platform): + priority = 89 + front_end = 'x86_64' + back_end = 'x86_64' + default = 'x86_64' + + def __init__(self): + super(Darwin, self).__init__('darwin') + self.add_target(self.default, Target(self.default)) + mac_os = MacOsx() + self.default_os = mac_os.name + self.add_operating_system(mac_os.name, mac_os) + + @classmethod + def detect(self): + platform = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) + platform, _ = platform.communicate() + return 'darwin' in platform.strip().lower() diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py new file mode 100644 index 0000000000..3243a1dcdf --- /dev/null +++ b/lib/spack/spack/platforms/linux.py @@ -0,0 +1,22 @@ +import subprocess +from spack.architecture import Platform, Target +from spack.operating_systems.linux_distro import LinuxDistro + +class Linux(Platform): + priority = 90 + front_end = 'x86_64' + back_end = 'x86_64' + default = 'x86_64' + + def __init__(self): + super(Linux, self).__init__('linux') + self.add_target(self.default, Target(self.default)) + linux_dist = LinuxDistro() + self.default_os = linux_dist.name + self.add_operating_system(linux_dist.name, linux_dist) + + @classmethod + def detect(self): + platform = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) + platform, _ = platform.communicate() + return 'linux' in platform.strip().lower() diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index c045e80365..4d2c61c0c4 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -90,7 +90,9 @@ thing. Spack uses ~variant in directory names and in the canonical form of specs to avoid ambiguity. Both are provided because ~ can cause shell expansion when it is the first character in an id typed on the command line. """ +from collections import namedtuple import sys +import imp import itertools import hashlib import base64 @@ -100,15 +102,18 @@ import yaml from yaml.error import MarkedYAMLError import llnl.util.tty as tty +from llnl.util.filesystem import join_path from llnl.util.lang import * from llnl.util.tty.color import * import spack +import spack.architecture import spack.parse import spack.error import spack.compilers as compilers from spack.version import * +from spack.util.naming import mod_to_class from spack.util.string import * from spack.util.prefix import Prefix from spack.virtual import ProviderIndex @@ -119,7 +124,7 @@ identifier_re = r'\w[\w-]*' # Convenient names for color formats so that other things can use them compiler_color = '@g' version_color = '@c' -architecture_color = '@m' +architecture_color = '@m' enabled_variant_color = '@B' disabled_variant_color = '@r' dependency_color = '@.' @@ -421,6 +426,7 @@ class Spec(object): self._normal = kwargs.get('normal', False) self._concrete = kwargs.get('concrete', False) self.external = None + self.external_module = None # This allows users to construct a spec DAG with literals. # Note that given two specs a and b, Spec(a) copies a, but @@ -456,7 +462,13 @@ class Spec(object): """Called by the parser to set the architecture.""" if self.architecture: raise DuplicateArchitectureError( "Spec for '%s' cannot have two architectures." % self.name) - self.architecture = architecture + platform = spack.architecture.sys_type() + if '-' in architecture: + os, target = architecture.split('-') + else: + os = None + target = architecture + self.architecture = spack.architecture.Arch(os, target) def _add_dependency(self, spec): @@ -571,7 +583,7 @@ class Spec(object): in the traversal. root [=True] - If false, this won't yield the root node, just its descendents. + If False, this won't yield the root node, just its descendents. direction [=children|parents] If 'children', does a traversal of this spec's children. If @@ -664,7 +676,6 @@ class Spec(object): d = { 'variants' : dict( (name,v.enabled) for name, v in self.variants.items()), - 'arch' : self.architecture, 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } @@ -674,6 +685,13 @@ class Spec(object): if not self.concrete or self.namespace: d['namespace'] = self.namespace + if self.architecture: + # TODO: Fix the target.to_dict to account for the tuple + # Want it to be a dict of dicts + d['architecture'] = self.architecture.to_dict() + else: + d['architecture'] = None + if self.compiler: d.update(self.compiler.to_dict()) else: @@ -700,7 +718,8 @@ class Spec(object): spec = Spec(name) spec.namespace = node.get('namespace', None) spec.versions = VersionList.from_dict(node) - spec.architecture = node['arch'] + # TODO: Need to fix the architecture.Target.from_dict + spec.architecture = spack.architecture.arch_from_dict(node['architecture']) if node['compiler'] is None: spec.compiler = None @@ -766,7 +785,6 @@ class Spec(object): if self.name in presets: changed |= self.constrain(presets[self.name]) - else: # Concretize virtual dependencies last. Because they're added # to presets below, their constraints will all be merged, but we'll @@ -786,8 +804,9 @@ class Spec(object): """Replace this virtual spec with a concrete spec.""" assert(self.virtual) for name, dependent in self.dependents.items(): - del dependent.dependencies[self.name] - dependent._add_dependency(concrete) + if not dependent.external: + del dependent.dependencies[self.name] + dependent._add_dependency(concrete) def _expand_virtual_packages(self): @@ -1188,9 +1207,10 @@ class Spec(object): raise UnsatisfiableVariantSpecError(self.variants[v], other.variants[v]) + # TODO: Check out the logic here if self.architecture is not None and other.architecture is not None: if self.architecture != other.architecture: - raise UnsatisfiableArchitectureSpecError(self.architecture, + raise UnsatisfiableTargetSpecError(self.architecture, other.architecture) changed = False @@ -1277,11 +1297,34 @@ class Spec(object): except SpecError: return parse_anonymous_spec(spec_like, self.name) + def _is_valid_platform(self, platform, platform_list): + if platform in platform_list: + return True + return False + + def _is_valid_target(self, target, platform): + return target in platform.targets + + def _is_valid_os(self, os_string, platform): + return os_string in platform.operating_sys + + def add_target_from_string(self, target): + if target is None: + self.architecture.target = self.architecture.platform.target('default_target') + else: + self.architecture.target = self.architecture.platform.target(target) + + def add_operating_system_from_string(self, os): + if os is None: + self.architecture.platform_os = self.architecture.platform.operating_system('default_os') + else: + self.architecture.platform_os = self.architecture.platform.operating_system(os) + def satisfies(self, other, deps=True, strict=False): - """Determine if this spec satisfies all constraints of another. + """determine if this spec satisfies all constraints of another. - There are two senses for satisfies: + there are two senses for satisfies: * `loose` (default): the absence of a constraint in self implies that it *could* be satisfied by other, so we only @@ -1293,7 +1336,7 @@ class Spec(object): """ other = self._autospec(other) - # A concrete provider can satisfy a virtual dependency. + # a concrete provider can satisfy a virtual dependency. if not self.virtual and other.virtual: pkg = spack.repo.get(self.fullname) if pkg.provides(other.name): @@ -1303,7 +1346,7 @@ class Spec(object): return True return False - # Otherwise, first thing we care about is whether the name matches + # otherwise, first thing we care about is whether the name matches if self.name != other.name: return False @@ -1318,18 +1361,25 @@ class Spec(object): elif strict and (self.versions or other.versions): return False - # None indicates no constraints when not strict. + # none indicates no constraints when not strict. if self.compiler and other.compiler: if not self.compiler.satisfies(other.compiler, strict=strict): - return False + return False elif strict and (other.compiler and not self.compiler): return False if not self.variants.satisfies(other.variants, strict=strict): return False - # Architecture satisfaction is currently just string equality. + + # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. + if isinstance(self.architecture, basestring): + self.add_architecture_from_string(self.architecture) + if isinstance(other.architecture, basestring): + other.add_architecture_from_string(other.architecture) + + # TODO: Need to make sure that comparisons can be made via classes if self.architecture and other.architecture: if self.architecture != other.architecture: return False @@ -1399,17 +1449,19 @@ class Spec(object): Options: dependencies[=True] - Whether deps should be copied too. Set to false to copy a + Whether deps should be copied too. Set to False to copy a spec but not its dependencies. """ + # TODO: Check if comparisons for tuple are valid # We don't count dependencies as changes here changed = True if hasattr(self, 'name'): changed = (self.name != other.name and self.versions != other.versions and \ self.architecture != other.architecture and self.compiler != other.compiler and \ self.variants != other.variants and self._normal != other._normal and \ - self.concrete != other.concrete and self.external != other.external) + self.concrete != other.concrete and self.external != other.external and \ + self.external_module != other.external_module) # Local node attributes get copied first. self.name = other.name @@ -1423,6 +1475,7 @@ class Spec(object): self.variants.spec = self self.external = other.external self.namespace = other.namespace + self.external_module = other.external_module # If we copy dependencies, preserve DAG structure in the new spec if kwargs.get('deps', True): @@ -1441,6 +1494,7 @@ class Spec(object): self._normal = other._normal self._concrete = other._concrete self.external = other.external + self.external_module = other.external_module return changed @@ -1598,7 +1652,7 @@ class Spec(object): ${COMPILERNAME} Compiler name ${COMPILERVER} Compiler version ${OPTIONS} Options - ${ARCHITECTURE} Architecture + ${TARGET} Target ${SHA1} Dependencies 8-char sha1 prefix ${SPACK_ROOT} The spack root directory @@ -1611,7 +1665,7 @@ class Spec(object): Anything else is copied verbatim into the output stream. *Example:* ``$_$@$+`` translates to the name, version, and options - of the package, but no dependencies, arch, or compiler. + of the package, but no dependencies, architecture, or compiler. TODO: allow, e.g., $6# to customize short hash length TODO: allow, e.g., $## for full hash. @@ -1656,6 +1710,7 @@ class Spec(object): elif c == '+': if self.variants: write(fmt % str(self.variants), c) + # TODO: Check string methods here elif c == '=': if self.architecture: write(fmt % (c + str(self.architecture)), c) @@ -1707,7 +1762,7 @@ class Spec(object): write(fmt % str(self.variants), '+') elif named_str == 'ARCHITECTURE': if self.architecture: - write(fmt % str(self.architecture), '=') + write(fmt % self.architecture, '=') elif named_str == 'SHA1': if self.dependencies: out.write(fmt % str(self.dag_hash(7))) @@ -1734,6 +1789,40 @@ class Spec(object): return ''.join("^" + dep.format() for dep in self.sorted_deps()) + def __cmp__(self, other): + #Package name sort order is not configurable, always goes alphabetical + if self.name != other.name: + return cmp(self.name, other.name) + + #Package version is second in compare order + pkgname = self.name + if self.versions != other.versions: + return spack.pkgsort.version_compare(pkgname, + self.versions, other.versions) + + #Compiler is third + if self.compiler != other.compiler: + return spack.pkgsort.compiler_compare(pkgname, + self.compiler, other.compiler) + + #Variants + if self.variants != other.variants: + return spack.pkgsort.variant_compare(pkgname, + self.variants, other.variants) + + #Target + if self.target != other.target: + return spack.pkgsort.target_compare(pkgname, + self.target, other.target) + + #Dependency is not configurable + if self.dep_hash() != other.dep_hash(): + return -1 if self.dep_hash() < other.dep_hash() else 1 + + #Equal specs + return 0 + + def __str__(self): return self.format() + self.dep_string() @@ -1806,7 +1895,6 @@ class SpecParser(spack.parse.Parser): def do_parse(self): specs = [] - try: while self.next: if self.accept(ID): @@ -1849,6 +1937,7 @@ class SpecParser(spack.parse.Parser): spec.architecture = None spec.compiler = None spec.external = None + spec.external_module = None spec.dependents = DependencyMap() spec.dependencies = DependencyMap() spec.namespace = spec_namespace @@ -2036,9 +2125,16 @@ class UnknownVariantError(SpecError): super(UnknownVariantError, self).__init__( "Package %s has no variant %s!" % (pkg, variant)) +class UnknownArchitectureSpecError(SpecError): + """ Raised when an entry in a string field is neither a platform, + operating system or a target. """ + def __init__(self, architecture_spec_entry): + super(UnknownArchitectureSpecError, self).__init__( + "Architecture spec %s is not a valid spec entry" % ( + architecture_spec_entry)) class DuplicateArchitectureError(SpecError): - """Raised when the same architecture occurs in a spec twice.""" + """Raised when the same target occurs in a spec twice.""" def __init__(self, message): super(DuplicateArchitectureError, self).__init__(message) @@ -2119,11 +2215,11 @@ class UnsatisfiableVariantSpecError(UnsatisfiableSpecError): provided, required, "variant") -class UnsatisfiableArchitectureSpecError(UnsatisfiableSpecError): - """Raised when a spec architecture conflicts with package constraints.""" +class UnsatisfiableTargetSpecError(UnsatisfiableSpecError): + """Raised when a spec target conflicts with package constraints.""" def __init__(self, provided, required): - super(UnsatisfiableArchitectureSpecError, self).__init__( - provided, required, "architecture") + super(UnsatisfiableTargetSpecError, self).__init__( + provided, required, "target") class UnsatisfiableProviderSpecError(UnsatisfiableSpecError): diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index d5d8b64765..e851b123ed 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -34,7 +34,8 @@ from llnl.util.tty.colify import colify import spack """Names of tests to be included in Spack's test suite""" -test_names = ['versions', +test_names = ['architecture', + 'versions', 'url_parse', 'url_substitution', 'packages', diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py new file mode 100644 index 0000000000..75a67bf02f --- /dev/null +++ b/lib/spack/spack/test/architecture.py @@ -0,0 +1,63 @@ +""" Test checks if the architecture class is created correctly and also that + the functions are looking for the correct architecture name +""" +import unittest +import os +import platform +import spack +from spack.architecture import * +import spack.spec +from spack.platforms.cray_xc import CrayXc +from spack.platforms.linux import Linux +from spack.platforms.bgq import Bgq +from spack.platforms.darwin import Darwin + +class ArchitectureTest(unittest.TestCase): + + def test_to_dict_function_with_architecture(self): + arch = Arch() + arch.platform_os = arch.platform.operating_system('default_os') + arch.target = arch.platform.target('default') + + d = arch.to_dict() + self.assertEqual(d, {'platform' : 'crayxc', + 'platform_os' : {'name': 'CNL', + 'compiler_strategy' : 'MODULES', + 'version':'10'}, + 'target' : {'name': 'haswell', + 'module_name': 'craype-haswell'}}) + + def test_from_dict_function_with_architecture(self): + d = {'platform':'crayxc', + 'platform_os' : {'name' : 'CNL', 'compiler_strategy': 'MODULES', + 'version': '10'}, + 'target' : {'name':'haswell', 'module_name': 'craype-haswell'}} + + arch = spack.architecture.arch_from_dict(d) + self.assertTrue( isinstance(arch, Arch) ) + self.assertTrue( isinstance(arch.platform, Platform) ) + self.assertTrue( isinstance(arch.platform_os, OperatingSystem) ) + self.assertTrue( isinstance(arch.target, Target) ) + + + def test_platform_class_and_compiler_strategies(self): + a = CrayXc() + t = a.operating_system('default_os') + self.assertEquals(t.compiler_strategy, 'MODULES') + b = Linux() + s = b.operating_system('default_os') + self.assertEquals(s.compiler_strategy, 'PATH') + + def test_sys_type(self): + output_platform_class = sys_type() + my_arch_class = None + if os.path.exists('/opt/cray/craype'): + my_platform_class = CrayXc() + elif os.path.exists('/bgsys'): + my_platform_class = Bgq() + elif 'Linux' in platform.system(): + my_platform_class = Linux() + elif 'Darwin' in platform.system(): + my_platform_class = Darwin() + + self.assertEqual(str(output_platform_class), str(my_platform_class)) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 07828d8ea6..3fb8417c3e 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -45,8 +45,8 @@ class ConcretizeTest(MockPackagesTest): if abstract.compiler and abstract.compiler.concrete: self.assertEqual(abstract.compiler, concrete.compiler) - if abstract.architecture and abstract.architecture.concrete: - self.assertEqual(abstract.architecture, concrete.architecture) + if abstract.architecture and abstract.architecture.target.concrete: + self.assertEqual(abstract.target, concrete.target) def check_concretize(self, abstract_spec): diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 2d4b8cd584..020d25015d 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -88,21 +88,18 @@ class MultiMethodTest(MockPackagesTest): self.assertEqual(pkg.has_a_default(), 'default') - def test_architecture_match(self): - pkg = spack.repo.get('multimethod=x86_64') - self.assertEqual(pkg.different_by_architecture(), 'x86_64') - - pkg = spack.repo.get('multimethod=ppc64') - self.assertEqual(pkg.different_by_architecture(), 'ppc64') - - pkg = spack.repo.get('multimethod=ppc32') - self.assertEqual(pkg.different_by_architecture(), 'ppc32') - - pkg = spack.repo.get('multimethod=arm64') - self.assertEqual(pkg.different_by_architecture(), 'arm64') - - pkg = spack.repo.get('multimethod=macos') - self.assertRaises(NoSuchMethodError, pkg.different_by_architecture) + def test_target_match(self): + platform = spack.architecture.sys_type() + targets = platform.targets.values() + for target in targets[:-1]: + pkg = spack.db.get('multimethod='+target.name) + self.assertEqual(pkg.different_by_target(), target.name) + + pkg = spack.db.get('multimethod='+targets[-1].name) + if len(targets) == 1: + self.assertEqual(pkg.different_by_target(), targets[-1].name) + else: + self.assertRaises(NoSuchMethodError, pkg.different_by_target) def test_dependency_match(self): diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py new file mode 100644 index 0000000000..205408db3f --- /dev/null +++ b/lib/spack/spack/test/operating_system.py @@ -0,0 +1,58 @@ +""" Test checks if the operating_system class is created correctly and that +the functions are using the correct operating_system. Also checks whether +the operating_system correctly uses the compiler_strategy +""" + +import unittest +import os +import platform +from spack.platforms.cray_xc import CrayXc +from spack.platforms.linux import Linux +from spack.platforms.darwin import Darwin +from spack.operating_system.linux_distro import LinuxDistro +from spack.operating_system.mac_osx import MacOSX +from spack.operating_system.cnl import ComputeNodeLinux + +class TestOperatingSystem(unittest.TestCase): + + def setUp(self): + cray_xc = CrayXc() + linux = Linux() + darwin = Darwin() + self.cray_operating_sys = cray_xc.operating_system('front_os') + self.cray_default_os = cray_xc.operating_system('default_os') + self.cray_back_os = cray_xc.operating_system('back_os') + self.darwin_operating_sys = darwin.operating_system('default_os') + self.linux_operating_sys = linux.operating_system('default_os') + + def test_cray_front_end_operating_system(self): + self.assertIsInstance(self.cray_operating_sys, LinuxDistro) + + def test_cray_front_end_compiler_strategy(self): + self.assertEquals(self.cray_operating_sys.compiler_strategy, "PATH") + + def test_cray_back_end_operating_system(self): + self.assertIsInstance(self.cray_back_os,ComputeNodeLinux) + + def test_cray_back_end_compiler_strategy(self): + self.assertEquals(self.cray_back_os.compiler_strategy, "MODULES") + + def test_linux_operating_system(self): + self.assertIsInstance(self.linux_operating_sys, LinuxDistro) + + def test_linux_compiler_strategy(self): + self.assertEquals(self.linux_operating_sys.compiler_strategy, "PATH") + + + def test_cray_front_end_compiler_list(self): + """ Operating systems will now be in charge of finding compilers. + So, depending on which operating system you want to build for + or which operating system you are on, then you could detect + compilers in a certain way. Cray linux environment on the front + end is just a regular linux distro whereas the Cray linux compute + node is a stripped down version which modules are important + """ + self.assertEquals(True, False) + + + diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 5e6162b6e6..eee65c5c09 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -238,10 +238,14 @@ class SpecDagTest(MockPackagesTest): self.assertRaises(spack.spec.UnsatisfiableCompilerSpecError, spec.normalize) - def test_unsatisfiable_architecture(self): - self.set_pkg_dep('mpileaks', 'mpich=bgqos_0') - spec = Spec('mpileaks ^mpich=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') - self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) + def test_unsatisfiable_target(self): + platform = spack.architecture.sys_type() + if len(platform.targets) > 1: + first = platform.targets.values()[0].name + second = platform.targets.values()[1].name + set_pkg_dep('mpileaks', 'mpich='+first) + spec = Spec('mpileaks ^mpich='+ second +' ^callpath ^dyninst ^libelf ^libdwarf') + self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) def test_invalid_dep(self): diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 8c33d1ff6e..0b310488a9 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -137,13 +137,14 @@ class SpecSematicsTest(MockPackagesTest): self.check_unsatisfiable('foo %gcc@4.7', '%gcc@4.7.3') - def test_satisfies_architecture(self): - self.check_satisfies('foo=chaos_5_x86_64_ib', '=chaos_5_x86_64_ib') - self.check_satisfies('foo=bgqos_0', '=bgqos_0') - - self.check_unsatisfiable('foo=bgqos_0', '=chaos_5_x86_64_ib') - self.check_unsatisfiable('foo=chaos_5_x86_64_ib', '=bgqos_0') + def test_satisfies_target(self): + platform = spack.architecture.sys_type() + targets = platform.targets.values() + for target in targets: + self.check_satisfies('foo='+target.name, '='+target.name) + for i in range(1,len(targets)): + self.check_unsatisfiable('foo='+targets[i-1].name, '='+targets[i].name) def test_satisfies_dependencies(self): self.check_satisfies('mpileaks^mpich', '^mpich') @@ -305,14 +306,16 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain('libelf+debug~foo', 'libelf+debug', 'libelf+debug~foo') - def test_constrain_arch(self): - self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0') - self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0') + def test_constrain_target(self): + platform = spack.architecture.sys_type() + target = platform.target('default').name + self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) + self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) def test_constrain_compiler(self): - self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0') - self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0') + self.check_constrain('libelf%intel', 'libelf%intel', 'libelf%intel') + self.check_constrain('libelf%intel', 'libelf', 'libelf%intel') def test_invalid_constraint(self): @@ -322,7 +325,10 @@ class SpecSematicsTest(MockPackagesTest): self.check_invalid_constraint('libelf+debug', 'libelf~debug') self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') - self.check_invalid_constraint('libelf=bgqos_0', 'libelf=x86_54') + platform = spack.architecture.sys_type() + targets = platform.targets.values() + if len(targets) > 1: + self.check_invalid_constraint('libelf='+targets[0].name, 'libelf='+targets[1].name) def test_constrain_changed(self): @@ -332,7 +338,8 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_changed('libelf%gcc', '%gcc@4.5') self.check_constrain_changed('libelf', '+debug') self.check_constrain_changed('libelf', '~debug') - self.check_constrain_changed('libelf', '=bgqos_0') + platform = spack.architecture.sys_type() + self.check_constrain_changed('libelf', '='+platform.target('default').name) def test_constrain_not_changed(self): @@ -343,7 +350,9 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_not_changed('libelf%gcc@4.5', '%gcc@4.5') self.check_constrain_not_changed('libelf+debug', '+debug') self.check_constrain_not_changed('libelf~debug', '~debug') - self.check_constrain_not_changed('libelf=bgqos_0', '=bgqos_0') + platform = spack.architecture.sys_type() + default = platform.target('default').name + self.check_constrain_not_changed('libelf='+default, '='+default) self.check_constrain_not_changed('libelf^foo', 'libelf^foo') self.check_constrain_not_changed('libelf^foo^bar', 'libelf^foo^bar') @@ -355,7 +364,9 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_changed('libelf^foo%gcc', 'libelf^foo%gcc@4.5') self.check_constrain_changed('libelf^foo', 'libelf^foo+debug') self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') - self.check_constrain_changed('libelf^foo', 'libelf^foo=bgqos_0') + platform = spack.architecture.sys_type() + default = platform.target('default').name + self.check_constrain_changed('libelf^foo', 'libelf^foo='+default) def test_constrain_dependency_not_changed(self): @@ -365,4 +376,7 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_not_changed('libelf^foo%gcc@4.5', 'libelf^foo%gcc@4.5') self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') - self.check_constrain_not_changed('libelf^foo=bgqos_0', 'libelf^foo=bgqos_0') + platform = spack.architecture.sys_type() + default = platform.target('default').name + self.check_constrain_not_changed('libelf^foo='+default, 'libelf^foo='+default) + diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index fc27b789d0..c349d071d2 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -157,6 +157,7 @@ class Executable(object): raise ProcessError("Command exited with status %d:" % proc.returncode, cmd_line) + if output is str or error is str: result = '' if output is str: result += out diff --git a/var/spack/mock_configs/site_spackconfig/compilers.yaml b/var/spack/mock_configs/site_spackconfig/compilers.yaml new file mode 100644 index 0000000000..5f8b38007b --- /dev/null +++ b/var/spack/mock_configs/site_spackconfig/compilers.yaml @@ -0,0 +1,40 @@ +compilers: + all: + clang@3.3: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: None + strategy: PATH + gcc@4.5.0: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + modules: None + strategy: PATH + gcc@5.2.0: + cc: cc + cxx: CC + f77: ftn + fc: ftn + modules: + - PrgEnv-gnu + - gcc/5.2.0 + strategy: MODULES + intel@15.0.1: + cc: cc + ccx: CC + f77: ftn + fc: ftn + modules: + - PrgEnv-intel + - intel/15.0.1 + strategy: MODULES + intel@15.1.2: + cc: /path/to/icc + cxx: /path/to/ic++ + f77: /path/to/ifort + fc: /path/to/ifort + strategy: PATH
\ No newline at end of file diff --git a/var/spack/packages/adios/package.py b/var/spack/packages/adios/package.py new file mode 100644 index 0000000000..260dcbe851 --- /dev/null +++ b/var/spack/packages/adios/package.py @@ -0,0 +1,38 @@ +import os + +from spack import * +class Adios(Package): + """The Adaptable IO System (ADIOS) provides a simple, + flexible way for scientists to describe the + data in their code that may need to be written, + read, or processed outside of the running simulation + """ + + homepage = "http://www.olcf.ornl.gov/center-projects/adios/" + url = "http://users.nccs.gov/~pnorbert/adios-1.9.0.tar.gz" + + version('1.9.0', 'dbf5cb10e32add2f04c9b4052b7ffa76') + + # Lots of setting up here for this package + # module swap PrgEnv-intel PrgEnv-$COMP + # module load cray-netcdf/4.3.3.1 + # module load cray-hdf5/1.8.14 + # module load python/2.7.10 + depends_on('hdf5') + depends_on('mxml') + + def install(self, spec, prefix): + configure_args = ["--prefix=%s" % prefix, + "--with-mxml=%s" % spec['mxml'].prefix, + "--with-hdf5=%s" % spec['hdf5'].prefix, + "--with-netcdf=%s" % os.environ["NETCDF_DIR"], + "--with-infiniband=no", + "MPICC=cc","MPICXX=CC","MPIFC=ftn", + "CPPFLAGS=-DMPICH_IGNORE_CXX_SEEK"] + + if spec.satisfies('%gcc'): + configure_args.extend(["CC=gcc", "CXX=g++", "FC=gfortran"]) + + configure(*configure_args) + make() + make("install") diff --git a/var/spack/packages/mxml/package.py b/var/spack/packages/mxml/package.py new file mode 100644 index 0000000000..f79251d312 --- /dev/null +++ b/var/spack/packages/mxml/package.py @@ -0,0 +1,26 @@ +import os +from spack import * + +class Mxml(Package): + """Mini-XML is a small XML library that you can use to read and write XML + and XML-like data files in your application without requiring large + non-standard libraries + """ + + homepage = "http://www.msweet.org" + url = "http://www.msweet.org/files/project3/mxml-2.9.tar.gz" + + version('2.9', 'e21cad0f7aacd18f942aa0568a8dee19') + version('2.8', 'd85ee6d30de053581242c4a86e79a5d2') + version('2.7', '76f2ae49bf0f5745d5cb5d9507774dc9') + version('2.6', '68977789ae64985dddbd1a1a1652642e') + version('2.5', 'f706377fba630b39fa02fd63642b17e5') + + # module swap PrgEnv-intel PrgEnv-$COMP (Can use whatever compiler you want to use) + # Case statement to change CC and CXX flags + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, "--disable-shared", 'CFLAGS=-static') + make() + make("install") + diff --git a/var/spack/repos/builtin.mock/packages/multimethod/package.py b/var/spack/repos/builtin.mock/packages/multimethod/package.py index ea103fe175..ea98f8d429 100644 --- a/var/spack/repos/builtin.mock/packages/multimethod/package.py +++ b/var/spack/repos/builtin.mock/packages/multimethod/package.py @@ -22,8 +22,11 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import imp +from llnl.util.filesystem import join_path +from spack.util.naming import mod_to_class from spack import * - +import spack.architecture class Multimethod(Package): """This package is designed for use with Spack's multimethod test. @@ -101,25 +104,26 @@ class Multimethod(Package): # - # Make sure we can switch methods on different architectures + # Make sure we can switch methods on different target # - @when('=x86_64') - def different_by_architecture(self): - return 'x86_64' - - @when('=ppc64') - def different_by_architecture(self): - return 'ppc64' - - @when('=ppc32') - def different_by_architecture(self): - return 'ppc32' - - @when('=arm64') - def different_by_architecture(self): - return 'arm64' - - +# for platform_name in ['cray_xc', 'darwin', 'linux']: +# file_path = join_path(spack.platform_path, platform_name) +# platform_mod = imp.load_source('spack.platforms', file_path + '.py') +# cls = getattr(platform_mod, mod_to_class(platform_name)) + +# platform = cls() + platform = spack.architecture.sys_type() + targets = platform.targets.values() + if len(targets) > 1: + targets = targets[:-1] + + for target in targets: + @when('='+target.name) + def different_by_target(self): + if isinstance(self.spec.architecture.target,basestring): + return self.spec.architecture.target + else: + return self.spec.architecture.target.name # # Make sure we can switch methods on different dependencies # diff --git a/var/spack/repos/builtin/packages/py-h5py/package.py b/var/spack/repos/builtin/packages/py-h5py/package.py index 6293da5407..04072ca76b 100644 --- a/var/spack/repos/builtin/packages/py-h5py/package.py +++ b/var/spack/repos/builtin/packages/py-h5py/package.py @@ -13,6 +13,8 @@ class PyH5py(Package): depends_on('hdf5') depends_on('py-numpy') depends_on('py-cython') + depends_on('py-six') + depends_on('py-pkgconfig') def install(self, spec, prefix): python('setup.py', 'configure', '--hdf5=%s' % spec['hdf5'].prefix) |