From e9f7d033ff04138905c99cb427e2071cc55cdc16 Mon Sep 17 00:00:00 2001 From: karenyyng Date: Sat, 3 Oct 2015 19:13:08 -0700 Subject: make sure submodule is added in the correct location --- lib/spack/spack/util/python_recipe_parser | 1 + 1 file changed, 1 insertion(+) create mode 160000 lib/spack/spack/util/python_recipe_parser (limited to 'lib') diff --git a/lib/spack/spack/util/python_recipe_parser b/lib/spack/spack/util/python_recipe_parser new file mode 160000 index 0000000000..6777dd98e7 --- /dev/null +++ b/lib/spack/spack/util/python_recipe_parser @@ -0,0 +1 @@ +Subproject commit 6777dd98e75229856d94da6671676b2a66e75986 -- cgit v1.2.3-60-g2f50 From d00314c621bd427b142fecafc11c25006f3b5279 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 5 Oct 2015 01:30:25 -0700 Subject: Protptype cray compiler detection and support --- lib/spack/env/cc | 17 +++++++++ lib/spack/spack/build_environment.py | 9 +++++ lib/spack/spack/compiler.py | 67 ++++++++++++++++++++++++++++++++++- lib/spack/spack/compilers/__init__.py | 9 +++++ lib/spack/spack/compilers/clang.py | 2 +- lib/spack/spack/compilers/gcc.py | 7 ++-- lib/spack/spack/compilers/intel.py | 3 ++ lib/spack/spack/compilers/pgi.py | 3 ++ lib/spack/spack/compilers/xl.py | 4 ++- 9 files changed, 116 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/env/cc b/lib/spack/env/cc index fa85bb595e..b6c6e03e42 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -70,6 +70,23 @@ for param in $parameters; do fi done + +# +# Cray module environment-related stuff. +# +if [ ! -z "$SPACK_CRAYPE" ]; then + cur_pe=$(module list 2>&1 | grep PrgEnv | grep -o 'PrgEnv-[^/]*') + if [ ! -z "$cur_pe" ]; then + module swap $cur_pe $SPACK_CRAYPE + else + module load $SPACK_CRAYPE + fi +fi + +if [ ! -z "$SPACK_COMP_MODULE" ]; then + module load $SPACK_COMP_MODULE +fi + # # Figure out the type of compiler, the language, and the mode so that # the compiler script knows what to do. diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index a133faa629..68388958f5 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -57,6 +57,9 @@ SPACK_DEBUG = 'SPACK_DEBUG' SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC' SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR' +SPACK_CRAYPE = 'SPACK_CRAYPE' +SPACK_COMP_MODULE = 'SPACK_COMP_MODULE' + class MakeExecutable(Executable): """Special callable executable object for make so the user can @@ -105,6 +108,12 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) + if compiler.PrgEnv: + os.environ['SPACK_CRAYPE'] = compiler.PrgEnv + os.environ['SPACK_COMP_MODULE'] = compiler.module + + + def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 1e800a8979..e7d450ee8b 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -25,6 +25,7 @@ import os import re import itertools +import subprocess from datetime import datetime import llnl.util.tty as tty @@ -98,7 +99,14 @@ class Compiler(object): cxx11_flag = "-std=c++11" - def __init__(self, cspec, cc, cxx, f77, fc): + # Cray PrgEnv name that can be used to load this compiler + PrgEnv = None + + # Name of module used to switch versions of this compiler + PrgEnv_compiler = None + + + def __init__(self, cspec, cc, cxx, f77, fc, module=None): def check(exe): if exe is None: return None @@ -111,6 +119,8 @@ class Compiler(object): self.fc = check(fc) self.spec = cspec + self.module = module + @property @@ -255,6 +265,61 @@ class Compiler(object): return list(compilers.values()) + @classmethod + def find_in_modules(cls): + compilers = [] + + if cls.PrgEnv: + if not cls.PrgEnv_compiler: + tty.die('Must supply PrgEnv_compiler with PrgEnv') + + output = _shell('module avail %s' % cls.PrgEnv_compiler) + matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) + + for name, version in matches: + v = version + '-craype' + comp = cls(spack.spec.CompilerSpec(name + '@' + v), + 'cc', 'CC', 'ftn', 'ftn', name +'/' + v) + + compilers.append(comp) + + return compilers + + +def _cur_prgenv(): + out, err = subprocess.Popen( + ['module list'], shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + matches = re.findall(r'(PrgEnv-[^/]*)/', err) + return matches[0] + + +def _module_shell(module, *args): + cmd = 'module swap %s %s;' % (_cur_prgenv(), module) + cmd += 'module load %s;' % compiler + cmd += 'module unload cray-libsci;' + +# + +# 'module load craype-network-gemini;' + +# 'module load %s;' % module + +# 'module swap gcc/4.6.1;' + +# 'module load eswrap; ' + +# 'module load craype-mc12; ' + +# 'module load cray-shmem; ' + +# 'module load cray-mpich; ') + cmd += ' '.join(args) + out, err = subprocess.Popen([cmd + ' '.join(args)], shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + return out + + +def _shell(*args): + return subprocess.Popen([' '.join(args)], shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1] + + + + def __repr__(self): """Return a string represntation of the compiler toolchain.""" return self.__str__() diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index b7b021a1ac..8c4cb38926 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -44,6 +44,7 @@ from spack.util.environment import get_path _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] +_optional_instance_vars = ['module'] _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] @@ -127,11 +128,18 @@ def add_compilers_to_config(scope, *compilers): compiler_config_tree = {} for compiler in compilers: compiler_entry = {} + for c in _required_instance_vars: val = getattr(compiler, c) if not val: val = "None" compiler_entry[c] = val + + for c in _optional_instance_vars: + val = getattr(compiler, c) + if val: + compiler_entry[c] = val + compiler_config_tree[str(compiler.spec)] = compiler_entry spack.config.add_to_compiler_config(compiler_config_tree, scope) @@ -220,6 +228,7 @@ def class_for_compiler_name(compiler_name): def all_compiler_types(): +# return [class_for_compiler_name(c) for c in ['gcc']] return [class_for_compiler_name(c) for c in supported_compilers()] diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 790901c86e..8d32608ff4 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -39,7 +39,7 @@ class Clang(Compiler): @classmethod - def default_version(self, comp): + def default_version(cls, comp): """The '--version' option works for clang compilers. Output looks like this:: diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index f0d27d590e..ff0b8889a8 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -42,6 +42,9 @@ class Gcc(Compiler): # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. suffixes = [r'-mp-\d\.\d'] + PrgEnv = 'gnu' + PrgEnv_compiler = 'gcc' + @property def cxx11_flag(self): if self.version < ver('4.3'): @@ -56,9 +59,9 @@ class Gcc(Compiler): return get_compiler_version( fc, '-dumpversion', # older gfortran versions don't have simple dumpversion output. - r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)') + r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)', module) @classmethod def f77_version(cls, f77): - return cls.fc_version(f77) + return cls.fc_version(f77, module) diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 2a72c4eaea..7c485fe69d 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -37,6 +37,9 @@ class Intel(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['ifort'] + PrgEnv = 'intel' + PrgEnv_compiler = 'intel' + @property def cxx11_flag(self): if self.version < ver('11.1'): diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index d97f24c12e..8f1ed28825 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -37,6 +37,9 @@ class Pgi(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['pgf95', 'pgf90'] + PrgEnv = 'pgi' + PrgEnv_compiler = 'pgi' + @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index 562186b865..179b720918 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -45,8 +45,9 @@ class Xl(Compiler): else: return "-qlanglvl=extended0x" + @classmethod - def default_version(self, comp): + def default_version(cls, comp): """The '-qversion' is the standard option fo XL compilers. Output looks like this:: @@ -72,6 +73,7 @@ class Xl(Compiler): return get_compiler_version( comp, '-qversion',r'([0-9]?[0-9]\.[0-9])') + @classmethod def fc_version(cls, fc): """The fortran and C/C++ versions of the XL compiler are always two units apart. -- cgit v1.2.3-60-g2f50 From f4e72f33c8ba988507aba667e318d1861c7b2b20 Mon Sep 17 00:00:00 2001 From: karenyyng Date: Mon, 5 Oct 2015 08:24:33 -0700 Subject: added dependencies for py-h5py --- lib/spack/spack/util/python_recipe_parser | 2 +- var/spack/packages/py-h5py/package.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/util/python_recipe_parser b/lib/spack/spack/util/python_recipe_parser index 6777dd98e7..437a62abb3 160000 --- a/lib/spack/spack/util/python_recipe_parser +++ b/lib/spack/spack/util/python_recipe_parser @@ -1 +1 @@ -Subproject commit 6777dd98e75229856d94da6671676b2a66e75986 +Subproject commit 437a62abb3df7212e3ee20269c0089a0a9766fe0 diff --git a/var/spack/packages/py-h5py/package.py b/var/spack/packages/py-h5py/package.py index 6293da5407..04072ca76b 100755 --- a/var/spack/packages/py-h5py/package.py +++ b/var/spack/packages/py-h5py/package.py @@ -13,6 +13,8 @@ class PyH5py(Package): depends_on('hdf5') depends_on('py-numpy') depends_on('py-cython') + depends_on('py-six') + depends_on('py-pkgconfig') def install(self, spec, prefix): python('setup.py', 'configure', '--hdf5=%s' % spec['hdf5'].prefix) -- cgit v1.2.3-60-g2f50 From b6d2a12ceb090692aa2be363b46d7f25486c0245 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 7 Oct 2015 15:57:29 -0700 Subject: Started changing the find in modules method written by Todd --- lib/spack/spack/compiler.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index e7d450ee8b..cbe6d8cd7d 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -272,9 +272,11 @@ class Compiler(object): if cls.PrgEnv: if not cls.PrgEnv_compiler: tty.die('Must supply PrgEnv_compiler with PrgEnv') - - output = _shell('module avail %s' % cls.PrgEnv_compiler) - matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) + + loaded_modules = os.environ["LOADEDMODULES"].split(":") + #output = _shell('module avail %s' % cls.PrgEnv_compiler) + for module in loaded_modules: + match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) for name, version in matches: v = version + '-craype' -- cgit v1.2.3-60-g2f50 From 4f21344e87da327b4166fff1fd2ce32afaa07dbc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 14 Oct 2015 19:41:07 -0700 Subject: Started created the Architecture class for Spack to use --- lib/spack/spack/architecture.py | 73 ++++++++++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 0c4b605e91..829aaa4c1c 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -30,7 +30,7 @@ from llnl.util.lang import memoized import spack import spack.error as serr from spack.version import Version - +from external import yaml class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): @@ -43,41 +43,81 @@ class NoSysTypeError(serr.SpackError): super(NoSysTypeError, self).__init__( "Could not determine sys_type for this machine.") +class Architecture(object): + def __init__(self, *arch_name): + + """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) and a strategy for + searching for that architecture's compiler. The target passed to it should be a dictionary of names and strategies. + """ + self.arch_dict = {} + self.arch_name = arch_name + + def add_arch_strategy(self): + """ Create a dictionary using the tuples of arch_names""" + for n in self.arch_name: + if 'cray' in n.lower(): + self.arch_dict[n] = "MODULES" + if 'linux' in n.lower() or 'x86_64' in n.lower(): + self.arch_dict[n] = "PATH" + else: + self.arch_dict[n] = None def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" + """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" if not hasattr(spack, "sys_type"): - return None + return None elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() + return Architecture(spack.sys_type()) else: - return spack.sys_type - - -def get_sys_type_from_environment(): - """Return $SYS_TYPE or None if it's not defined.""" - return os.environ.get('SYS_TYPE') + return Architecture(spack.sys_type) +# This is livermore dependent. Hard coded for livermore +#def get_sys_type_from_environment(): +# """Return $SYS_TYPE or None if it's not defined.""" +# return os.environ.get('SYS_TYPE') def get_mac_sys_type(): - """Return a Mac OS SYS_TYPE or None if this isn't a mac.""" + """Return a Mac OS SYS_TYPE or None if this isn't a mac. + Front-end config + """ + mac_ver = py_platform.mac_ver()[0] if not mac_ver: return None - return "macosx_%s_%s" % ( - Version(mac_ver).up_to(2), py_platform.machine()) - + return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) + +def get_sys_type_from_uname(): + """ Returns a sys_type from the uname argument + Front-end config + """ + return Architecture(os.uname()[0] + " " + os.uname()[-1]) + +def get_sys_type_from_config_file(): + """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since + The user can specify that the architecture is a cray-xc40 + """ + + home_dir = os.environ["HOME"] + yaml_file = os.path.join(home_dir, ".spack/architecture.yaml") + if os.path.isfile(yaml_file): + with open(yaml_file) as config: + config_dict = config['architecture'] + front_end = config_dict['front'] + back_end = config_dict['back'] + return Architecture(front_end) @memoized def sys_type(): - """Returns a SysType for the current machine.""" + """Returns a SysType for the current machine. Should return output to an + Architecture class + """ methods = [get_sys_type_from_spack_globals, get_sys_type_from_environment, get_mac_sys_type] # search for a method that doesn't return None - sys_type = None + sys_type = (None,None) for method in methods: sys_type = method() if sys_type: break @@ -90,3 +130,4 @@ def sys_type(): raise InvalidSysTypeError(sys_type) return sys_type + -- cgit v1.2.3-60-g2f50 From 29e03ac851f807e9fb5aefb687902e703d208615 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 12:46:44 -0700 Subject: Added __eq__ testing method. Created tests for it in test/ folder --- lib/spack/spack/architecture.py | 56 +++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 829aaa4c1c..97c2cdbd6d 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -57,10 +57,20 @@ class Architecture(object): for n in self.arch_name: if 'cray' in n.lower(): self.arch_dict[n] = "MODULES" - if 'linux' in n.lower() or 'x86_64' in n.lower(): + elif 'linux' in n.lower() or 'x86_64' in n.lower(): self.arch_dict[n] = "PATH" else: self.arch_dict[n] = None + + def get_arch_dict(self): + """ Grab the dictionary from the Architecture class, rather than access the internal Architecture attributes """ + return self.arch_dict + + def __eq__(self, other): + if self.arch_dict != {} and other.arch_dict != {}: + return self.arch_dict == other.arch_dict + else: + return self.arch_name == self.arch_name def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" @@ -95,29 +105,43 @@ def get_sys_type_from_uname(): def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since - The user can specify that the architecture is a cray-xc40 + The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack + is installed. Similar to .spackconfig """ - home_dir = os.environ["HOME"] - yaml_file = os.path.join(home_dir, ".spack/architecture.yaml") - if os.path.isfile(yaml_file): - with open(yaml_file) as config: - config_dict = config['architecture'] - front_end = config_dict['front'] - back_end = config_dict['back'] - return Architecture(front_end) + spack_home_dir = os.environ["HOME"] + "/.spack" + yaml_file = os.path.join(spack_home_dir, "architecture.yaml") + + try: + config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() + arch = config_dict['architecture'] + front = arch['front'] + back = arch['back'] + + except: + print "No architecture.yaml config file found" + + return Architecture(front,back) @memoized def sys_type(): - """Returns a SysType for the current machine. Should return output to an - Architecture class + """Priority of gathering sys-type. + 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 + 2. UNAME + 3. GLOBALS + 4. MAC OSX + Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. + If there is no yaml present then it should move on to the next function and stop immediately once it gets a + arch name + """ - methods = [get_sys_type_from_spack_globals, - get_sys_type_from_environment, - get_mac_sys_type] + methods = [get_sys_type_from_config_file, + get_sys_type_from_uname, + get_sys_type_from_spack_globals, + get_mac_sys_type] # search for a method that doesn't return None - sys_type = (None,None) + sys_type = None for method in methods: sys_type = method() if sys_type: break -- cgit v1.2.3-60-g2f50 From d328f4c3b631e59ed14f52cf38f3d05774100c15 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 12:47:26 -0700 Subject: Test suite for architecture class and functions --- lib/spack/spack/test/architecture.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 lib/spack/spack/test/architecture.py (limited to 'lib') diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py new file mode 100644 index 0000000000..6ff22aaa59 --- /dev/null +++ b/lib/spack/spack/test/architecture.py @@ -0,0 +1,19 @@ +""" Test checks if the architecture class is created correctly and also that + the functions are looking for the correct architecture name +""" +import unittest +import spack +from spack.architecture import * + +class ArchitectureTest(unittest.TestCase): + + def test_Architecture_class(self): + a = Architecture('Cray-XC40') + a.add_arch_strategy() + self.assertEquals(a.get_arch_dict(), {'Cray-XC40': 'MODULES'}) + + def test_get_sys_type_from_config_file(self): + output_arch_class = get_sys_type_from_config_file() + my_arch_class = Architecture('Linux x86_64','Cray-xc40') + + self.assertEqual(output_arch_class, my_arch_class) -- cgit v1.2.3-60-g2f50 From ccdf1057592b48e13ba3b98df9972324e91d0be8 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 12:48:12 -0700 Subject: Commented out a long list of tests to just include my arch test --- lib/spack/spack/test/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 6b3715be6f..ed51fac33a 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -31,7 +31,8 @@ from llnl.util.tty.colify import colify import spack """Names of tests to be included in Spack's test suite""" -test_names = ['versions', +"""test_names = ['architecture', + 'versions', 'url_parse', 'url_substitution', 'packages', @@ -57,7 +58,8 @@ test_names = ['versions', 'optional_deps', 'make_executable', 'configure_guess'] - +""" +test_names = ['architecture'] def list_tests(): """Return names of all tests that can be run for Spack.""" -- cgit v1.2.3-60-g2f50 From fec197ccac94af485745b6b40d8150b4ae030e99 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 15:25:13 -0700 Subject: Fixed the output of sys_type(), might need to add back the error handling part --- lib/spack/spack/architecture.py | 47 ++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 97c2cdbd6d..8f74fbc2e1 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -60,7 +60,7 @@ class Architecture(object): elif 'linux' in n.lower() or 'x86_64' in n.lower(): self.arch_dict[n] = "PATH" else: - self.arch_dict[n] = None + self.arch_dict[n] = "" def get_arch_dict(self): """ Grab the dictionary from the Architecture class, rather than access the internal Architecture attributes """ @@ -72,6 +72,7 @@ class Architecture(object): else: return self.arch_name == self.arch_name + def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" if not hasattr(spack, "sys_type"): @@ -86,6 +87,7 @@ def get_sys_type_from_spack_globals(): # """Return $SYS_TYPE or None if it's not defined.""" # return os.environ.get('SYS_TYPE') + def get_mac_sys_type(): """Return a Mac OS SYS_TYPE or None if this isn't a mac. Front-end config @@ -97,12 +99,14 @@ def get_mac_sys_type(): return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) + def get_sys_type_from_uname(): """ Returns a sys_type from the uname argument Front-end config """ return Architecture(os.uname()[0] + " " + os.uname()[-1]) + def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack @@ -110,21 +114,22 @@ def get_sys_type_from_config_file(): """ spack_home_dir = os.environ["HOME"] + "/.spack" - yaml_file = os.path.join(spack_home_dir, "architecture.yaml") + yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') try: config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() arch = config_dict['architecture'] front = arch['front'] back = arch['back'] + return Architecture(front,back) except: print "No architecture.yaml config file found" - - return Architecture(front,back) + return None + @memoized -def sys_type(): +def sys_type(): # This function is going to give me issues isn't it?? """Priority of gathering sys-type. 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 2. UNAME @@ -133,25 +138,19 @@ def sys_type(): Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. If there is no yaml present then it should move on to the next function and stop immediately once it gets a arch name - """ - methods = [get_sys_type_from_config_file, - get_sys_type_from_uname, - get_sys_type_from_spack_globals, - get_mac_sys_type] - - # search for a method that doesn't return None - sys_type = None - for method in methods: - sys_type = method() - if sys_type: break - - # Couldn't determine the sys_type for this machine. - if sys_type is None: - return "unknown_arch" - - if not isinstance(sys_type, basestring): - raise InvalidSysTypeError(sys_type) - + # Try to create an architecture object using the config file FIRST + functions = [get_sys_type_from_config_file, + get_sys_type_from_uname, + get_sys_type_from_spack_globals, + get_mac_sys_type] + + # TODO: Test for mac OSX system type but I'm sure it will be okay + for func in functions: + sys_type = None + sys_type = func() + if sys_type: + break + return sys_type -- cgit v1.2.3-60-g2f50 From 3ba2842b53752e9a3ec8766f3a1350c6cc7577a2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 20 Oct 2015 13:33:21 -0700 Subject: Got Architecture class working the way i wanted to. Next to write tests --- lib/spack/spack/architecture.py | 70 ++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 8f74fbc2e1..1a9f9de2cb 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -34,47 +34,50 @@ from external import yaml class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__( - "Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__( - "Could not determine sys_type for this machine.") + super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") + class Architecture(object): - def __init__(self, *arch_name): + """ Architecture class that contains a dictionary of architecture name and compiler search strategy methods. + The idea is to create an object that Spack can interact with and know how to search for the compiler + If it is on a Cray architecture it should look in modules. If it is anything else search $PATH. + """ + + def __init__(self, front=None, back=None): - """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) and a strategy for - searching for that architecture's compiler. The target passed to it should be a dictionary of names and strategies. + """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) + and a strategy for searching for that architecture's compiler. + The target passed to it should be a dictionary of names and strategies. """ - self.arch_dict = {} - self.arch_name = arch_name - - def add_arch_strategy(self): - """ Create a dictionary using the tuples of arch_names""" - for n in self.arch_name: - if 'cray' in n.lower(): - self.arch_dict[n] = "MODULES" - elif 'linux' in n.lower() or 'x86_64' in n.lower(): - self.arch_dict[n] = "PATH" - else: - self.arch_dict[n] = "" - - def get_arch_dict(self): - """ Grab the dictionary from the Architecture class, rather than access the internal Architecture attributes """ - return self.arch_dict - - def __eq__(self, other): - if self.arch_dict != {} and other.arch_dict != {}: - return self.arch_dict == other.arch_dict - else: - return self.arch_name == self.arch_name - + names = [] + names.append(front) + names.append(back) + + def add_compiler_strategy(names): + """ Create a dictionary of {'arch-name': 'strategy'} + This will tell Spack whether to look in the $PATH + or $MODULES location for compilers + """ + d = {} + for n in names: + if n: + if 'cray' in n.lower(): + d[n] = "MODULES" + elif 'linux' in n.lower(): + d[n] = "PATH" + else: + d[n] = 'No Strategy' + return d + + self.arch_dict = add_compiler_strategy(names) -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" +def get_sys_type_from_spack_globals(): #TODO: Figure out how this function works + """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): return None elif hasattr(spack.sys_type, "__call__"): @@ -104,7 +107,7 @@ def get_sys_type_from_uname(): """ Returns a sys_type from the uname argument Front-end config """ - return Architecture(os.uname()[0] + " " + os.uname()[-1]) + return Architecture(os.uname()[0]) def get_sys_type_from_config_file(): @@ -149,6 +152,7 @@ def sys_type(): # This function is going to give me issues isn't it?? for func in functions: sys_type = None sys_type = func() + if sys_type: break -- cgit v1.2.3-60-g2f50 From ca3cc5b23e94b8168ba768ad343822b39aac4a25 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 20 Oct 2015 13:37:06 -0700 Subject: Refactored architecture class. Now it will automagically create a dict upon instantiation. --- lib/spack/spack/architecture.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 1a9f9de2cb..760c9cddd1 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -49,7 +49,6 @@ class Architecture(object): """ def __init__(self, front=None, back=None): - """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) and a strategy for searching for that architecture's compiler. The target passed to it should be a dictionary of names and strategies. @@ -63,6 +62,7 @@ class Architecture(object): This will tell Spack whether to look in the $PATH or $MODULES location for compilers """ + #TODO: Look for other strategies d = {} for n in names: if n: @@ -76,21 +76,20 @@ class Architecture(object): self.arch_dict = add_compiler_strategy(names) -def get_sys_type_from_spack_globals(): #TODO: Figure out how this function works +def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): return None elif hasattr(spack.sys_type, "__call__"): - return Architecture(spack.sys_type()) + return Architecture(spack.sys_type()) #If in __init__.py there is a sys_type() then call that else: - return Architecture(spack.sys_type) + return Architecture(spack.sys_type) # Else use the attributed which defaults to None # This is livermore dependent. Hard coded for livermore #def get_sys_type_from_environment(): # """Return $SYS_TYPE or None if it's not defined.""" # return os.environ.get('SYS_TYPE') - def get_mac_sys_type(): """Return a Mac OS SYS_TYPE or None if this isn't a mac. Front-end config -- cgit v1.2.3-60-g2f50 From c31da9bc8f10c3a74b3bf26c2dbac9a13c073eba Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 20 Oct 2015 14:10:54 -0700 Subject: Made sure architecture works with yaml file --- lib/spack/spack/architecture.py | 46 ++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 760c9cddd1..8460b0de1e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -49,18 +49,24 @@ class Architecture(object): """ def __init__(self, front=None, back=None): - """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) - and a strategy for searching for that architecture's compiler. - The target passed to it should be a dictionary of names and strategies. + """ Constructor for the architecture class. It will create a list from the given arguments and iterate + through that list. It will then create a dictionary of {arch_name : strategy} + Takes in two parameters: + + front = None defaults to None. Should be the front-end architecture of the machine + back = None defaults to None. Should be the back-end architecture of the machine + + If no arguments are given it will return an empty dictionary """ - names = [] - names.append(front) - names.append(back) + _names = [] + _names.append(front) + _names.append(back) - def add_compiler_strategy(names): + def _add_compiler_strategy(names): """ Create a dictionary of {'arch-name': 'strategy'} This will tell Spack whether to look in the $PATH or $MODULES location for compilers + Else it will return No Strategy """ #TODO: Look for other strategies d = {} @@ -74,7 +80,7 @@ class Architecture(object): d[n] = 'No Strategy' return d - self.arch_dict = add_compiler_strategy(names) + self.arch_dict = _add_compiler_strategy(_names) def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" @@ -85,20 +91,20 @@ def get_sys_type_from_spack_globals(): else: return Architecture(spack.sys_type) # Else use the attributed which defaults to None + # This is livermore dependent. Hard coded for livermore #def get_sys_type_from_environment(): # """Return $SYS_TYPE or None if it's not defined.""" # return os.environ.get('SYS_TYPE') + def get_mac_sys_type(): """Return a Mac OS SYS_TYPE or None if this isn't a mac. Front-end config """ - mac_ver = py_platform.mac_ver()[0] if not mac_ver: return None - return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) @@ -113,11 +119,9 @@ def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack is installed. Similar to .spackconfig - """ - + """ spack_home_dir = os.environ["HOME"] + "/.spack" yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') - try: config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() arch = config_dict['architecture'] @@ -131,7 +135,7 @@ def get_sys_type_from_config_file(): @memoized -def sys_type(): # This function is going to give me issues isn't it?? +def sys_type(): """Priority of gathering sys-type. 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 2. UNAME @@ -146,14 +150,18 @@ def sys_type(): # This function is going to give me issues isn't it?? get_sys_type_from_uname, get_sys_type_from_spack_globals, get_mac_sys_type] - - # TODO: Test for mac OSX system type but I'm sure it will be okay + + sys_type = None for func in functions: - sys_type = None sys_type = func() - if sys_type: - break + break + if sys_type is None: + return Architecture("unknown_arch") + + if not isinstance(sys_type, Architecture): + raise InvalidSysTypeError(sys_type) + return sys_type -- cgit v1.2.3-60-g2f50 From 51b69ef00b947f4cf1801358b1a5da413d0b21bc Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 21 Oct 2015 09:27:57 -0700 Subject: initial fetch on crayport --- lib/spack/spack/compiler.py | 2 +- lib/spack/spack/directives.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index e7d450ee8b..4ce6e3f11d 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -119,7 +119,7 @@ class Compiler(object): self.fc = check(fc) self.spec = cspec - self.module = module + self.modules = modules.split() diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 9297d6dac3..4bdc8772b8 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -252,7 +252,7 @@ def variant(pkg, name, default=False, description=""): """Define a variant for the package. Packager can specify a default value (on or off) as well as a text description.""" - default = bool(default) + default = default description = str(description).strip() if not re.match(spack.spec.identifier_re, name): -- cgit v1.2.3-60-g2f50 From 9b387e7682c22b74661e3363442adcfb569d8680 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 21 Oct 2015 11:09:05 -0700 Subject: Added strategy method in init --- lib/spack/spack/architecture.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 8460b0de1e..3968b82124 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -57,18 +57,13 @@ class Architecture(object): back = None defaults to None. Should be the back-end architecture of the machine If no arguments are given it will return an empty dictionary + Uses the _add_compiler_strategy(front, back) to create the dictionary """ - _names = [] - _names.append(front) - _names.append(back) - - def _add_compiler_strategy(names): - """ Create a dictionary of {'arch-name': 'strategy'} - This will tell Spack whether to look in the $PATH - or $MODULES location for compilers - Else it will return No Strategy - """ - #TODO: Look for other strategies + + def _add_compiler_strategy(front,back): + names = [] + names.append(front) + names.append(back) d = {} for n in names: if n: @@ -80,7 +75,8 @@ class Architecture(object): d[n] = 'No Strategy' return d - self.arch_dict = _add_compiler_strategy(_names) + self.arch_dict = _add_compiler_strategy(front, back) + def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" -- cgit v1.2.3-60-g2f50 From a89abb435f288c2864d9a21089b20f6eaac5d98b Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 21 Oct 2015 11:32:59 -0700 Subject: Changed structure of class, add compiler strategy is a method and can create a dict --- lib/spack/spack/architecture.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3968b82124..7c1bdfb20f 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -48,6 +48,21 @@ class Architecture(object): If it is on a Cray architecture it should look in modules. If it is anything else search $PATH. """ + def add_compiler_strategy(self, front,back): + names = [] + names.append(front) + names.append(back) + d = {} + for n in names: + if n: + if 'cray' in n.lower(): + d[n] = "MODULES" + elif 'linux' in n.lower(): + d[n] = "PATH" + else: + d[n] = 'No Strategy' + return d + def __init__(self, front=None, back=None): """ Constructor for the architecture class. It will create a list from the given arguments and iterate through that list. It will then create a dictionary of {arch_name : strategy} @@ -59,23 +74,9 @@ class Architecture(object): If no arguments are given it will return an empty dictionary Uses the _add_compiler_strategy(front, back) to create the dictionary """ - - def _add_compiler_strategy(front,back): - names = [] - names.append(front) - names.append(back) - d = {} - for n in names: - if n: - if 'cray' in n.lower(): - d[n] = "MODULES" - elif 'linux' in n.lower(): - d[n] = "PATH" - else: - d[n] = 'No Strategy' - return d - - self.arch_dict = _add_compiler_strategy(front, back) + self.front = front + self.back = back + self.arch_dict = self.add_compiler_strategy(front, back) def get_sys_type_from_spack_globals(): -- cgit v1.2.3-60-g2f50 From 38508c5a3f294f730b31e209dfa90545e1920588 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 22 Oct 2015 12:02:26 -0700 Subject: Created a control flow logic that will loop through the strategies and find compilers using that. TODO: Need to find a way to locate their executables --- lib/spack/spack/cmd/compiler.py | 51 +++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 2a64dc914e..cbe3aba864 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -29,6 +29,8 @@ from llnl.util.tty.color import colorize from llnl.util.tty.colify import colify from llnl.util.lang import index_by +import spack.architecture +import spack.compiler import spack.compilers import spack.spec import spack.config @@ -36,13 +38,12 @@ from spack.util.environment import get_path from spack.spec import CompilerSpec description = "Manage compilers" +ARCHITECTURE = spack.architecture.sys_type() def setup_parser(subparser): - sp = subparser.add_subparsers( - metavar='SUBCOMMAND', dest='compiler_command') + sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='compiler_command') - update_parser = sp.add_parser( - 'add', help='Add compilers to the Spack configuration.') + update_parser = sp.add_parser('add', help='Add compilers to the Spack configuration.') update_parser.add_argument('add_paths', nargs=argparse.REMAINDER) remove_parser = sp.add_parser('remove', help='remove compiler') @@ -55,23 +56,33 @@ def setup_parser(subparser): def compiler_add(args): - """Search either $PATH or a list of paths for compilers and add them + """Search either $PATH or a list of paths OR MODULES for compilers and add them to Spack's configuration.""" - paths = args.add_paths - if not paths: - paths = get_path('PATH') - - compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) - if c.spec not in spack.compilers.all_compilers()] - - if compilers: - spack.compilers.add_compilers_to_config('user', *compilers) - n = len(compilers) - tty.msg("Added %d new compiler%s to %s" % ( - n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) - colify(reversed(sorted(c.spec for c in compilers)), indent=4) - else: - tty.msg("Found no new compilers") + + strategies = ARCHITECTURE.strategy() + + for strategy in strategies: + if strategy == 'PATH': + paths = args.add_paths # This might be a parser method. Parsing method to add_paths + if not paths: + paths = get_path('PATH') + + compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) + if c.spec not in spack.compilers.all_compilers()] + + elif strategy == "MODULES": + from spack.compilers.cray import Cray + compilers = Cray.find_in_modules() + #TODO: Find a way to locate the executables + + if compilers: + spack.compilers.add_compilers_to_config('user', *compilers) + n = len(compilers) + tty.msg("Added %d new compiler%s to %s" % ( + n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) + colify(reversed(sorted(c.spec for c in compilers)), indent=4) + else: + tty.msg("Found no new compilers") def compiler_remove(args): -- cgit v1.2.3-60-g2f50 From 09597fe8dccbd7d49acf1b3198f24ab928874cdb Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 09:44:28 -0700 Subject: updated the executible to return stderr when specified. Added load_module to build_environment.py, loads target --- lib/spack/spack/build_environment.py | 36 +++++++++++++++++++++++++++++------- lib/spack/spack/util/executable.py | 21 ++++++++++++++++----- 2 files changed, 45 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 68388958f5..191e858735 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -86,6 +86,28 @@ class MakeExecutable(Executable): return super(MakeExecutable, self).__call__(*args, **kwargs) +def load_module(mod): + """Takes a module name and removes modules until it is possible to + load that module. It then loads the provided module. Depends on the + modulecmd implementation of modules used in cray and lmod. + """ + #Create an executable of the module command that will output python code + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module and remove any conflicting modules + # We do this without checking that they are already installed + # for ease of programming because unloading a module that is not + # loaded does nothing. + text = modulecmd('show', mod, return_oe=True).split() + for i, word in enumerate(text): + if word == 'conflict': + exec(compile(modulecmd('unload', text[i+1], return_oe=True), '', 'exec')) + # Load the module now that there are no conflicts + load = modulecmd('load', mod, return_oe=True) + exec(compile(load, '', 'exec')) + + def set_compiler_environment_variables(pkg): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -108,11 +130,9 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) - if compiler.PrgEnv: - os.environ['SPACK_CRAYPE'] = compiler.PrgEnv - os.environ['SPACK_COMP_MODULE'] = compiler.module - - + if compiler.modules: + for mod in compiler.modules: + load_module(mod) def set_build_environment_variables(pkg): @@ -163,8 +183,10 @@ def set_build_environment_variables(pkg): pcdir = join_path(p, libdir, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) - path_set("PKG_CONFIG_PATH", pkg_config_dirs) + path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) + if pkg.spec.architecture.compiler_strategy.lower() == 'module': + load_module(pkg.spec.architecture.module_name) def set_module_variables_for_package(pkg): """Populate the module scope of install() with some useful functions. @@ -239,8 +261,8 @@ def get_rpaths(pkg): def setup_package(pkg): """Execute all environment setup routines.""" - set_compiler_environment_variables(pkg) set_build_environment_variables(pkg) + set_compiler_environment_variables(pkg) set_module_variables_for_package(pkg) # Allow dependencies to set up environment as well. diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index d1dfb62ffb..15354089ac 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -56,7 +56,11 @@ class Executable(object): def __call__(self, *args, **kwargs): """Run the executable with subprocess.check_output, return output.""" - return_output = kwargs.get("return_output", False) + # Return oe returns a combined stream, setting both output and error + # without setting return oe returns them concatenated by a double line break + return_oe = kwargs.get("return_oe", False) + return_output = True if return_oe else kwargs.get("return_output", False) + return_error = True if return_oe else kwargs.get("return_error", False) fail_on_error = kwargs.get("fail_on_error", True) ignore_errors = kwargs.get("ignore_errors", ()) @@ -95,8 +99,8 @@ class Executable(object): proc = subprocess.Popen( cmd, stdin=input, - stderr=error, - stdout=subprocess.PIPE if return_output else output) + stdout=subprocess.PIPE if return_output else output, + stderr=subprocess.STDOUT if return_oe else (subprocess.PIPE if return_error else error)) out, err = proc.communicate() self.returncode = proc.returncode @@ -104,8 +108,15 @@ class Executable(object): if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:" % proc.returncode, cmd_line) - if return_output: - return out + # Return out or error if specified. Return combined stream if requested, + # otherwise return them concatenated by double line break if both requested. + if return_output or return_error: + if return_oe or not return_error: + return out + elif return_output: + return out+'\n\n'+err + else: + return err except OSError, e: raise ProcessError( -- cgit v1.2.3-60-g2f50 From 382d8478bf325099582b4814e0d71ff8522696aa Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 10:23:33 -0700 Subject: prototype of new architecture concretization saved as new_concretize_architecture in concretize.py --- lib/spack/spack/concretize.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 66002492cb..c5041d67be 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -122,6 +122,34 @@ class DefaultConcretizer(object): return True # changed + def new_concretize_architecture(self, spec): + """If the spec already has an architecture and it is a an architecture type, + return. Otherwise, if it has an architecture that is a string type, generate an + architecture based on that type. If it has no architecture and the root of the + DAG has an architecture, then use that. Otherwise, take the system's default + architecture. + """ + if spec.architecture is not None: + if isinstance(spec.architecture,spack.architecture.Target): + return False + else: + arch = spack.architecture.sys_type() + spec.architecture = arch.target(spec.architecture) + return True #changed + + if spec.root.architecture: + if isinstance(spec.root.architecture,spack.architecture.Target): + spec.architecture = spec.root.architecture + else: + arch = spack.architecture.sys_type() + spec.architecture = arch.target(spec.root.architecture) + else: + arch = spack.architecture.sys_type() + spec.architecture = arch.target('default') + + return True #changed + + def concretize_variants(self, spec): """If the spec already has variants filled in, return. Otherwise, add the default variants from the package specification. -- cgit v1.2.3-60-g2f50 From 7ab921ff02a7d0a05287e4a49738ec9c51361ef0 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 14:46:26 -0700 Subject: Changed architecture class and added class Target --- lib/spack/spack/architecture.py | 137 ++++++++++++++++++++++++---------------- 1 file changed, 83 insertions(+), 54 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 7c1bdfb20f..ea1f98e06e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -23,11 +23,16 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import imp import platform as py_platform +import inspect -from llnl.util.lang import memoized +from llnl.util.lang import memoized, list_modules +from llnl.util.filesystem import join_path +import llnl.util.tty as tty import spack +from spack.util.naming import mod_to_class import spack.error as serr from spack.version import Version from external import yaml @@ -42,41 +47,55 @@ class NoSysTypeError(serr.SpackError): super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") +class Target(object): + """ This is the processor type e.g. cray-ivybridge """ + # Front end or back end target. Target needs to know which one this is + # Should autodetect from the machine + # features of a target + # - a module name + # -a compiler finding strategy + # -a name + # architecture classes handling the aliasing for front-end, back-end and default + + def __init__(self,name, module_name=None): + self.name = name # case of cray "ivybridge but if it's x86_64 + self.module_name = module_name # craype-ivybridge + + def compiler_strategy(self): + if self.module_name: # If there is a module_name given then use MODULES + return "MODULES" + else: + return "PATH" + class Architecture(object): - """ Architecture class that contains a dictionary of architecture name and compiler search strategy methods. - The idea is to create an object that Spack can interact with and know how to search for the compiler - If it is on a Cray architecture it should look in modules. If it is anything else search $PATH. + """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it + is returned """ - - def add_compiler_strategy(self, front,back): - names = [] - names.append(front) - names.append(back) - d = {} - for n in names: - if n: - if 'cray' in n.lower(): - d[n] = "MODULES" - elif 'linux' in n.lower(): - d[n] = "PATH" - else: - d[n] = 'No Strategy' - return d - - def __init__(self, front=None, back=None): - """ Constructor for the architecture class. It will create a list from the given arguments and iterate - through that list. It will then create a dictionary of {arch_name : strategy} - Takes in two parameters: - front = None defaults to None. Should be the front-end architecture of the machine - back = None defaults to None. Should be the back-end architecture of the machine + priority = None # Subclass needs to set this number. This controls order in which arch is detected. + front = None + back = None + default_front = None # The default front end target. On cray sandybridge + default_back = None # The default back end target. On cray ivybridge + + def __init__(self, name): + self.targets = {} + self.name = name + + def add_target(self, name, target): + self.targets[name] = target - If no arguments are given it will return an empty dictionary - Uses the _add_compiler_strategy(front, back) to create the dictionary + + @classmethod + def detect(self): + """ Subclass is responsible for implementing this method. + Returns True if the architecture detects if it is the current architecture + and False if it's not. """ - self.front = front - self.back = back - self.arch_dict = self.add_compiler_strategy(front, back) + raise NotImplementedError() + + def __str__(self): + return self.name def get_sys_type_from_spack_globals(): @@ -84,9 +103,9 @@ def get_sys_type_from_spack_globals(): if not hasattr(spack, "sys_type"): return None elif hasattr(spack.sys_type, "__call__"): - return Architecture(spack.sys_type()) #If in __init__.py there is a sys_type() then call that + return spack.sys_type() #If in __init__.py there is a sys_type() then call that else: - return Architecture(spack.sys_type) # Else use the attributed which defaults to None + return spack.sys_type # Else use the attributed which defaults to None # This is livermore dependent. Hard coded for livermore @@ -102,15 +121,19 @@ def get_mac_sys_type(): mac_ver = py_platform.mac_ver()[0] if not mac_ver: return None - return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) + return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine()) def get_sys_type_from_uname(): """ Returns a sys_type from the uname argument Front-end config """ - return Architecture(os.uname()[0]) - + try: + arch_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + arch, _ = arch_proc.communicate() + return arch.strip() + except: + return None def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since @@ -131,34 +154,40 @@ def get_sys_type_from_config_file(): return None +@memoized +def all_architectures(): + modules = [] + for name in list_modules(spack.arch_path): + mod_name = 'spack.architectures.' + name + path = join_path(spack.arch_path, name) + ".py" + mod = imp.load_source(mod_name, path) + class_name = mod_to_class(name) + if not hasattr(mod, class_name): + tty.die('No class %s defined in %s' % (class_name, mod_name)) + cls = getattr(mod, class_name) + if not inspect.isclass(cls): + tty.die('%s.%s is not a class' % (mod_name, class_name)) + + modules.append(cls) + + return modules + @memoized def sys_type(): """Priority of gathering sys-type. 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 2. UNAME 3. GLOBALS - 4. MAC OSX + 4. MAC OSX Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. If there is no yaml present then it should move on to the next function and stop immediately once it gets a arch name """ # Try to create an architecture object using the config file FIRST - functions = [get_sys_type_from_config_file, - get_sys_type_from_uname, - get_sys_type_from_spack_globals, - get_mac_sys_type] - - sys_type = None - for func in functions: - sys_type = func() - if sys_type: - break + architecture_list = all_architectures() + architecture_list.sort(key = lambda a: a.priority) - if sys_type is None: - return Architecture("unknown_arch") - - if not isinstance(sys_type, Architecture): - raise InvalidSysTypeError(sys_type) - - return sys_type + for arch in architecture_list: + if arch.detect(): + return arch() -- cgit v1.2.3-60-g2f50 From d177184777591d21cc6a59771ed6f817960394f5 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 15:12:29 -0700 Subject: added target method to architecture.py and minor cleanup --- lib/spack/spack/architecture.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index ea1f98e06e..f04fda2ee9 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -58,9 +58,10 @@ class Target(object): # architecture classes handling the aliasing for front-end, back-end and default def __init__(self,name, module_name=None): - self.name = name # case of cray "ivybridge but if it's x86_64 + self.name = name # case of cray "ivybridge" but if it's x86_64 self.module_name = module_name # craype-ivybridge + @property def compiler_strategy(self): if self.module_name: # If there is a module_name given then use MODULES return "MODULES" @@ -73,10 +74,9 @@ class Architecture(object): """ priority = None # Subclass needs to set this number. This controls order in which arch is detected. - front = None - back = None - default_front = None # The default front end target. On cray sandybridge - default_back = None # The default back end target. On cray ivybridge + front-end = None + back-end = None + default = None # The default back end target. On cray ivybridge def __init__(self, name): self.targets = {} @@ -85,7 +85,19 @@ class Architecture(object): def add_target(self, name, target): self.targets[name] = target - + def target(self, name): + """This is a getter method for the target dictionary that handles defaulting based + on the values provided by default, front-end, and back-end. This can be overwritten + by a subclass for which we want to provide further aliasing options. + """ + if name == 'default': + name = default + elif name == 'front_end': + name = front-end + elif name == 'back_end': + name = back-end + return self.targets[name] + @classmethod def detect(self): """ Subclass is responsible for implementing this method. -- cgit v1.2.3-60-g2f50 From 6e560703862794632176733a1d063550215dc573 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 15:15:36 -0700 Subject: improved aliasing in target method in architecture.py --- lib/spack/spack/architecture.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index f04fda2ee9..3f38540c0e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -92,9 +92,9 @@ class Architecture(object): """ if name == 'default': name = default - elif name == 'front_end': + elif name == 'front_end' or name == 'fe': name = front-end - elif name == 'back_end': + elif name == 'back_end' or name == 'be': name = back-end return self.targets[name] -- cgit v1.2.3-60-g2f50 From db0695e46a6c6fd4d2ac10dba04f7feb18c25a39 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:57:00 -0700 Subject: architectures folder where the Architecture subclasses reside --- lib/spack/spack/architectures/__init__.py | 0 lib/spack/spack/architectures/cray.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 lib/spack/spack/architectures/__init__.py create mode 100644 lib/spack/spack/architectures/cray.py (limited to 'lib') diff --git a/lib/spack/spack/architectures/__init__.py b/lib/spack/spack/architectures/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py new file mode 100644 index 0000000000..e6d6d84922 --- /dev/null +++ b/lib/spack/spack/architectures/cray.py @@ -0,0 +1,17 @@ +import os + +from spack.architecture import Architecture + +class Cray(Architecture): + priority = 20 + front_end = None + back_end = None + default = None + + def __init__(self): + super(Cray, self).__init__('cray') + + @classmethod + def detect(self): + return os.path.exists('/opt/cray/craype') + -- cgit v1.2.3-60-g2f50 From 2d87bb92edb95cf1c9ec7d58f5d8a34e44bd11a4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:58:20 -0700 Subject: Changed front-end and back-end to front_end and back_end to avoid error --- lib/spack/spack/architecture.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3f38540c0e..f488b65cc6 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -74,8 +74,8 @@ class Architecture(object): """ priority = None # Subclass needs to set this number. This controls order in which arch is detected. - front-end = None - back-end = None + front_end = None + back_end = None default = None # The default back end target. On cray ivybridge def __init__(self, name): -- cgit v1.2.3-60-g2f50 From e39586c81da152e8d02fd26bf48c66e00e26519a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:58:49 -0700 Subject: Reverted back all architecture way of finding modules --- lib/spack/spack/cmd/compiler.py | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index cbe3aba864..3e86928977 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -38,7 +38,6 @@ from spack.util.environment import get_path from spack.spec import CompilerSpec description = "Manage compilers" -ARCHITECTURE = spack.architecture.sys_type() def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='compiler_command') @@ -59,30 +58,23 @@ def compiler_add(args): """Search either $PATH or a list of paths OR MODULES for compilers and add them to Spack's configuration.""" - strategies = ARCHITECTURE.strategy() - for strategy in strategies: - if strategy == 'PATH': - paths = args.add_paths # This might be a parser method. Parsing method to add_paths - if not paths: - paths = get_path('PATH') - - compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) - if c.spec not in spack.compilers.all_compilers()] - - elif strategy == "MODULES": - from spack.compilers.cray import Cray - compilers = Cray.find_in_modules() - #TODO: Find a way to locate the executables - - if compilers: - spack.compilers.add_compilers_to_config('user', *compilers) - n = len(compilers) - tty.msg("Added %d new compiler%s to %s" % ( - n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) - colify(reversed(sorted(c.spec for c in compilers)), indent=4) - else: - tty.msg("Found no new compilers") + paths = args.add_paths # This might be a parser method. Parsing method to add_paths + if not paths: + paths = get_path('PATH') + + compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) + if c.spec not in spack.compilers.all_compilers()] + + + if compilers: + spack.compilers.add_compilers_to_config('user', *compilers) + n = len(compilers) + tty.msg("Added %d new compiler%s to %s" % ( + n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) + colify(reversed(sorted(c.spec for c in compilers)), indent=4) + else: + tty.msg("Found no new compilers") def compiler_remove(args): -- cgit v1.2.3-60-g2f50 From 9a91da9ccd8044e147a416fcfa9b16405ef22b3d Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:59:28 -0700 Subject: Added arch_path to list of modules path --- lib/spack/spack/__init__.py | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index caa09eb6e0..0f23f61614 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -37,6 +37,7 @@ etc_path = join_path(prefix, "etc") lib_path = join_path(prefix, "lib", "spack") build_env_path = join_path(lib_path, "env") module_path = join_path(lib_path, "spack") +arch_path = join_path(module_path, 'architectures') compilers_path = join_path(module_path, "compilers") test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") -- cgit v1.2.3-60-g2f50 From eb2cf1698fa8cd606a8b80bcb39de2ee0b724c69 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 2 Nov 2015 11:08:08 -0800 Subject: Fixed some issues with naming --- lib/spack/spack/architecture.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index f488b65cc6..442180242b 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -49,13 +49,6 @@ class NoSysTypeError(serr.SpackError): class Target(object): """ This is the processor type e.g. cray-ivybridge """ - # Front end or back end target. Target needs to know which one this is - # Should autodetect from the machine - # features of a target - # - a module name - # -a compiler finding strategy - # -a name - # architecture classes handling the aliasing for front-end, back-end and default def __init__(self,name, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 @@ -67,7 +60,8 @@ class Target(object): return "MODULES" else: return "PATH" - + + class Architecture(object): """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned @@ -85,17 +79,19 @@ class Architecture(object): def add_target(self, name, target): self.targets[name] = target + def target(self, name): """This is a getter method for the target dictionary that handles defaulting based on the values provided by default, front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ if name == 'default': - name = default + name = self.default elif name == 'front_end' or name == 'fe': - name = front-end + name = self.front_end elif name == 'back_end' or name == 'be': - name = back-end + name = self.back_end + return self.targets[name] @classmethod @@ -148,10 +144,7 @@ def get_sys_type_from_uname(): return None def get_sys_type_from_config_file(): - """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since - The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack - is installed. Similar to .spackconfig - """ + spack_home_dir = os.environ["HOME"] + "/.spack" yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') try: -- cgit v1.2.3-60-g2f50 From 058e72d29c3cd934f91ba626392b25ebaa50e2cc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 2 Nov 2015 11:08:55 -0800 Subject: Added default target and also front end and back end targets --- lib/spack/spack/architectures/cray.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index e6d6d84922..a79c916684 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -4,9 +4,9 @@ from spack.architecture import Architecture class Cray(Architecture): priority = 20 - front_end = None - back_end = None - default = None + front_end = 'sandybridge' + back_end = 'ivybridge' + default = os.environ["CRAY_CPU_TARGET"] def __init__(self): super(Cray, self).__init__('cray') -- cgit v1.2.3-60-g2f50 From 5ac974c9b2072631eab490cce8f2922420eef9e4 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 4 Nov 2015 12:50:22 -0800 Subject: Enforced that the architecture subclass cannot add a target that shares a name with a target alias --- lib/spack/spack/architecture.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 442180242b..890df9b1e5 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -77,9 +77,14 @@ class Architecture(object): self.name = name def add_target(self, name, target): - self.targets[name] = target - - + """Used by the architecture specific subclass to list available targets. Raises an error + if the architecture specifies a name that is reserved by spack as an alias. + """ + if name in ['front_end', 'fe', 'back_end', 'be', 'default']: + raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) + self.targets[name] = target + + def target(self, name): """This is a getter method for the target dictionary that handles defaulting based on the values provided by default, front-end, and back-end. This can be overwritten -- cgit v1.2.3-60-g2f50 From 37260962e545366d0d882b205be807562c36f3b7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 4 Nov 2015 12:57:29 -0800 Subject: changed some potential syntax errors and added a way for target to recognize class --- lib/spack/spack/architecture.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 442180242b..b251b82dcc 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -48,20 +48,29 @@ class NoSysTypeError(serr.SpackError): class Target(object): - """ This is the processor type e.g. cray-ivybridge """ + """ Target is the processor of the host machine. The host machine may have different front-end + and back-end targets, especially if it is a Cray machine. The target will have a name and + also the module_name (e.g craype-compiler). Targets will also recognize which architecture + they came from using the set_architecture method. Targets will have compiler finding strategies + """ + default_strategy = None # Can probably add a compiler path here def __init__(self,name, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 self.module_name = module_name # craype-ivybridge + def set_architecture(self, architecture): # Target should get the architecture class. + self.architecture = architecture + @property def compiler_strategy(self): - if self.module_name: # If there is a module_name given then use MODULES + if default_strategy: + return default_strategy + elif self.module_name: # If there is a module_name given then use MODULES return "MODULES" else: return "PATH" - class Architecture(object): """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned @@ -77,8 +86,8 @@ class Architecture(object): self.name = name def add_target(self, name, target): - self.targets[name] = target - + target.set_architecture(self) + self.targets[name] = target def target(self, name): """This is a getter method for the target dictionary that handles defaulting based @@ -163,7 +172,7 @@ def get_sys_type_from_config_file(): def all_architectures(): modules = [] for name in list_modules(spack.arch_path): - mod_name = 'spack.architectures.' + name + mod_name = 'spack.architectures' + name path = join_path(spack.arch_path, name) + ".py" mod = imp.load_source(mod_name, path) class_name = mod_to_class(name) -- cgit v1.2.3-60-g2f50 From 35532d6b0aaa428e1d8234513e9e887d08c3914c Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 4 Nov 2015 13:00:35 -0800 Subject: Changed cray architecture subclass to add proper targets for front at back end nodes --- lib/spack/spack/architectures/cray.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index a79c916684..420b7c589e 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -10,6 +10,11 @@ class Cray(Architecture): def __init__(self): super(Cray, self).__init__('cray') + # Back End compiler needs the proper target module loaded. + self.add_target('ivybridge','craype-ivybridge') + # Could switch to use modules and fe targets for front end + # Currently using compilers by path for front end. + self.add_target('sandybridge') @classmethod def detect(self): -- cgit v1.2.3-60-g2f50 From 9bf8e8573cff42581074ed9e492ab0e0bc6c7253 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 4 Nov 2015 13:08:48 -0800 Subject: generic linux architecture subclass --- lib/spack/spack/architectures/linux.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 lib/spack/spack/architectures/linux.py (limited to 'lib') diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py new file mode 100644 index 0000000000..7238575660 --- /dev/null +++ b/lib/spack/spack/architectures/linux.py @@ -0,0 +1,17 @@ +import subprocess +from spack.architecture import Architecture + +class Linux(Architecture): + priority = 60 + front_end = "x86_64" + back_end = "x86_64" + default = "x86_64" + + def __init__(self): + super(Linux, self).__init__('linux') + + @classmethod + def detect(self): + arch = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + arch, _ = arch.communicate() + return 'x86_64' in arch.strip() -- cgit v1.2.3-60-g2f50 From 3a73ae1683c4e83791f273d2034a99824e81aeca Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 4 Nov 2015 13:12:11 -0800 Subject: Fixed the previous commit --- lib/spack/spack/architectures/cray.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index 420b7c589e..640a3933e0 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -1,20 +1,20 @@ import os -from spack.architecture import Architecture +from spack.architecture import Architecture, Target class Cray(Architecture): priority = 20 front_end = 'sandybridge' back_end = 'ivybridge' - default = os.environ["CRAY_CPU_TARGET"] + default = 'ivybridge' def __init__(self): super(Cray, self).__init__('cray') # Back End compiler needs the proper target module loaded. - self.add_target('ivybridge','craype-ivybridge') + self.add_target('ivybridge', Target('ivybridge','craype-ivybridge')) # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target('sandybridge') + self.add_target('sandybridge', Target('sandybridge')) @classmethod def detect(self): -- cgit v1.2.3-60-g2f50 From b61d554dc818c2514384b017c74dfa61c04a8c3a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 4 Nov 2015 13:28:12 -0800 Subject: Fixed architecture.py file so it doesn't have the weird merge changes i.e <<>>>>>> 8b3f2ec1d117e1a8b206927f51db8684396c231b def target(self, name): """This is a getter method for the target dictionary that handles defaulting based on the values provided by default, front-end, and back-end. This can be overwritten -- cgit v1.2.3-60-g2f50 From 08729315c617b42e6fd1eaa1af4919aa52f96a0f Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 5 Nov 2015 09:21:56 -0800 Subject: Added bgq architecture subclass: may require additional functionality to support --- lib/spack/spack/architectures/bgq.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 lib/spack/spack/architectures/bgq.py (limited to 'lib') diff --git a/lib/spack/spack/architectures/bgq.py b/lib/spack/spack/architectures/bgq.py new file mode 100644 index 0000000000..d3d4446e09 --- /dev/null +++ b/lib/spack/spack/architectures/bgq.py @@ -0,0 +1,19 @@ +import os + +from spack.architecture import Architecture, Target + +class Bgq(Architecture): + priority = 30 + front_end = 'power7' + back_end = 'powerpc' + default = 'powerpc' + + def __init__(self): + super(Bgq, self).__init__('cray') + self.add_target('power7', Target('power7')) + self.add_target('powerpc', Target('powerpc')) + + @classmethod + def detect(self): + return os.path.exists('/bgsys') + -- cgit v1.2.3-60-g2f50 From 5347f460b4f2ce8ad64f6ec225779c2a2f944ad4 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 10:31:37 -0800 Subject: adding module find to find command --- lib/spack/spack/compiler.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 4ce6e3f11d..e4bfeea608 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -106,7 +106,7 @@ class Compiler(object): PrgEnv_compiler = None - def __init__(self, cspec, cc, cxx, f77, fc, module=None): + def __init__(self, cspec, cc, cxx, f77, fc, modules=None): def check(exe): if exe is None: return None @@ -119,7 +119,7 @@ class Compiler(object): self.fc = check(fc) self.spec = cspec - self.modules = modules.split() + self.modules = modules @@ -216,6 +216,10 @@ class Compiler(object): @classmethod def find(cls, *path): + return cls.find_in_path(*path) + cls.find_in_modules() + + @classmethod + def find_in_path(cls, *path): """Try to find this type of compiler in the user's environment. For each set of compilers found, this returns compiler objects with the cc, cxx, f77, fc paths and the @@ -273,7 +277,9 @@ class Compiler(object): if not cls.PrgEnv_compiler: tty.die('Must supply PrgEnv_compiler with PrgEnv') - output = _shell('module avail %s' % cls.PrgEnv_compiler) +# output = _shell('module avail %s' % cls.PrgEnv_compiler) + modulecmd = which('modulecmd') + modulecmd matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) for name, version in matches: -- cgit v1.2.3-60-g2f50 From 95a34628a3bebe3d263afbd2707cb5d47a2b7a33 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 11:32:18 -0800 Subject: Add modules to compilers. Changed compiler to take paths as a list. Changed compiler_for_spec to be aware of different compiler stratigies --- lib/spack/spack/compiler.py | 61 +++++++++-------------------------- lib/spack/spack/compilers/__init__.py | 13 ++++++-- lib/spack/spack/package.py | 2 +- 3 files changed, 27 insertions(+), 49 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 67765dfb7b..4be573fb12 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -106,17 +106,20 @@ class Compiler(object): PrgEnv_compiler = None - def __init__(self, cspec, cc, cxx, f77, fc, modules=None): + def __init__(self, cspec, paths, modules=None): def check(exe): if exe is None: return None _verify_executables(exe) return exe - self.cc = check(cc) - self.cxx = check(cxx) - self.f77 = check(f77) - self.fc = check(fc) + self.cc = check(paths[0]) + self.cxx = check(paths[1]) + self.f77 = check(paths[2]) + if len(paths) == 3: + self.fc = self.f77 + else: + self.fc = check(paths[3]) self.spec = cspec self.modules = modules @@ -264,7 +267,7 @@ class Compiler(object): if newcount <= prevcount: continue - compilers[ver] = cls(spec, *paths) + compilers[ver] = cls(spec, paths) return list(compilers.values()) @@ -277,60 +280,26 @@ class Compiler(object): if not cls.PrgEnv_compiler: tty.die('Must supply PrgEnv_compiler with PrgEnv') -# output = _shell('module avail %s' % cls.PrgEnv_compiler) modulecmd = which('modulecmd') - modulecmd + modulecmd.add_default_arg('python') + output = modulecmd('avail', return_oe=True) matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) - loaded_modules = os.environ["LOADEDMODULES"].split(":") +# loaded_modules = os.environ["LOADEDMODULES"].split(":") #output = _shell('module avail %s' % cls.PrgEnv_compiler) - for module in loaded_modules: - match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) +# for module in loaded_modules: +# match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) for name, version in matches: v = version + '-craype' comp = cls(spack.spec.CompilerSpec(name + '@' + v), - 'cc', 'CC', 'ftn', 'ftn', name +'/' + v) + ['cc', 'CC', 'ftn'], [cls.PrgEnv, name +'/' + v]) compilers.append(comp) return compilers -def _cur_prgenv(): - out, err = subprocess.Popen( - ['module list'], shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - matches = re.findall(r'(PrgEnv-[^/]*)/', err) - return matches[0] - - -def _module_shell(module, *args): - cmd = 'module swap %s %s;' % (_cur_prgenv(), module) - cmd += 'module load %s;' % compiler - cmd += 'module unload cray-libsci;' - -# + -# 'module load craype-network-gemini;' + -# 'module load %s;' % module + -# 'module swap gcc/4.6.1;' + -# 'module load eswrap; ' + -# 'module load craype-mc12; ' + -# 'module load cray-shmem; ' + -# 'module load cray-mpich; ') - cmd += ' '.join(args) - out, err = subprocess.Popen([cmd + ' '.join(args)], shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - return out - - -def _shell(*args): - return subprocess.Popen([' '.join(args)], shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1] - - - - def __repr__(self): """Return a string represntation of the compiler toolchain.""" return self.__str__() diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 8c4cb38926..6fc54daa61 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -197,18 +197,27 @@ def compilers_for_spec(compiler_spec): else: compiler_paths.append(None) - return cls(cspec, *compiler_paths) + for m in _optional_instance_vars: + if m not in items: + items[m] = None + mods = items[m] + + return cls(cspec, compiler_paths, mods) matches = find(compiler_spec) return [get_compiler(cspec) for cspec in matches] @_auto_compiler_spec -def compiler_for_spec(compiler_spec): +def compiler_for_spec(compiler_spec, target): """Get the compiler that satisfies compiler_spec. compiler_spec must be concrete.""" assert(compiler_spec.concrete) compilers = compilers_for_spec(compiler_spec) + if target.compiler_strategy == "PATH": + filter(lambda c: c.modules is None, compilers) + elif target.compiler_strategy == "MODULES": + filter(lambda c: c.modules is not None, compilers) assert(len(compilers) == 1) return compilers[0] diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 61606d0590..b6a98f3ee6 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -592,7 +592,7 @@ class Package(object): """Get the spack.compiler.Compiler object used to build this package.""" if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") - return spack.compilers.compiler_for_spec(self.spec.compiler) + return spack.compilers.compiler_for_spec(self.spec.compiler, self.spec.architecture) def url_version(self, version): -- cgit v1.2.3-60-g2f50 From 271a839957ace370ce332f8edd3321e60127a9e6 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 14:22:07 -0800 Subject: First possibly working version of the crayport. Not sufficiently tested at all. --- lib/spack/spack/architecture.py | 41 +++++++++++++++++++--------------- lib/spack/spack/architectures/linux.py | 13 ++++++----- lib/spack/spack/build_environment.py | 6 ++--- lib/spack/spack/compiler.py | 8 +++++-- lib/spack/spack/compilers/__init__.py | 6 ++--- lib/spack/spack/concretize.py | 28 +---------------------- lib/spack/spack/config.py | 5 +++-- 7 files changed, 46 insertions(+), 61 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 64416e8b4c..b1da8ae4a2 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -61,18 +61,20 @@ class Target(object): def set_architecture(self, architecture): # Target should get the architecture class. self.architecture = architecture - + @property def compiler_strategy(self): - if default_strategy: - return default_strategy - elif self.module_name: # If there is a module_name given then use MODULES + if self.module_name: # If there is a module_name given then use MODULES return "MODULES" else: return "PATH" + def __str__(self): + return self.name + + class Architecture(object): - """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it + """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned """ @@ -110,12 +112,15 @@ class Architecture(object): @classmethod def detect(self): - """ Subclass is responsible for implementing this method. + """ Subclass is responsible for implementing this method. Returns True if the architecture detects if it is the current architecture and False if it's not. """ raise NotImplementedError() - + + def __repr__(self): + return self.__str__ + def __str__(self): return self.name @@ -123,7 +128,7 @@ class Architecture(object): def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): - return None + return None elif hasattr(spack.sys_type, "__call__"): return spack.sys_type() #If in __init__.py there is a sys_type() then call that else: @@ -147,7 +152,7 @@ def get_mac_sys_type(): def get_sys_type_from_uname(): - """ Returns a sys_type from the uname argument + """ Returns a sys_type from the uname argument Front-end config """ try: @@ -158,8 +163,8 @@ def get_sys_type_from_uname(): return None def get_sys_type_from_config_file(): - - spack_home_dir = os.environ["HOME"] + "/.spack" + + spack_home_dir = os.environ["HOME"] + "/.spack" yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') try: config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() @@ -167,7 +172,7 @@ def get_sys_type_from_config_file(): front = arch['front'] back = arch['back'] return Architecture(front,back) - + except: print "No architecture.yaml config file found" return None @@ -182,7 +187,7 @@ def all_architectures(): mod = imp.load_source(mod_name, path) class_name = mod_to_class(name) if not hasattr(mod, class_name): - tty.die('No class %s defined in %s' % (class_name, mod_name)) + tty.die('No class %s defined in %s' % (class_name, mod_name)) cls = getattr(mod, class_name) if not inspect.isclass(cls): tty.die('%s.%s is not a class' % (mod_name, class_name)) @@ -197,15 +202,15 @@ def sys_type(): 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 2. UNAME 3. GLOBALS - 4. MAC OSX + 4. MAC OSX Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. - If there is no yaml present then it should move on to the next function and stop immediately once it gets a + If there is no yaml present then it should move on to the next function and stop immediately once it gets a arch name """ # Try to create an architecture object using the config file FIRST - architecture_list = all_architectures() - architecture_list.sort(key = lambda a: a.priority) - + architecture_list = all_architectures() + architecture_list.sort(key = lambda a: a.priority) + for arch in architecture_list: if arch.detect(): return arch() diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py index 7238575660..d7d9a994fa 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/architectures/linux.py @@ -1,14 +1,15 @@ -import subprocess -from spack.architecture import Architecture +import subprocess +from spack.architecture import Architecture, Target class Linux(Architecture): - priority = 60 - front_end = "x86_64" - back_end = "x86_64" - default = "x86_64" + priority = 60 + front_end = 'linux' + back_end = 'linux' + default = 'linux' def __init__(self): super(Linux, self).__init__('linux') + self.add_target('linux', Target('linux')) @classmethod def detect(self): diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 191e858735..9e3be433fb 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -88,13 +88,13 @@ class MakeExecutable(Executable): def load_module(mod): """Takes a module name and removes modules until it is possible to - load that module. It then loads the provided module. Depends on the + load that module. It then loads the provided module. Depends on the modulecmd implementation of modules used in cray and lmod. """ #Create an executable of the module command that will output python code modulecmd = which('modulecmd') modulecmd.add_default_arg('python') - + # Read the module and remove any conflicting modules # We do this without checking that they are already installed # for ease of programming because unloading a module that is not @@ -102,7 +102,7 @@ def load_module(mod): text = modulecmd('show', mod, return_oe=True).split() for i, word in enumerate(text): if word == 'conflict': - exec(compile(modulecmd('unload', text[i+1], return_oe=True), '', 'exec')) + exec(compile(modulecmd('unload', text[i+1], return_oe=True), '', 'exec')) # Load the module now that there are no conflicts load = modulecmd('load', mod, return_oe=True) exec(compile(load, '', 'exec')) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 4be573fb12..41ff89a151 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -307,8 +307,12 @@ class Compiler(object): def __str__(self): """Return a string represntation of the compiler toolchain.""" - return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) + if self.modules: + return "%s(%s)" % ( + self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc, self.modules)))) + else: + return "%s(%s)" % ( + self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) class CompilerAccessError(spack.error.SpackError): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 6fc54daa61..23ff6cced4 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -44,15 +44,15 @@ from spack.util.environment import get_path _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] -_optional_instance_vars = ['module'] +_optional_instance_vars = ['modules'] _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] def _auto_compiler_spec(function): - def converter(cspec_like): + def converter(cspec_like, *args): if not isinstance(cspec_like, spack.spec.CompilerSpec): cspec_like = spack.spec.CompilerSpec(cspec_like) - return function(cspec_like) + return function(cspec_like, *args) return converter diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index c5041d67be..4b4cb69033 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -97,32 +97,6 @@ class DefaultConcretizer(object): def concretize_architecture(self, spec): - """If the spec already had an architecture, return. Otherwise if - the root of the DAG has an architecture, then use that. - Otherwise take the system's default architecture. - - Intuition: Architectures won't be set a lot, and generally you - want the host system's architecture. When architectures are - mised in a spec, it is likely because the tool requries a - cross-compiled component, e.g. for tools that run on BlueGene - or Cray machines. These constraints will likely come directly - from packages, so require the user to be explicit if they want - to mess with the architecture, and revert to the default when - they're not explicit. - """ - if spec.architecture is not None: - return False - - if spec.root.architecture: - spec.architecture = spec.root.architecture - else: - spec.architecture = spack.architecture.sys_type() - - assert(spec.architecture is not None) - return True # changed - - - def new_concretize_architecture(self, spec): """If the spec already has an architecture and it is a an architecture type, return. Otherwise, if it has an architecture that is a string type, generate an architecture based on that type. If it has no architecture and the root of the @@ -146,7 +120,7 @@ class DefaultConcretizer(object): else: arch = spack.architecture.sys_type() spec.architecture = arch.target('default') - + return True #changed diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3e91958c2c..bc655d6051 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -199,6 +199,7 @@ def get_config(category_name): category.result_dict = _merge_dicts(category.result_dict, result) else: category.result_dict = result + return category.result_dict @@ -208,7 +209,7 @@ def get_compilers_config(arch=None): configuration""" global _compiler_by_arch if not arch: - arch = spack.architecture.sys_type() + arch = str(spack.architecture.sys_type()) if arch in _compiler_by_arch: return _compiler_by_arch[arch] @@ -305,7 +306,7 @@ def add_to_compiler_config(addition_dict, scope=None, arch=None): """Add compilerss to the configuration files""" if not arch: arch = spack.architecture.sys_type() - add_to_config('compilers', { arch : addition_dict }, scope) + add_to_config('compilers', { str(arch) : addition_dict }, scope) clear_config_caches() -- cgit v1.2.3-60-g2f50 From 8a13d344a898195c6eb33d8bea151487d0d209bd Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 15:07:57 -0800 Subject: bug fix in spec.py --- lib/spack/spack/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index e1fbb84423..3ba9c139a6 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1998,4 +1998,4 @@ class UnsatisfiableDependencySpecError(UnsatisfiableSpecError): class SpackYAMLError(spack.error.SpackError): def __init__(self, msg, yaml_error): - super(SpackError, self).__init__(msg, str(yaml_error)) + super(SpackYAMLError, self).__init__(msg, str(yaml_error)) -- cgit v1.2.3-60-g2f50 From 3067705c109aef062823e62f0bb8a9d977a10a35 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 16:16:47 -0800 Subject: made a to_dict for targets. Also a from_dict for sanity --- lib/spack/spack/architecture.py | 19 +++++++++++++++++++ lib/spack/spack/spec.py | 4 ++-- 2 files changed, 21 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index b1da8ae4a2..642a589f35 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -69,6 +69,25 @@ class Target(object): else: return "PATH" + def to_dict(self): + print "to_dict" + d = {} + d['name'] = self.name + d['module_name'] = self.module_name + if self.architecture: + d['architecture'] = self.architecture + return d + + @staticmethod + def from_dict(d): + print "from_dict" + target = Target.__new__(Target) + target.name = d['name'] + target.module_name = d['module_name'] + if 'architecture' in d: + target.architecture = d['architecture'] + return target + def __str__(self): return self.name diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 3ba9c139a6..395faecde6 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -651,7 +651,7 @@ class Spec(object): d = { 'variants' : dict( (name,v.enabled) for name, v in self.variants.items()), - 'arch' : self.architecture, + 'arch' : self.architecture.to_dict(), 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } @@ -680,7 +680,7 @@ class Spec(object): spec = Spec(name) spec.versions = VersionList.from_dict(node) - spec.architecture = node['arch'] + spec.architecture = spack.architecture.Target.from_dict(node['arch']) if node['compiler'] is None: spec.compiler = None -- cgit v1.2.3-60-g2f50 From 185f40eb8bbc19ecd7e2634844c97067388d0422 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 16:33:22 -0800 Subject: fixed my git problems --- lib/spack/spack/compiler.py | 2 +- lib/spack/spack/compilers/gcc.py | 2 +- lib/spack/spack/compilers/intel.py | 2 +- lib/spack/spack/compilers/pgi.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 41ff89a151..abf7223117 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -291,7 +291,7 @@ class Compiler(object): # match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) for name, version in matches: - v = version + '-craype' + v = version comp = cls(spack.spec.CompilerSpec(name + '@' + v), ['cc', 'CC', 'ftn'], [cls.PrgEnv, name +'/' + v]) diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index ff0b8889a8..a9a4f729eb 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -42,7 +42,7 @@ class Gcc(Compiler): # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. suffixes = [r'-mp-\d\.\d'] - PrgEnv = 'gnu' + PrgEnv = 'PrgEnv-gnu' PrgEnv_compiler = 'gcc' @property diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 7c485fe69d..4096e32a08 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -37,7 +37,7 @@ class Intel(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['ifort'] - PrgEnv = 'intel' + PrgEnv = 'PrgEnv-intel' PrgEnv_compiler = 'intel' @property diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 8f1ed28825..8010c1d04d 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -37,7 +37,7 @@ class Pgi(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['pgf95', 'pgf90'] - PrgEnv = 'pgi' + PrgEnv = 'PrgEnv-pgi' PrgEnv_compiler = 'pgi' @classmethod -- cgit v1.2.3-60-g2f50 From eb2c08315905e0484dbd4d866396563487d7d249 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 17:29:47 -0800 Subject: Fixed the to_dict from_dict for targets on cray. First version to build/find properly on Cray --- lib/spack/spack/architecture.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 642a589f35..349a7b33db 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -70,12 +70,11 @@ class Target(object): return "PATH" def to_dict(self): - print "to_dict" d = {} d['name'] = self.name d['module_name'] = self.module_name - if self.architecture: - d['architecture'] = self.architecture +# if self.architecture: +# d['architecture'] = self.architecture return d @staticmethod @@ -84,10 +83,13 @@ class Target(object): target = Target.__new__(Target) target.name = d['name'] target.module_name = d['module_name'] - if 'architecture' in d: - target.architecture = d['architecture'] +# if 'architecture' in d: +# target.architecture = d['architecture'] return target + def __repr__(self): + return self.__str__() + def __str__(self): return self.name @@ -138,7 +140,7 @@ class Architecture(object): raise NotImplementedError() def __repr__(self): - return self.__str__ + return self.__str__() def __str__(self): return self.name -- cgit v1.2.3-60-g2f50 From e78b8c16be5334aa4aea29313a77da8923319a8f Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 12 Nov 2015 13:27:10 -0800 Subject: used key_comparator decorator from util.lang to compare targets. Fixes find bug --- lib/spack/spack/architecture.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 349a7b33db..f4b8585c2f 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -27,7 +27,7 @@ import imp import platform as py_platform import inspect -from llnl.util.lang import memoized, list_modules +from llnl.util.lang import memoized, list_modules, key_ordering from llnl.util.filesystem import join_path import llnl.util.tty as tty @@ -47,6 +47,7 @@ class NoSysTypeError(serr.SpackError): super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") +@key_ordering class Target(object): """ Target is the processor of the host machine. The host machine may have different front-end and back-end targets, especially if it is a Cray machine. The target will have a name and @@ -79,7 +80,6 @@ class Target(object): @staticmethod def from_dict(d): - print "from_dict" target = Target.__new__(Target) target.name = d['name'] target.module_name = d['module_name'] @@ -87,6 +87,10 @@ class Target(object): # target.architecture = d['architecture'] return target + + def _cmp_key(self): + return (self.name, self.module_name) + def __repr__(self): return self.__str__() -- cgit v1.2.3-60-g2f50 From 7c89f9d18c57e3e0abe40cf608f63015ea9b68f3 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 12 Nov 2015 14:31:41 -0800 Subject: Added the basic ouline of a cray compiler class in lib/spack/spack/compilers/cray.py --- lib/spack/spack/compilers/cray.py | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 lib/spack/spack/compilers/cray.py (limited to 'lib') diff --git a/lib/spack/spack/compilers/cray.py b/lib/spack/spack/compilers/cray.py new file mode 100644 index 0000000000..c2f3ecd6f4 --- /dev/null +++ b/lib/spack/spack/compilers/cray.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import llnl.util.tty as tty +from spack.compiler import * +from spack.version import ver + +class Cray(Compiler): + # Subclasses use possible names of C compiler + cc_names = ['cc'] + + # Subclasses use possible names of C++ compiler + cxx_names = ['CC'] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['ftn'] + + # Subclasses use possible names of Fortran 90 compiler + fc_names = ['ftn'] + + # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. + suffixes = [r'-mp-\d\.\d'] + + PrgEnv = 'PrgEnv-cray' + PrgEnv_compiler = 'craype' + + @property + def cxx11_flag(self): + return "-hstd=c++11" -- cgit v1.2.3-60-g2f50 From a5ba69d68d1472f22454a93d045632b1340e2b04 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 Nov 2015 11:46:13 -0800 Subject: Added a _cmp_key for the architecture class --- lib/spack/spack/architecture.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index f4b8585c2f..786ef34071 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -97,7 +97,7 @@ class Target(object): def __str__(self): return self.name - +@key_ordering class Architecture(object): """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned @@ -148,7 +148,9 @@ class Architecture(object): def __str__(self): return self.name - + + def _cmp_key(self): + return (self.name, (_cmp_key(t) for t in self.targets.values())) def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" -- cgit v1.2.3-60-g2f50 From 9458f7c7d0ffebfbe33345c1a6b4b07c39e41e5f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 Nov 2015 11:47:36 -0800 Subject: Got rid of my old method of finding in LOADEDMODULES --- lib/spack/spack/compiler.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index abf7223117..5d87588245 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -283,13 +283,9 @@ class Compiler(object): modulecmd = which('modulecmd') modulecmd.add_default_arg('python') output = modulecmd('avail', return_oe=True) - matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) - -# loaded_modules = os.environ["LOADEDMODULES"].split(":") - #output = _shell('module avail %s' % cls.PrgEnv_compiler) -# for module in loaded_modules: -# match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) - + matches = re.findall(r'(%s)/(\d+[\.\d]+)' % cls.PrgEnv_compiler, output) +# It's finding a weird third attribute + print matches for name, version in matches: v = version comp = cls(spack.spec.CompilerSpec(name + '@' + v), -- cgit v1.2.3-60-g2f50 From 72fa3c04922b3630c15c69ff51022d88a8bf1de9 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 Nov 2015 11:48:15 -0800 Subject: Added craype support --- lib/spack/spack/compilers/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 23ff6cced4..6731318f43 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -46,7 +46,7 @@ _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] _optional_instance_vars = ['modules'] -_default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] +_default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc','craype'] def _auto_compiler_spec(function): def converter(cspec_like, *args): @@ -180,6 +180,7 @@ def compilers_for_spec(compiler_spec): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ + # cray issue might be located here config = _get_config() def get_compiler(cspec): -- cgit v1.2.3-60-g2f50 From bfd05d3d2702bb2a75fe310fb04eb78ac2526824 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 Nov 2015 11:48:49 -0800 Subject: Changed name of file and class to craype to avoid spack freaking out about versions --- lib/spack/spack/compilers/craype.py | 57 +++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 lib/spack/spack/compilers/craype.py (limited to 'lib') diff --git a/lib/spack/spack/compilers/craype.py b/lib/spack/spack/compilers/craype.py new file mode 100644 index 0000000000..56d6d8d36c --- /dev/null +++ b/lib/spack/spack/compilers/craype.py @@ -0,0 +1,57 @@ +##############################################################################} +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import llnl.util.tty as tty + +from spack.build_environment import load_module +from spack.compiler import * +from spack.version import ver + +class Craype(Compiler): + # Subclasses use possible names of C compiler + cc_names = ['cc'] + + # Subclasses use possible names of C++ compiler + cxx_names = ['CC'] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['ftn'] + + # Subclasses use possible names of Fortran 90 compiler + fc_names = ['ftn'] + + # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. + suffixes = [r'-mp-\d\.\d'] + + PrgEnv = 'PrgEnv-cray' + PrgEnv_compiler = 'craype' + + @property + def cxx11_flag(self): + return "-hstd=c++11" + + @classmethod + def default_version(cls, comp): + return get_compiler_version(comp, r'([Vv]ersion).*(\d+(\.\d+)+)') + -- cgit v1.2.3-60-g2f50 From fceb5a75b06b639a6f6d7edf33c3b727e71a491b Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 Nov 2015 11:53:38 -0800 Subject: Adding new files to compilers --- lib/spack/spack/compilers/cray.py | 50 --------------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 lib/spack/spack/compilers/cray.py (limited to 'lib') diff --git a/lib/spack/spack/compilers/cray.py b/lib/spack/spack/compilers/cray.py deleted file mode 100644 index c2f3ecd6f4..0000000000 --- a/lib/spack/spack/compilers/cray.py +++ /dev/null @@ -1,50 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import llnl.util.tty as tty -from spack.compiler import * -from spack.version import ver - -class Cray(Compiler): - # Subclasses use possible names of C compiler - cc_names = ['cc'] - - # Subclasses use possible names of C++ compiler - cxx_names = ['CC'] - - # Subclasses use possible names of Fortran 77 compiler - f77_names = ['ftn'] - - # Subclasses use possible names of Fortran 90 compiler - fc_names = ['ftn'] - - # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. - suffixes = [r'-mp-\d\.\d'] - - PrgEnv = 'PrgEnv-cray' - PrgEnv_compiler = 'craype' - - @property - def cxx11_flag(self): - return "-hstd=c++11" -- cgit v1.2.3-60-g2f50 From 89fbe4fdfa01b2461a5355a8922e6083f528e409 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 1 Dec 2015 11:55:38 -0800 Subject: Changed add_target to include variables rather than hard-coded strings --- lib/spack/spack/architectures/bgq.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architectures/bgq.py b/lib/spack/spack/architectures/bgq.py index d3d4446e09..85487626f1 100644 --- a/lib/spack/spack/architectures/bgq.py +++ b/lib/spack/spack/architectures/bgq.py @@ -10,8 +10,8 @@ class Bgq(Architecture): def __init__(self): super(Bgq, self).__init__('cray') - self.add_target('power7', Target('power7')) - self.add_target('powerpc', Target('powerpc')) + self.add_target(self.front_end, Target(self.front_end)) + self.add_target(self.back_end, Target(self.back_end)) @classmethod def detect(self): -- cgit v1.2.3-60-g2f50 From 58f2b39bc844d1c8beffd6449a9047c5ec5522e5 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 1 Dec 2015 11:57:17 -0800 Subject: Changed default to CPU_TARGET env var. Helps deal with target differences between cori and edison --- lib/spack/spack/architectures/cray.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index 640a3933e0..e4a3617dc9 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -6,15 +6,23 @@ class Cray(Architecture): priority = 20 front_end = 'sandybridge' back_end = 'ivybridge' - default = 'ivybridge' - + default = os.environ['CRAY_CPU_TARGET'] + #default = 'ivybridge' + def __init__(self): + ''' Since cori doesn't have ivybridge as a front end it's better + if we use CRAY_CPU_TARGET as the default. This will ensure + that if we're on a XC-40 or XC-30 then we can detect the target + ''' super(Cray, self).__init__('cray') + # Back End compiler needs the proper target module loaded. - self.add_target('ivybridge', Target('ivybridge','craype-ivybridge')) + self.add_target(self.front_end, Target(self.front_end,'craype-' + self.front_end)) + self.add_target(self.default, Target(self.default,'craype-' + self.default)) # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target('sandybridge', Target('sandybridge')) + self.add_target(self.back_end, Target('craype-' + self.back_end)) + @classmethod def detect(self): -- cgit v1.2.3-60-g2f50 From d7fdb8e015d5e6c5b0e06608166307f50f2f046f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 1 Dec 2015 11:57:54 -0800 Subject: Got rid of hard coded strings for adding default --- lib/spack/spack/architectures/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py index d7d9a994fa..6c454f7a40 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/architectures/linux.py @@ -9,7 +9,7 @@ class Linux(Architecture): def __init__(self): super(Linux, self).__init__('linux') - self.add_target('linux', Target('linux')) + self.add_target(self.default, Target(self.default)) @classmethod def detect(self): -- cgit v1.2.3-60-g2f50 From 028cca16e66c4a7dd867ef6be4cf38db9daa6e51 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Dec 2015 10:15:33 -0800 Subject: Got rid of the unusued default strategy=None --- lib/spack/spack/architecture.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 786ef34071..a53222d36f 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -54,7 +54,6 @@ class Target(object): also the module_name (e.g craype-compiler). Targets will also recognize which architecture they came from using the set_architecture method. Targets will have compiler finding strategies """ - default_strategy = None # Can probably add a compiler path here def __init__(self,name, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 @@ -99,7 +98,8 @@ class Target(object): @key_ordering class Architecture(object): - """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it + """ Abstract class that each type of Architecture will subclass. + Will return a instance of it once it is returned """ -- cgit v1.2.3-60-g2f50 From fee88d289d00f6886d0cd336aab4cabbb5d45acc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 11 Dec 2015 12:03:13 -0800 Subject: Rewrote docstrings for sys_type() and got rid of unused functions --- lib/spack/spack/architecture.py | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index a53222d36f..beebeba4bf 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -189,22 +189,6 @@ def get_sys_type_from_uname(): except: return None -def get_sys_type_from_config_file(): - - spack_home_dir = os.environ["HOME"] + "/.spack" - yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') - try: - config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() - arch = config_dict['architecture'] - front = arch['front'] - back = arch['back'] - return Architecture(front,back) - - except: - print "No architecture.yaml config file found" - return None - - @memoized def all_architectures(): modules = [] @@ -225,14 +209,10 @@ def all_architectures(): @memoized def sys_type(): - """Priority of gathering sys-type. - 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 - 2. UNAME - 3. GLOBALS - 4. MAC OSX - Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. - If there is no yaml present then it should move on to the next function and stop immediately once it gets a - arch name + """ Gather a list of all available subclasses of architectures. + Sorts the list according to their priority looking. Priority is + an arbitrarily set number. Detects arch either using uname or + a file path (/opt/cray...) """ # Try to create an architecture object using the config file FIRST architecture_list = all_architectures() -- cgit v1.2.3-60-g2f50 From 31df2dd9dd713e75d161eb5a0187536886363469 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 11 Dec 2015 12:04:47 -0800 Subject: Fixed targets ti be x86_64, important note: have not tested this on linux machine --- lib/spack/spack/architectures/linux.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py index 6c454f7a40..052b10c019 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/architectures/linux.py @@ -3,9 +3,9 @@ from spack.architecture import Architecture, Target class Linux(Architecture): priority = 60 - front_end = 'linux' - back_end = 'linux' - default = 'linux' + front_end = 'x86_64' + back_end = 'x86_64' + default = 'x86_64' def __init__(self): super(Linux, self).__init__('linux') @@ -15,4 +15,4 @@ class Linux(Architecture): def detect(self): arch = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) arch, _ = arch.communicate() - return 'x86_64' in arch.strip() + return 'linux' in arch.strip().lower() -- cgit v1.2.3-60-g2f50 From b5216f6ec8fc5b1cd7061d8610b51ceec5f162e8 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 11 Dec 2015 12:07:32 -0800 Subject: Commented out cxxflag. Not 100% sure it's needed --- lib/spack/spack/compilers/craype.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/craype.py b/lib/spack/spack/compilers/craype.py index 56d6d8d36c..e8ae284f5b 100644 --- a/lib/spack/spack/compilers/craype.py +++ b/lib/spack/spack/compilers/craype.py @@ -24,9 +24,9 @@ ############################################################################## import llnl.util.tty as tty -from spack.build_environment import load_module +#from spack.build_environment import load_module from spack.compiler import * -from spack.version import ver +#from spack.version import ver class Craype(Compiler): # Subclasses use possible names of C compiler @@ -47,9 +47,9 @@ class Craype(Compiler): PrgEnv = 'PrgEnv-cray' PrgEnv_compiler = 'craype' - @property - def cxx11_flag(self): - return "-hstd=c++11" +# @property +# def cxx11_flag(self): +# return "-hstd=c++11" @classmethod def default_version(cls, comp): -- cgit v1.2.3-60-g2f50 From 217a2d9ea5bd6184d36ea0c32429c979aa088fbc Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 17 Dec 2015 15:26:27 -0800 Subject: fixed yaml error --- lib/spack/spack/architecture.py | 2 ++ lib/spack/spack/spec.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index beebeba4bf..3d1de39354 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -79,6 +79,8 @@ class Target(object): @staticmethod def from_dict(d): + if d is None: + return None target = Target.__new__(Target) target.name = d['name'] target.module_name = d['module_name'] diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 395faecde6..ff58091656 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -651,10 +651,13 @@ class Spec(object): d = { 'variants' : dict( (name,v.enabled) for name, v in self.variants.items()), - 'arch' : self.architecture.to_dict(), 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } + if self.architecture: + d['arch'] = self.architecture.to_dict() + else: + d['arch'] = None if self.compiler: d.update(self.compiler.to_dict()) else: -- cgit v1.2.3-60-g2f50 From 53808f254efc6919ec66043c709039c78c34e11a Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 4 Jan 2016 12:36:48 -0800 Subject: Support for cray external dependencies implemented in modules --- lib/spack/spack/build_environment.py | 54 +++++++++++++++++++++++++++++++++++- lib/spack/spack/concretize.py | 6 +++- lib/spack/spack/config.py | 13 +++++++-- lib/spack/spack/spec.py | 9 ++++-- 4 files changed, 75 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 9e3be433fb..781039e073 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -32,10 +32,11 @@ import sys import shutil import multiprocessing import platform +import re + from llnl.util.filesystem import * import spack -import spack.compilers as compilers from spack.util.executable import Executable, which from spack.util.environment import * @@ -108,6 +109,46 @@ def load_module(mod): exec(compile(load, '', 'exec')) +def get_path_from_module(mod): + """Inspects a TCL module for entries that indicate the absolute path + at which the library supported by said module can be found. + """ + # Create a modulecmd executable + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module + text = modulecmd('show', mod, return_oe=True).split('\n') + + # If it lists its package directory, return that + for line in text: + if line.find(mod.upper()+'_DIR') >= 0: + words = line.split() + return words[2] + + # If it lists a -rpath instruction, use that + for line in text: + rpath = line.find('-rpath/') + if rpath >= 0: + return line[rpath+6:line.find('/lib')] + + # If it lists a -L instruction, use that + for line in text: + L = line.find('-L/') + if L >= 0: + return line[L+2:line.find('/lib')] + + # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that + for line in text: + if line.find('LD_LIBRARY_PATH') >= 0: + words = line.split() + path = words[2] + return path[:path.find('/lib')] + + # Unable to find module path + return None + + def set_compiler_environment_variables(pkg): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -251,6 +292,17 @@ def set_module_variables_for_package(pkg): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" + + # First load all modules for external packages and update the external + # packages' paths to reflect what is found in the modules so that we can + # rpath through the modules when possible, but if not possible they are + # already loaded. + for spec in pkg.spec.traverse(root=False): + if spec.external_module: + load_module(spec.external_module) + spec.external = get_path_from_module(spec.external_module) + + # Construct rpaths from the paths of each dep rpaths = [pkg.prefix.lib, pkg.prefix.lib64] rpaths.extend(d.prefix.lib for d in pkg.spec.traverse(root=False) if os.path.isdir(d.prefix.lib)) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 13e9b477dc..7c7453353a 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -97,7 +97,7 @@ class DefaultConcretizer(object): externals = spec_externals(pkg) buildable = not is_spec_nobuild(pkg) if buildable: - result.append((pkg, None)) + result.append((pkg, None, None)) if externals: sorted_externals = sorted(externals, cmp=lambda a,b: a[0].__cmp__(b[0])) for external in sorted_externals: @@ -131,6 +131,7 @@ class DefaultConcretizer(object): if not candidate: #No ABI matches. Pick the top choice based on the orignal preferences. candidate = candidates[0] + external_module = candidate[2] external = candidate[1] candidate_spec = candidate[0] @@ -144,6 +145,9 @@ class DefaultConcretizer(object): if not spec.external and external: spec.external = external changed = True + if not spec.external_module and external_module: + spec.external_module = external_module + changed = True #If we're external then trim the dependencies if external and spec.dependencies: diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 9919dcc045..d66ffb338e 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -103,6 +103,8 @@ import llnl.util.tty as tty from llnl.util.filesystem import mkdirp import copy +from spack.build_environment import get_path_from_module + _config_sections = {} class _ConfigCategory: name = None @@ -255,7 +257,8 @@ def get_packages_config(): package_name = spack.spec.Spec(p.keys()[0]).name if package_name not in indexed_packages: indexed_packages[package_name] = [] - indexed_packages[package_name].append({ spack.spec.Spec(key) : val for key, val in p.iteritems() }) + pkg_dict = dict([ (spack.spec.Spec(key), val) for key, val in p.iteritems()]) + indexed_packages[package_name].append( pkg_dict ) return indexed_packages @@ -286,9 +289,13 @@ def spec_externals(spec): if not pkg.satisfies(spec): continue path = conf.get('path', None) + module = conf.get('module', None) if not path: - continue - spec_locations.append( (pkg, path) ) + if not module: + continue + else: + path = get_path_from_module(module) + spec_locations.append( (pkg, path, module) ) return spec_locations diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 4289fe19cb..f496011d62 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -420,6 +420,7 @@ class Spec(object): self._normal = kwargs.get('normal', False) self._concrete = kwargs.get('concrete', False) self.external = None + self.external_module = None # This allows users to construct a spec DAG with literals. # Note that given two specs a and b, Spec(a) copies a, but @@ -1352,8 +1353,9 @@ class Spec(object): changed = (self.name != other.name and self.versions != other.versions and \ self.architecture != other.architecture and self.compiler != other.compiler and \ self.variants != other.variants and self._normal != other._normal and \ - self.concrete != other.concrete and self.external != other.external) - + self.concrete != other.concrete and self.external != other.external and \ + self.external_module != other.external_module) + # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() @@ -1365,6 +1367,7 @@ class Spec(object): self.variants = other.variants.copy() self.variants.spec = self self.external = other.external + self.external_module = other.external_module # If we copy dependencies, preserve DAG structure in the new spec if kwargs.get('deps', True): @@ -1383,6 +1386,7 @@ class Spec(object): self._normal = other._normal self._concrete = other._concrete self.external = other.external + self.external_module = other.external_module return changed @@ -1809,6 +1813,7 @@ class SpecParser(spack.parse.Parser): spec.architecture = None spec.compiler = None spec.external = None + spec.external_module = None spec.dependents = DependencyMap() spec.dependencies = DependencyMap() -- cgit v1.2.3-60-g2f50 From 751208cedf3506ebcd5119fd0d4d904bc28392e8 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 4 Jan 2016 16:35:06 -0800 Subject: pick last compiler instead of first, will be overwritten by more advanced preferences as mentioned in the email group --- lib/spack/spack/concretize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 7c7453353a..48d05f7f8d 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -287,7 +287,7 @@ class DefaultConcretizer(object): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - spec.compiler = matches[0].copy() + spec.compiler = matches[len(matches)-1].copy() assert(spec.compiler.concrete) return True # things changed. -- cgit v1.2.3-60-g2f50 From d2c2c46541d54e4dcde34e6199ba5dafa949bef1 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 11:12:14 -0800 Subject: bug hunting --- lib/spack/spack/compilers/__init__.py | 4 ++-- lib/spack/spack/config.py | 2 +- lib/spack/spack/spec.py | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 6731318f43..87106282cf 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -216,9 +216,9 @@ def compiler_for_spec(compiler_spec, target): assert(compiler_spec.concrete) compilers = compilers_for_spec(compiler_spec) if target.compiler_strategy == "PATH": - filter(lambda c: c.modules is None, compilers) + compilers = [c for c in compilers if c.modules is None] elif target.compiler_strategy == "MODULES": - filter(lambda c: c.modules is not None, compilers) + compilers = [c for c in compilers if c.modules is not None] assert(len(compilers) == 1) return compilers[0] diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index d66ffb338e..6e11cfa475 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -277,7 +277,7 @@ def is_spec_nobuild(spec): def spec_externals(spec): - """Return a list of spec, directory pairs for each external location for spec""" + """Return a list of spec, directory, module triples for each external location for spec""" allpkgs = get_packages_config() name = spec.name spec_locations = [] diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index f496011d62..126ffc4f95 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -771,8 +771,9 @@ class Spec(object): """Replace this virtual spec with a concrete spec.""" assert(self.virtual) for name, dependent in self.dependents.items(): - del dependent.dependencies[self.name] - dependent._add_dependency(concrete) + if not dependent.external: + del dependent.dependencies[self.name] + dependent._add_dependency(concrete) def _expand_virtual_packages(self): -- cgit v1.2.3-60-g2f50 From 9848ad32fdadaf7decfb19aba03829708d8b6c85 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 13:39:53 -0800 Subject: fixed errors caused by crayport on linux and added a darwin architecture --- lib/spack/spack/architecture.py | 6 ++++-- lib/spack/spack/architectures/cray.py | 21 ++++++++++++--------- lib/spack/spack/architectures/darwin.py | 18 ++++++++++++++++++ lib/spack/spack/architectures/linux.py | 4 ++-- lib/spack/spack/test/__init__.py | 5 ++--- lib/spack/spack/test/architecture.py | 32 ++++++++++++++++++++++++-------- 6 files changed, 62 insertions(+), 24 deletions(-) create mode 100644 lib/spack/spack/architectures/darwin.py (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3d1de39354..1f608bd187 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -96,11 +96,13 @@ class Target(object): return self.__str__() def __str__(self): + if self.module_name: + return self.name + ' module: ' + self.module_name return self.name @key_ordering class Architecture(object): - """ Abstract class that each type of Architecture will subclass. + """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned """ @@ -150,7 +152,7 @@ class Architecture(object): def __str__(self): return self.name - + def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values())) diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index e4a3617dc9..47ede30145 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -4,27 +4,30 @@ from spack.architecture import Architecture, Target class Cray(Architecture): priority = 20 - front_end = 'sandybridge' + front_end = 'sandybridge' back_end = 'ivybridge' - default = os.environ['CRAY_CPU_TARGET'] - #default = 'ivybridge' - + default = 'ivybridge' + def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better if we use CRAY_CPU_TARGET as the default. This will ensure that if we're on a XC-40 or XC-30 then we can detect the target ''' super(Cray, self).__init__('cray') - + + # Handle the default here so we can check for a key error + if 'CRAY_CPU_TARGET' in os.environ: + default = os.environ['CRAY_CPU_TARGET'] + # Back End compiler needs the proper target module loaded. - self.add_target(self.front_end, Target(self.front_end,'craype-' + self.front_end)) + self.add_target(self.back_end, Target(self.front_end,'craype-'+ self.back_end)) self.add_target(self.default, Target(self.default,'craype-' + self.default)) # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target(self.back_end, Target('craype-' + self.back_end)) - + self.add_target(self.front_end, Target(self.front_end)) + @classmethod def detect(self): return os.path.exists('/opt/cray/craype') - + diff --git a/lib/spack/spack/architectures/darwin.py b/lib/spack/spack/architectures/darwin.py new file mode 100644 index 0000000000..2f0d34c38d --- /dev/null +++ b/lib/spack/spack/architectures/darwin.py @@ -0,0 +1,18 @@ +import subprocess +from spack.architecture import Architecture, Target + +class Darwin(Architecture): + priority = 89 + front_end = 'x86_64' + back_end = 'x86_64' + default = 'x86_64' + + def __init__(self): + super(Darwin, self).__init__('darwin') + self.add_target(self.default, Target(self.default)) + + @classmethod + def detect(self): + arch = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) + arch, _ = arch.communicate() + return 'darwin' in arch.strip().lower() diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py index 052b10c019..d63cf9179a 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/architectures/linux.py @@ -2,7 +2,7 @@ import subprocess from spack.architecture import Architecture, Target class Linux(Architecture): - priority = 60 + priority = 90 front_end = 'x86_64' back_end = 'x86_64' default = 'x86_64' @@ -13,6 +13,6 @@ class Linux(Architecture): @classmethod def detect(self): - arch = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + arch = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) arch, _ = arch.communicate() return 'linux' in arch.strip().lower() diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index ed51fac33a..96dc1f4331 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -31,7 +31,7 @@ from llnl.util.tty.colify import colify import spack """Names of tests to be included in Spack's test suite""" -"""test_names = ['architecture', +test_names = ['architecture', 'versions', 'url_parse', 'url_substitution', @@ -58,8 +58,7 @@ import spack 'optional_deps', 'make_executable', 'configure_guess'] -""" -test_names = ['architecture'] + def list_tests(): """Return names of all tests that can be run for Spack.""" diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 6ff22aaa59..3a7474799a 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -2,18 +2,34 @@ the functions are looking for the correct architecture name """ import unittest +import os +import platform import spack from spack.architecture import * +from spack.architectures.cray import Cray +from spack.architectures.linux import Linux +from spack.architectures.bgq import Bgq class ArchitectureTest(unittest.TestCase): - def test_Architecture_class(self): - a = Architecture('Cray-XC40') - a.add_arch_strategy() - self.assertEquals(a.get_arch_dict(), {'Cray-XC40': 'MODULES'}) + def test_Architecture_class_and_compiler_strategies(self): + a = Cray() + t = a.target('default') + self.assertEquals(t.compiler_strategy, 'MODULES') + b = Linux() + s = b.target('default') + self.assertEquals(s.compiler_strategy, 'PATH') - def test_get_sys_type_from_config_file(self): - output_arch_class = get_sys_type_from_config_file() - my_arch_class = Architecture('Linux x86_64','Cray-xc40') + def test_sys_type(self): + output_arch_class = sys_type() + my_arch_class = None + if os.path.exists('/opt/cray/craype'): + my_arch_class = Cray() + elif os.path.exists('/bgsys'): + my_arch_class = Bgq() + elif 'Linux' in platform.system(): + my_arch_class = Linux() +# elif 'Darwin' in platform.system(): +# my_arch_class = Darwin() - self.assertEqual(output_arch_class, my_arch_class) + self.assertEqual(str(output_arch_class), str(my_arch_class)) -- cgit v1.2.3-60-g2f50 From 1edbaa4e62d2eb59b53cf6b10f7854d4edcf8497 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 14:13:09 -0800 Subject: removed debug printing --- lib/spack/spack/architecture.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 1f608bd187..1b4b7730b2 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -96,8 +96,6 @@ class Target(object): return self.__str__() def __str__(self): - if self.module_name: - return self.name + ' module: ' + self.module_name return self.name @key_ordering -- cgit v1.2.3-60-g2f50 From 751503c434bd249ed9f76b6ad353d2adcdcc3997 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 14:47:14 -0800 Subject: fixed haswell targeting bug --- lib/spack/spack/architectures/cray.py | 17 ++++++++++++----- lib/spack/spack/compilers/__init__.py | 1 + 2 files changed, 13 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index 47ede30145..dac3943fb7 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -17,15 +17,22 @@ class Cray(Architecture): # Handle the default here so we can check for a key error if 'CRAY_CPU_TARGET' in os.environ: - default = os.environ['CRAY_CPU_TARGET'] + self.default = os.environ['CRAY_CPU_TARGET'] + + # Change the defaults to haswell if we're on an XC40 + if self.default == 'haswell': + self.front_end = self.default + self.back_end = self.default - # Back End compiler needs the proper target module loaded. - self.add_target(self.back_end, Target(self.front_end,'craype-'+ self.back_end)) - self.add_target(self.default, Target(self.default,'craype-' + self.default)) # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. self.add_target(self.front_end, Target(self.front_end)) - + # Back End compiler needs the proper target module loaded. + self.add_target(self.back_end, Target(self.front_end,'craype-'+ self.back_end)) + self.add_target(self.default, Target(self.default,'craype-' + self.default)) + # This is kludgy and the order matters when the targets are all haswell + # This is because the last one overwrites the others when they have the + # same name. @classmethod def detect(self): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 87106282cf..21a41f38ea 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -206,6 +206,7 @@ def compilers_for_spec(compiler_spec): return cls(cspec, compiler_paths, mods) matches = find(compiler_spec) + print matches, 'matches' return [get_compiler(cspec) for cspec in matches] -- cgit v1.2.3-60-g2f50 From 48b9023de48ba01f789705db421c9bb326202c0e Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 14:51:18 -0800 Subject: removed debug printing --- lib/spack/spack/compilers/__init__.py | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 21a41f38ea..87106282cf 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -206,7 +206,6 @@ def compilers_for_spec(compiler_spec): return cls(cspec, compiler_paths, mods) matches = find(compiler_spec) - print matches, 'matches' return [get_compiler(cspec) for cspec in matches] -- cgit v1.2.3-60-g2f50 From d6768cf9210288ec562330fac0978cfa0b8b8683 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 16:23:32 -0800 Subject: minor bug fix --- lib/spack/spack/compiler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 5d87588245..b40f68b2c5 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -284,8 +284,7 @@ class Compiler(object): modulecmd.add_default_arg('python') output = modulecmd('avail', return_oe=True) matches = re.findall(r'(%s)/(\d+[\.\d]+)' % cls.PrgEnv_compiler, output) -# It's finding a weird third attribute - print matches + for name, version in matches: v = version comp = cls(spack.spec.CompilerSpec(name + '@' + v), -- cgit v1.2.3-60-g2f50 From c649610473a722388a09bf1000a8e3e9d6a4f9ff Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 17:00:28 -0800 Subject: fixed concretization to only consider compilers found by the proper strategy --- lib/spack/spack/concretize.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 48d05f7f8d..f32e956a13 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -260,8 +260,27 @@ class DefaultConcretizer(object): build with the compiler that will be used by libraries that link to this one, to maximize compatibility. """ + # Pass on concretizing the compiler if the architecture is not yet determined + if not spec.architecture: + #Although this usually means changed, this means awaiting other changes + return True + + # Examine only those compilers found by the proper compiler strategy for this architecture + # Takes advantage of the proper logic already existing in compiler_for_spec + # Should be redone more efficiently if this works all_compilers = spack.compilers.all_compilers() + def _proper_compiler_style(cspec, target): + compilers = spack.compilers.compilers_for_spec(cspec) + if target.compiler_strategy == 'PATH': + filter(lambda c: not c.modules, compilers) + if target.compiler_strategy == 'MODULES': + filter(lambda c: c.modules, compilers) + return compilers + + filter(lambda c: _proper_compiler_style(c, spec.architecture), all_compilers) + + if (spec.compiler and spec.compiler.concrete and spec.compiler in all_compilers): @@ -287,7 +306,7 @@ class DefaultConcretizer(object): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - spec.compiler = matches[len(matches)-1].copy() + spec.compiler = matches[0].copy() assert(spec.compiler.concrete) return True # things changed. -- cgit v1.2.3-60-g2f50 From 29e0ff61d7dc10b20b4b5d0d627cc066690ea84b Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 17:14:35 -0800 Subject: fixed compiler finding so as not to identify non-existent versions of the intel compiler based on the version numbers of the PrgEnv-intel module --- lib/spack/spack/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index b40f68b2c5..225c65917a 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -282,7 +282,7 @@ class Compiler(object): modulecmd = which('modulecmd') modulecmd.add_default_arg('python') - output = modulecmd('avail', return_oe=True) + output = modulecmd('avail', cls.PrgEnv_compiler, return_oe=True) matches = re.findall(r'(%s)/(\d+[\.\d]+)' % cls.PrgEnv_compiler, output) for name, version in matches: -- cgit v1.2.3-60-g2f50 From 93c9c45580d89ccbab441c5808192e149731c85f Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 17:19:04 -0800 Subject: minor tweak of compiler priority while waiting for customizable compiler priorities --- lib/spack/spack/concretize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index f32e956a13..2013ae2e84 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -306,7 +306,7 @@ class DefaultConcretizer(object): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - spec.compiler = matches[0].copy() + spec.compiler = matches[-1].copy() assert(spec.compiler.concrete) return True # things changed. -- cgit v1.2.3-60-g2f50 From 61b03b72b0fb1aa5c4ddbaff8079d9c0258efb76 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 19:03:25 -0800 Subject: improved concretize efficiency for determining whether compilers come from the proper strategy --- lib/spack/spack/concretize.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 2013ae2e84..854db949e3 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -265,11 +265,9 @@ class DefaultConcretizer(object): #Although this usually means changed, this means awaiting other changes return True - # Examine only those compilers found by the proper compiler strategy for this architecture + # Only use a matching compiler if it is of the proper style # Takes advantage of the proper logic already existing in compiler_for_spec - # Should be redone more efficiently if this works - all_compilers = spack.compilers.all_compilers() - + # Should think whether this can be more efficient def _proper_compiler_style(cspec, target): compilers = spack.compilers.compilers_for_spec(cspec) if target.compiler_strategy == 'PATH': @@ -278,8 +276,8 @@ class DefaultConcretizer(object): filter(lambda c: c.modules, compilers) return compilers - filter(lambda c: _proper_compiler_style(c, spec.architecture), all_compilers) - + + all_compilers = spack.compilers.all_compilers() if (spec.compiler and spec.compiler.concrete and @@ -306,7 +304,12 @@ class DefaultConcretizer(object): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - spec.compiler = matches[-1].copy() + index = len(matches)-1 + while not _proper_compiler_style(matches[index], spec.architecture): + index -= 1 + if index == 0: + raise NoValidVersionError(spec) + spec.compiler = matches[index].copy() assert(spec.compiler.concrete) return True # things changed. -- cgit v1.2.3-60-g2f50 From 51bd91edc0679d21c26df8af17eaafb2a0ca6d56 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 6 Jan 2016 13:00:22 -0800 Subject: made the compiler strategy more easily controllable --- lib/spack/spack/architecture.py | 15 ++++++++------- lib/spack/spack/architectures/bgq.py | 4 ++-- lib/spack/spack/architectures/cray.py | 6 +++--- lib/spack/spack/architectures/darwin.py | 2 +- lib/spack/spack/architectures/linux.py | 2 +- lib/spack/spack/build_environment.py | 2 +- 6 files changed, 16 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 1b4b7730b2..81aac6cc33 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -55,19 +55,20 @@ class Target(object): they came from using the set_architecture method. Targets will have compiler finding strategies """ - def __init__(self,name, module_name=None): + def __init__(self, name, compiler_strategy, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 + self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge def set_architecture(self, architecture): # Target should get the architecture class. self.architecture = architecture - @property - def compiler_strategy(self): - if self.module_name: # If there is a module_name given then use MODULES - return "MODULES" - else: - return "PATH" +# @property +# def compiler_strategy(self): +# if self.module_name: # If there is a module_name given then use MODULES +# return "MODULES" +# else: +# return "PATH" def to_dict(self): d = {} diff --git a/lib/spack/spack/architectures/bgq.py b/lib/spack/spack/architectures/bgq.py index 85487626f1..3ac5a59546 100644 --- a/lib/spack/spack/architectures/bgq.py +++ b/lib/spack/spack/architectures/bgq.py @@ -10,8 +10,8 @@ class Bgq(Architecture): def __init__(self): super(Bgq, self).__init__('cray') - self.add_target(self.front_end, Target(self.front_end)) - self.add_target(self.back_end, Target(self.back_end)) + self.add_target(self.front_end, Target(self.front_end, 'PATH')) + self.add_target(self.back_end, Target(self.back_end, 'PATH')) @classmethod def detect(self): diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index dac3943fb7..f109b48867 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -26,10 +26,10 @@ class Cray(Architecture): # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target(self.front_end, Target(self.front_end)) + self.add_target(self.front_end, Target(self.front_end, 'PATH')) # Back End compiler needs the proper target module loaded. - self.add_target(self.back_end, Target(self.front_end,'craype-'+ self.back_end)) - self.add_target(self.default, Target(self.default,'craype-' + self.default)) +# self.add_target(self.back_end, Target(self.front_end, 'MODULES', 'craype-'+ self.back_end)) + self.add_target(self.default, Target(self.default, 'MODULES', 'craype-' + self.default)) # This is kludgy and the order matters when the targets are all haswell # This is because the last one overwrites the others when they have the # same name. diff --git a/lib/spack/spack/architectures/darwin.py b/lib/spack/spack/architectures/darwin.py index 2f0d34c38d..30fbde39bb 100644 --- a/lib/spack/spack/architectures/darwin.py +++ b/lib/spack/spack/architectures/darwin.py @@ -9,7 +9,7 @@ class Darwin(Architecture): def __init__(self): super(Darwin, self).__init__('darwin') - self.add_target(self.default, Target(self.default)) + self.add_target(self.default, Target(self.default, 'PATH')) @classmethod def detect(self): diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py index d63cf9179a..bb3089ebf6 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/architectures/linux.py @@ -9,7 +9,7 @@ class Linux(Architecture): def __init__(self): super(Linux, self).__init__('linux') - self.add_target(self.default, Target(self.default)) + self.add_target(self.default, Target(self.default, 'PATH')) @classmethod def detect(self): diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 781039e073..fc71ae0331 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -226,7 +226,7 @@ def set_build_environment_variables(pkg): pkg_config_dirs.append(pcdir) path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) - if pkg.spec.architecture.compiler_strategy.lower() == 'module': + if pkg.spec.architecture.module_name: load_module(pkg.spec.architecture.module_name) def set_module_variables_for_package(pkg): -- cgit v1.2.3-60-g2f50 From b4a0004f44af89c034423adef5444a234c49d464 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 6 Jan 2016 14:50:31 -0800 Subject: Improved target cmp_key and to/from yaml functions --- lib/spack/spack/architecture.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 81aac6cc33..aaea7c81ac 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -60,22 +60,17 @@ class Target(object): self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge - def set_architecture(self, architecture): # Target should get the architecture class. - self.architecture = architecture - -# @property -# def compiler_strategy(self): -# if self.module_name: # If there is a module_name given then use MODULES -# return "MODULES" -# else: -# return "PATH" + # Sets only the architecture name to avoid recursiveness + def set_architecture(self, architecture): + self.architecture_name = architecture.name def to_dict(self): d = {} d['name'] = self.name + d['compiler_strategy'] = self.compiler_strategy d['module_name'] = self.module_name -# if self.architecture: -# d['architecture'] = self.architecture + if self.architecture_name: + d['architecture'] = self.architecture_name return d @staticmethod @@ -84,14 +79,15 @@ class Target(object): return None target = Target.__new__(Target) target.name = d['name'] + target.compiler_strategy = d['compiler_strategy'] target.module_name = d['module_name'] -# if 'architecture' in d: -# target.architecture = d['architecture'] + if 'architecture' in d: + target.architecture_name = d['architecture'] return target def _cmp_key(self): - return (self.name, self.module_name) + return (self.name, self.compiler_strategy, self.module_name) def __repr__(self): return self.__str__() -- cgit v1.2.3-60-g2f50 From 6ccd9d6fa4baa9b14e0cea19e55b24ad19d053f0 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 7 Jan 2016 11:49:01 -0800 Subject: Cleaned up naming conventions for architecture, split into platform and target --- lib/spack/spack/__init__.py | 2 +- lib/spack/spack/abi.py | 8 +- lib/spack/spack/architecture.py | 62 +++++++------- lib/spack/spack/architectures/__init__.py | 0 lib/spack/spack/architectures/bgq.py | 19 ----- lib/spack/spack/architectures/cray.py | 40 --------- lib/spack/spack/architectures/darwin.py | 18 ----- lib/spack/spack/architectures/linux.py | 18 ----- lib/spack/spack/build_environment.py | 4 +- lib/spack/spack/cmd/find.py | 10 +-- lib/spack/spack/concretize.py | 40 ++++----- lib/spack/spack/config.py | 42 +++++----- lib/spack/spack/directory_layout.py | 4 +- lib/spack/spack/modules.py | 4 +- lib/spack/spack/package.py | 2 +- lib/spack/spack/platforms/__init__.py | 0 lib/spack/spack/platforms/bgq.py | 19 +++++ lib/spack/spack/platforms/cray.py | 40 +++++++++ lib/spack/spack/platforms/darwin.py | 18 +++++ lib/spack/spack/platforms/linux.py | 18 +++++ lib/spack/spack/preferred_packages.py | 10 +-- lib/spack/spack/spec.py | 108 ++++++++++++------------- lib/spack/spack/test/architecture.py | 23 +++--- lib/spack/spack/test/concretize.py | 4 +- lib/spack/spack/test/multimethod.py | 12 +-- lib/spack/spack/test/spec_dag.py | 4 +- lib/spack/spack/test/spec_semantics.py | 4 +- var/spack/mock_packages/multimethod/package.py | 10 +-- 28 files changed, 272 insertions(+), 271 deletions(-) delete mode 100644 lib/spack/spack/architectures/__init__.py delete mode 100644 lib/spack/spack/architectures/bgq.py delete mode 100644 lib/spack/spack/architectures/cray.py delete mode 100644 lib/spack/spack/architectures/darwin.py delete mode 100644 lib/spack/spack/architectures/linux.py create mode 100644 lib/spack/spack/platforms/__init__.py create mode 100644 lib/spack/spack/platforms/bgq.py create mode 100644 lib/spack/spack/platforms/cray.py create mode 100644 lib/spack/spack/platforms/darwin.py create mode 100644 lib/spack/spack/platforms/linux.py (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index b2ff2c692b..e1ef094f17 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -37,7 +37,7 @@ etc_path = join_path(prefix, "etc") lib_path = join_path(prefix, "lib", "spack") build_env_path = join_path(lib_path, "env") module_path = join_path(lib_path, "spack") -arch_path = join_path(module_path, 'architectures') +platform_path = join_path(module_path, 'platforms') compilers_path = join_path(module_path, "compilers") test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index f0a997703c..1dd49d6b2c 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -34,9 +34,9 @@ class ABI(object): """This class provides methods to test ABI compatibility between specs. The current implementation is rather rough and could be improved.""" - def architecture_compatible(self, parent, child): - """Returns true iff the parent and child specs have ABI compatible architectures.""" - return not parent.architecture or not child.architecture or parent.architecture == child.architecture + def target_compatible(self, parent, child): + """Returns true iff the parent and child specs have ABI compatible targets.""" + return not parent.target or not child.target or parent.target == child.target @memoized @@ -123,6 +123,6 @@ class ABI(object): def compatible(self, parent, child, **kwargs): """Returns true iff a parent and child spec are ABI compatible""" loosematch = kwargs.get('loose', False) - return self.architecture_compatible(parent, child) and \ + return self.target_compatible(parent, child) and \ self.compiler_compatible(parent, child, loose=loosematch) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index aaea7c81ac..a269767fab 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -51,8 +51,8 @@ class NoSysTypeError(serr.SpackError): class Target(object): """ Target is the processor of the host machine. The host machine may have different front-end and back-end targets, especially if it is a Cray machine. The target will have a name and - also the module_name (e.g craype-compiler). Targets will also recognize which architecture - they came from using the set_architecture method. Targets will have compiler finding strategies + also the module_name (e.g craype-compiler). Targets will also recognize which platform + they came from using the set_platform method. Targets will have compiler finding strategies """ def __init__(self, name, compiler_strategy, module_name=None): @@ -60,17 +60,17 @@ class Target(object): self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge - # Sets only the architecture name to avoid recursiveness - def set_architecture(self, architecture): - self.architecture_name = architecture.name + # Sets only the platform name to avoid recursiveness + def set_platform(self, platform): + self.platform_name = platform.name def to_dict(self): d = {} d['name'] = self.name d['compiler_strategy'] = self.compiler_strategy d['module_name'] = self.module_name - if self.architecture_name: - d['architecture'] = self.architecture_name + if self.platform_name: + d['platform'] = self.platform_name return d @staticmethod @@ -81,8 +81,8 @@ class Target(object): target.name = d['name'] target.compiler_strategy = d['compiler_strategy'] target.module_name = d['module_name'] - if 'architecture' in d: - target.architecture_name = d['architecture'] + if 'platform' in d: + target.platform_name = d['platform'] return target @@ -96,13 +96,13 @@ class Target(object): return self.name @key_ordering -class Architecture(object): - """ Abstract class that each type of Architecture will subclass. +class Platform(object): + """ Abstract class that each type of Platform will subclass. Will return a instance of it once it is returned """ - priority = None # Subclass needs to set this number. This controls order in which arch is detected. + priority = None # Subclass needs to set this number. This controls order in which platform is detected. front_end = None back_end = None default = None # The default back end target. On cray ivybridge @@ -112,12 +112,12 @@ class Architecture(object): self.name = name def add_target(self, name, target): - """Used by the architecture specific subclass to list available targets. Raises an error - if the architecture specifies a name that is reserved by spack as an alias. + """Used by the platform specific subclass to list available targets. Raises an error + if the platform specifies a name that is reserved by spack as an alias. """ if name in ['front_end', 'fe', 'back_end', 'be', 'default']: raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) - target.set_architecture(self) + target.set_platform(self) self.targets[name] = target def target(self, name): @@ -137,7 +137,7 @@ class Architecture(object): @classmethod def detect(self): """ Subclass is responsible for implementing this method. - Returns True if the architecture detects if it is the current architecture + Returns True if the Platform class detects that it is the current platform and False if it's not. """ raise NotImplementedError() @@ -182,18 +182,18 @@ def get_sys_type_from_uname(): Front-end config """ try: - arch_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) - arch, _ = arch_proc.communicate() - return arch.strip() + platform_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + platform, _ = platform_proc.communicate() + return platform.strip() except: return None @memoized -def all_architectures(): +def all_platforms(): modules = [] - for name in list_modules(spack.arch_path): - mod_name = 'spack.architectures' + name - path = join_path(spack.arch_path, name) + ".py" + for name in list_modules(spack.platform_path): + mod_name = 'spack.platformss' + name + path = join_path(spack.platform_path, name) + ".py" mod = imp.load_source(mod_name, path) class_name = mod_to_class(name) if not hasattr(mod, class_name): @@ -208,16 +208,16 @@ def all_architectures(): @memoized def sys_type(): - """ Gather a list of all available subclasses of architectures. + """ Gather a list of all available subclasses of platforms. Sorts the list according to their priority looking. Priority is - an arbitrarily set number. Detects arch either using uname or + an arbitrarily set number. Detects platform either using uname or a file path (/opt/cray...) """ - # Try to create an architecture object using the config file FIRST - architecture_list = all_architectures() - architecture_list.sort(key = lambda a: a.priority) + # Try to create a Platform object using the config file FIRST + platform_list = all_platforms() + platform_list.sort(key = lambda a: a.priority) - for arch in architecture_list: - if arch.detect(): - return arch() + for platform in platform_list: + if platform.detect(): + return platform() diff --git a/lib/spack/spack/architectures/__init__.py b/lib/spack/spack/architectures/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lib/spack/spack/architectures/bgq.py b/lib/spack/spack/architectures/bgq.py deleted file mode 100644 index 3ac5a59546..0000000000 --- a/lib/spack/spack/architectures/bgq.py +++ /dev/null @@ -1,19 +0,0 @@ -import os - -from spack.architecture import Architecture, Target - -class Bgq(Architecture): - priority = 30 - front_end = 'power7' - back_end = 'powerpc' - default = 'powerpc' - - def __init__(self): - super(Bgq, self).__init__('cray') - self.add_target(self.front_end, Target(self.front_end, 'PATH')) - self.add_target(self.back_end, Target(self.back_end, 'PATH')) - - @classmethod - def detect(self): - return os.path.exists('/bgsys') - diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py deleted file mode 100644 index f109b48867..0000000000 --- a/lib/spack/spack/architectures/cray.py +++ /dev/null @@ -1,40 +0,0 @@ -import os - -from spack.architecture import Architecture, Target - -class Cray(Architecture): - priority = 20 - front_end = 'sandybridge' - back_end = 'ivybridge' - default = 'ivybridge' - - def __init__(self): - ''' Since cori doesn't have ivybridge as a front end it's better - if we use CRAY_CPU_TARGET as the default. This will ensure - that if we're on a XC-40 or XC-30 then we can detect the target - ''' - super(Cray, self).__init__('cray') - - # Handle the default here so we can check for a key error - if 'CRAY_CPU_TARGET' in os.environ: - self.default = os.environ['CRAY_CPU_TARGET'] - - # Change the defaults to haswell if we're on an XC40 - if self.default == 'haswell': - self.front_end = self.default - self.back_end = self.default - - # Could switch to use modules and fe targets for front end - # Currently using compilers by path for front end. - self.add_target(self.front_end, Target(self.front_end, 'PATH')) - # Back End compiler needs the proper target module loaded. -# self.add_target(self.back_end, Target(self.front_end, 'MODULES', 'craype-'+ self.back_end)) - self.add_target(self.default, Target(self.default, 'MODULES', 'craype-' + self.default)) - # This is kludgy and the order matters when the targets are all haswell - # This is because the last one overwrites the others when they have the - # same name. - - @classmethod - def detect(self): - return os.path.exists('/opt/cray/craype') - diff --git a/lib/spack/spack/architectures/darwin.py b/lib/spack/spack/architectures/darwin.py deleted file mode 100644 index 30fbde39bb..0000000000 --- a/lib/spack/spack/architectures/darwin.py +++ /dev/null @@ -1,18 +0,0 @@ -import subprocess -from spack.architecture import Architecture, Target - -class Darwin(Architecture): - priority = 89 - front_end = 'x86_64' - back_end = 'x86_64' - default = 'x86_64' - - def __init__(self): - super(Darwin, self).__init__('darwin') - self.add_target(self.default, Target(self.default, 'PATH')) - - @classmethod - def detect(self): - arch = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) - arch, _ = arch.communicate() - return 'darwin' in arch.strip().lower() diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py deleted file mode 100644 index bb3089ebf6..0000000000 --- a/lib/spack/spack/architectures/linux.py +++ /dev/null @@ -1,18 +0,0 @@ -import subprocess -from spack.architecture import Architecture, Target - -class Linux(Architecture): - priority = 90 - front_end = 'x86_64' - back_end = 'x86_64' - default = 'x86_64' - - def __init__(self): - super(Linux, self).__init__('linux') - self.add_target(self.default, Target(self.default, 'PATH')) - - @classmethod - def detect(self): - arch = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) - arch, _ = arch.communicate() - return 'linux' in arch.strip().lower() diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index fc71ae0331..d849b4b56c 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -226,8 +226,8 @@ def set_build_environment_variables(pkg): pkg_config_dirs.append(pcdir) path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) - if pkg.spec.architecture.module_name: - load_module(pkg.spec.architecture.module_name) + if pkg.spec.target.module_name: + load_module(pkg.spec.target.module_name) def set_module_variables_for_package(pkg): """Populate the module scope of install() with some useful functions. diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 3c993990b1..7ea16bf09f 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -72,19 +72,19 @@ def display_specs(specs, **kwargs): hashes = True hlen = None - # Make a dict with specs keyed by architecture and compiler. - index = index_by(specs, ('architecture', 'compiler')) + # Make a dict with specs keyed by target and compiler. + index = index_by(specs, ('target', 'compiler')) # Traverse the index and print out each package - for i, (architecture, compiler) in enumerate(sorted(index)): + for i, (target, compiler) in enumerate(sorted(index)): if i > 0: print header = "%s{%s} / %s{%s}" % ( - spack.spec.architecture_color, architecture, + spack.spec.target_color, target, spack.spec.compiler_color, compiler) tty.hline(colorize(header), char='-') - specs = index[(architecture,compiler)] + specs = index[(target,compiler)] specs.sort() abbreviated = [s.format('$_$@$+', color=True) for s in specs] diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 854db949e3..fcd23a6055 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -208,30 +208,30 @@ class DefaultConcretizer(object): return True # Things changed - def concretize_architecture(self, spec): - """If the spec already has an architecture and it is a an architecture type, - return. Otherwise, if it has an architecture that is a string type, generate an - architecture based on that type. If it has no architecture and the root of the - DAG has an architecture, then use that. Otherwise, take the system's default - architecture. + def concretize_target(self, spec): + """If the spec already has an target and it is a an target type, + return. Otherwise, if it has a target that is a string type, generate a + target based on that type. If it has no target and the root of the + DAG has an target, then use that. Otherwise, take the system's default + target. """ - if spec.architecture is not None: - if isinstance(spec.architecture,spack.architecture.Target): + if spec.target is not None: + if isinstance(spec.target,spack.architecture.Target): return False else: - arch = spack.architecture.sys_type() - spec.architecture = arch.target(spec.architecture) + platform = spack.architecture.sys_type() + spec.target = platform.target(spec.target) return True #changed - if spec.root.architecture: - if isinstance(spec.root.architecture,spack.architecture.Target): - spec.architecture = spec.root.architecture + if spec.root.target: + if isinstance(spec.root.target,spack.architecture.Target): + spec.target = spec.root.target else: - arch = spack.architecture.sys_type() - spec.architecture = arch.target(spec.root.architecture) + platform = spack.architecture.sys_type() + spec.target = platform.target(spec.root.target) else: - arch = spack.architecture.sys_type() - spec.architecture = arch.target('default') + platform = spack.architecture.sys_type() + spec.target = platform.target('default') return True #changed @@ -260,8 +260,8 @@ class DefaultConcretizer(object): build with the compiler that will be used by libraries that link to this one, to maximize compatibility. """ - # Pass on concretizing the compiler if the architecture is not yet determined - if not spec.architecture: + # Pass on concretizing the compiler if the target is not yet determined + if not spec.target: #Although this usually means changed, this means awaiting other changes return True @@ -305,7 +305,7 @@ class DefaultConcretizer(object): # copy concrete version into other_compiler index = len(matches)-1 - while not _proper_compiler_style(matches[index], spec.architecture): + while not _proper_compiler_style(matches[index], spec.target): index -= 1 if index == 0: raise NoValidVersionError(spec) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 6e11cfa475..0622606a4f 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -129,7 +129,7 @@ _ConfigCategory('packages', 'packages.yaml', True) config_scopes = [('site', os.path.join(spack.etc_path, 'spack')), ('user', os.path.expanduser('~/.spack'))] -_compiler_by_arch = {} +_compiler_by_platform = {} _read_config_file_result = {} def _read_config_file(filename): """Read a given YAML configuration file""" @@ -156,7 +156,7 @@ def clear_config_caches(): s.files_read_from = [] s.result_dict = {} spack.config._read_config_file_result = {} - spack.config._compiler_by_arch = {} + spack.config._compiler_by_platform = {} spack.compilers._cached_default_compiler = None @@ -213,27 +213,27 @@ def get_config(category_name): return category.result_dict -def get_compilers_config(arch=None): +def get_compilers_config(platform=None): """Get the compiler configuration from config files for the given - architecture. Strips off the architecture component of the + platform. Strips off the platform component of the configuration""" - global _compiler_by_arch - if not arch: - arch = str(spack.architecture.sys_type()) - if arch in _compiler_by_arch: - return _compiler_by_arch[arch] + global _compiler_by_platform + if not platform: + platform = str(spack.architecture.sys_type()) + if platform in _compiler_by_platform: + return _compiler_by_platform[platform] cc_config = get_config('compilers') - if arch in cc_config and 'all' in cc_config: - arch_compiler = dict(cc_config[arch]) - _compiler_by_arch[arch] = _merge_dict(arch_compiler, cc_config['all']) - elif arch in cc_config: - _compiler_by_arch[arch] = cc_config[arch] + if platform in cc_config and 'all' in cc_config: + platform_compiler = dict(cc_config[platform]) + _compiler_by_platform[platform] = _merge_dict(platform_compiler, cc_config['all']) + elif platform in cc_config: + _compiler_by_platform[platform] = cc_config[platform] elif 'all' in cc_config: - _compiler_by_arch[arch] = cc_config['all'] + _compiler_by_platform[platform] = cc_config['all'] else: - _compiler_by_arch[arch] = {} - return _compiler_by_arch[arch] + _compiler_by_platform[platform] = {} + return _compiler_by_platform[platform] def get_mirror_config(): @@ -371,11 +371,11 @@ def add_to_mirror_config(addition_dict, scope=None): add_to_config('mirrors', addition_dict, scope) -def add_to_compiler_config(addition_dict, scope=None, arch=None): +def add_to_compiler_config(addition_dict, scope=None, platform=None): """Add compilers to the configuration files""" - if not arch: - arch = spack.architecture.sys_type() - add_to_config('compilers', { str(arch) : addition_dict }, scope) + if not platform: + platform = spack.architecture.sys_type() + add_to_config('compilers', { str(platform) : addition_dict }, scope) clear_config_caches() diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 83e6eb566a..da3441fa13 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -156,7 +156,7 @@ class DirectoryLayout(object): class YamlDirectoryLayout(DirectoryLayout): """Lays out installation directories like this:: / - / + / -/ --- @@ -201,7 +201,7 @@ class YamlDirectoryLayout(DirectoryLayout): spec.dag_hash(self.hash_len)) path = join_path( - spec.architecture, + spec.target, "%s-%s" % (spec.compiler.name, spec.compiler.version), dir_name) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 56a61adefb..f84ac75c77 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -193,7 +193,7 @@ class Dotkit(EnvModule): @property def file_name(self): - return join_path(Dotkit.path, self.spec.architecture, + return join_path(Dotkit.path, self.spec.target, self.spec.format('$_$@$%@$+$#.dk')) @property @@ -230,7 +230,7 @@ class TclModule(EnvModule): @property def file_name(self): - return join_path(TclModule.path, self.spec.architecture, self.use_name) + return join_path(TclModule.path, self.spec.target, self.use_name) @property diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index e0097fa88d..1148e4ac3d 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -592,7 +592,7 @@ class Package(object): """Get the spack.compiler.Compiler object used to build this package.""" if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") - return spack.compilers.compiler_for_spec(self.spec.compiler, self.spec.architecture) + return spack.compilers.compiler_for_spec(self.spec.compiler, self.spec.target) def url_version(self, version): diff --git a/lib/spack/spack/platforms/__init__.py b/lib/spack/spack/platforms/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/spack/spack/platforms/bgq.py b/lib/spack/spack/platforms/bgq.py new file mode 100644 index 0000000000..988e67023c --- /dev/null +++ b/lib/spack/spack/platforms/bgq.py @@ -0,0 +1,19 @@ +import os + +from spack.architecture import Platform, Target + +class Bgq(Platform): + priority = 30 + front_end = 'power7' + back_end = 'powerpc' + default = 'powerpc' + + def __init__(self): + super(Bgq, self).__init__('cray') + self.add_target(self.front_end, Target(self.front_end, 'PATH')) + self.add_target(self.back_end, Target(self.back_end, 'PATH')) + + @classmethod + def detect(self): + return os.path.exists('/bgsys') + diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py new file mode 100644 index 0000000000..815a9aea98 --- /dev/null +++ b/lib/spack/spack/platforms/cray.py @@ -0,0 +1,40 @@ +import os + +from spack.architecture import Platform, Target + +class Cray(Platform): + priority = 20 + front_end = 'sandybridge' + back_end = 'ivybridge' + default = 'ivybridge' + + def __init__(self): + ''' Since cori doesn't have ivybridge as a front end it's better + if we use CRAY_CPU_TARGET as the default. This will ensure + that if we're on a XC-40 or XC-30 then we can detect the target + ''' + super(Cray, self).__init__('cray') + + # Handle the default here so we can check for a key error + if 'CRAY_CPU_TARGET' in os.environ: + self.default = os.environ['CRAY_CPU_TARGET'] + + # Change the defaults to haswell if we're on an XC40 + if self.default == 'haswell': + self.front_end = self.default + self.back_end = self.default + + # Could switch to use modules and fe targets for front end + # Currently using compilers by path for front end. + self.add_target(self.front_end, Target(self.front_end, 'PATH')) + # Back End compiler needs the proper target module loaded. +# self.add_target(self.back_end, Target(self.front_end, 'MODULES', 'craype-'+ self.back_end)) + self.add_target(self.default, Target(self.default, 'MODULES', 'craype-' + self.default)) + # This is kludgy and the order matters when the targets are all haswell + # This is because the last one overwrites the others when they have the + # same name. + + @classmethod + def detect(self): + return os.path.exists('/opt/cray/craype') + diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py new file mode 100644 index 0000000000..52284826b1 --- /dev/null +++ b/lib/spack/spack/platforms/darwin.py @@ -0,0 +1,18 @@ +import subprocess +from spack.architecture import Platform, Target + +class Darwin(Platform): + priority = 89 + front_end = 'x86_64' + back_end = 'x86_64' + default = 'x86_64' + + def __init__(self): + super(Darwin, self).__init__('darwin') + self.add_target(self.default, Target(self.default, 'PATH')) + + @classmethod + def detect(self): + platform = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) + platform, _ = platform.communicate() + return 'darwin' in platform.strip().lower() diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py new file mode 100644 index 0000000000..7f94d80c34 --- /dev/null +++ b/lib/spack/spack/platforms/linux.py @@ -0,0 +1,18 @@ +import subprocess +from spack.architecture import Platform, Target + +class Linux(Platform): + priority = 90 + front_end = 'x86_64' + back_end = 'x86_64' + default = 'x86_64' + + def __init__(self): + super(Linux, self).__init__('linux') + self.add_target(self.default, Target(self.default, 'PATH')) + + @classmethod + def detect(self): + platform = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) + platform, _ = platform.communicate() + return 'linux' in platform.strip().lower() diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index bc2a4ac234..993683a62f 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -163,11 +163,11 @@ class PreferredPackages(object): return self._component_compare(pkgname, 'variant', a, b, False, None) - def architecture_compare(self, pkgname, a, b): - """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is - respecively less-than, equal-to, or greater-than architecture b of pkgname. - One architecture is less-than another if it is preferred over the other.""" - return self._component_compare(pkgname, 'architecture', a, b, False, None) + def target_compare(self, pkgname, a, b): + """Return less-than-0, 0, or greater than 0 if target a of pkgname is + respecively less-than, equal-to, or greater-than target b of pkgname. + One target is less-than another if it is preferred over the other.""" + return self._component_compare(pkgname, 'target', a, b, False, None) def compiler_compare(self, pkgname, a, b): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 126ffc4f95..9e011bfb9f 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -63,7 +63,7 @@ line is a spec for a particular installation of the mpileaks package. if it comes immediately after the compiler name. Otherwise it will be associated with the current package spec. -6. The architecture to build with. This is needed on machines where +6. The target to build with. This is needed on machines where cross-compilation is required Here is the EBNF grammar for a spec:: @@ -72,9 +72,9 @@ Here is the EBNF grammar for a spec:: dep_list = { ^ spec } spec = id [ options ] options = { @version-list | +variant | -variant | ~variant | - %compiler | =architecture } + %compiler | =target } variant = id - architecture = id + target = id compiler = id [ version-list ] version-list = version [ { , version } ] version = id | id: | :id | id:id @@ -119,7 +119,7 @@ identifier_re = r'\w[\w-]*' # Convenient names for color formats so that other things can use them compiler_color = '@g' version_color = '@c' -architecture_color = '@m' +target_color = '@m' enabled_variant_color = '@B' disabled_variant_color = '@r' dependency_color = '@.' @@ -130,7 +130,7 @@ hash_color = '@K' See spack.color for descriptions of the color codes. """ color_formats = {'%' : compiler_color, '@' : version_color, - '=' : architecture_color, + '=' : target_color, '+' : enabled_variant_color, '~' : disabled_variant_color, '^' : dependency_color, @@ -407,7 +407,7 @@ class Spec(object): self.name = other.name self.dependents = other.dependents self.versions = other.versions - self.architecture = other.architecture + self.target = other.target self.compiler = other.compiler self.dependencies = other.dependencies self.variants = other.variants @@ -452,11 +452,11 @@ class Spec(object): self.compiler = compiler - def _set_architecture(self, architecture): - """Called by the parser to set the architecture.""" - if self.architecture: raise DuplicateArchitectureError( - "Spec for '%s' cannot have two architectures." % self.name) - self.architecture = architecture + def _set_target(self, target): + """Called by the parser to set the target.""" + if self.target: raise DuplicateTargetError( + "Spec for '%s' cannot have two targets." % self.name) + self.target = target def _add_dependency(self, spec): @@ -512,7 +512,7 @@ class Spec(object): @property def concrete(self): """A spec is concrete if it can describe only ONE build of a package. - If any of the name, version, architecture, compiler, + If any of the name, version, target, compiler, variants, or depdenencies are ambiguous,then it is not concrete. """ if self._concrete: @@ -521,7 +521,7 @@ class Spec(object): self._concrete = bool(not self.virtual and self.versions.concrete and self.variants.concrete - and self.architecture + and self.target and self.compiler and self.compiler.concrete and self.dependencies.concrete) return self._concrete @@ -656,10 +656,10 @@ class Spec(object): 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } - if self.architecture: - d['arch'] = self.architecture.to_dict() + if self.target: + d['target'] = self.target.to_dict() else: - d['arch'] = None + d['target'] = None if self.compiler: d.update(self.compiler.to_dict()) else: @@ -685,7 +685,7 @@ class Spec(object): spec = Spec(name) spec.versions = VersionList.from_dict(node) - spec.architecture = spack.architecture.Target.from_dict(node['arch']) + spec.target = spack.architecture.Target.from_dict(node['target']) if node['compiler'] is None: spec.compiler = None @@ -757,7 +757,7 @@ class Spec(object): # to presets below, their constraints will all be merged, but we'll # still need to select a concrete package later. changed |= any( - (spack.concretizer.concretize_architecture(self), + (spack.concretizer.concretize_target(self), spack.concretizer.concretize_compiler(self), spack.concretizer.concretize_version(self), spack.concretizer.concretize_variants(self))) @@ -1145,10 +1145,10 @@ class Spec(object): raise UnsatisfiableVariantSpecError(self.variants[v], other.variants[v]) - if self.architecture is not None and other.architecture is not None: - if self.architecture != other.architecture: - raise UnsatisfiableArchitectureSpecError(self.architecture, - other.architecture) + if self.target is not None and other.target is not None: + if self.target != other.target: + raise UnsatisfiableTargetSpecError(self.target, + other.target) changed = False if self.compiler is not None and other.compiler is not None: @@ -1160,9 +1160,9 @@ class Spec(object): changed |= self.versions.intersect(other.versions) changed |= self.variants.constrain(other.variants) - old = self.architecture - self.architecture = self.architecture or other.architecture - changed |= (self.architecture != old) + old = self.target + self.target = self.target or other.target + changed |= (self.target != old) if deps: changed |= self._constrain_dependencies(other) @@ -1273,12 +1273,12 @@ class Spec(object): if not self.variants.satisfies(other.variants, strict=strict): return False - # Architecture satisfaction is currently just string equality. + # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. - if self.architecture and other.architecture: - if self.architecture != other.architecture: + if self.target and other.target: + if self.target != other.target: return False - elif strict and (other.architecture and not self.architecture): + elif strict and (other.target and not self.target): return False # If we need to descend into dependencies, do it, otherwise we're done. @@ -1352,7 +1352,7 @@ class Spec(object): changed = True if hasattr(self, 'name'): changed = (self.name != other.name and self.versions != other.versions and \ - self.architecture != other.architecture and self.compiler != other.compiler and \ + self.target != other.target and self.compiler != other.compiler and \ self.variants != other.variants and self._normal != other._normal and \ self.concrete != other.concrete and self.external != other.external and \ self.external_module != other.external_module) @@ -1360,7 +1360,7 @@ class Spec(object): # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() - self.architecture = other.architecture + self.target = other.target self.compiler = other.compiler.copy() if other.compiler else None if kwargs.get('cleardeps', True): self.dependents = DependencyMap() @@ -1490,7 +1490,7 @@ class Spec(object): def _cmp_node(self): """Comparison key for just *this node* and not its deps.""" return (self.name, self.versions, self.variants, - self.architecture, self.compiler) + self.target, self.compiler) def eq_node(self, other): @@ -1524,7 +1524,7 @@ class Spec(object): $% Compiler with '%' prefix $%@ Compiler with '%' prefix & compiler version with '@' prefix $+ Options - $= Architecture with '=' prefix + $= Target with '=' prefix $# 7-char prefix of DAG hash with '-' prefix $$ $ @@ -1536,7 +1536,7 @@ class Spec(object): ${COMPILERNAME} Compiler name ${COMPILERVER} Compiler version ${OPTIONS} Options - ${ARCHITECTURE} Architecture + ${TARGET} Target ${SHA1} Dependencies 8-char sha1 prefix ${SPACK_ROOT} The spack root directory @@ -1549,7 +1549,7 @@ class Spec(object): Anything else is copied verbatim into the output stream. *Example:* ``$_$@$+`` translates to the name, version, and options - of the package, but no dependencies, arch, or compiler. + of the package, but no dependencies, target, or compiler. TODO: allow, e.g., $6# to customize short hash length TODO: allow, e.g., $## for full hash. @@ -1593,8 +1593,8 @@ class Spec(object): if self.variants: write(fmt % str(self.variants), c) elif c == '=': - if self.architecture: - write(fmt % (c + str(self.architecture)), c) + if self.target: + write(fmt % (c + str(self.target)), c) elif c == '#': out.write('-' + fmt % (self.dag_hash(7))) elif c == '$': @@ -1641,9 +1641,9 @@ class Spec(object): elif named_str == 'OPTIONS': if self.variants: write(fmt % str(self.variants), '+') - elif named_str == 'ARCHITECTURE': - if self.architecture: - write(fmt % str(self.architecture), '=') + elif named_str == 'TARGET': + if self.target: + write(fmt % str(self.target), '=') elif named_str == 'SHA1': if self.dependencies: out.write(fmt % str(self.dep_hash(8))) @@ -1691,10 +1691,10 @@ class Spec(object): return spack.pkgsort.variant_compare(pkgname, self.variants, other.variants) - #Architecture - if self.architecture != other.architecture: - return spack.pkgsort.architecture_compare(pkgname, - self.architecture, other.architecture) + #Target + if self.target != other.target: + return spack.pkgsort.target_compare(pkgname, + self.target, other.target) #Dependency is not configurable if self.dep_hash() != other.dep_hash(): @@ -1811,7 +1811,7 @@ class SpecParser(spack.parse.Parser): spec.name = self.token.value spec.versions = VersionList() spec.variants = VariantMap(spec) - spec.architecture = None + spec.target = None spec.compiler = None spec.external = None spec.external_module = None @@ -1842,7 +1842,7 @@ class SpecParser(spack.parse.Parser): spec._set_compiler(self.compiler()) elif self.accept(EQ): - spec._set_architecture(self.architecture()) + spec._set_target(self.target()) else: break @@ -1860,7 +1860,7 @@ class SpecParser(spack.parse.Parser): return self.token.value - def architecture(self): + def target(self): self.expect(ID) return self.token.value @@ -2000,10 +2000,10 @@ class UnknownVariantError(SpecError): "Package %s has no variant %s!" % (pkg, variant)) -class DuplicateArchitectureError(SpecError): - """Raised when the same architecture occurs in a spec twice.""" +class DuplicateTargetError(SpecError): + """Raised when the same target occurs in a spec twice.""" def __init__(self, message): - super(DuplicateArchitectureError, self).__init__(message) + super(DuplicateTargetError, self).__init__(message) class InconsistentSpecError(SpecError): @@ -2082,11 +2082,11 @@ class UnsatisfiableVariantSpecError(UnsatisfiableSpecError): provided, required, "variant") -class UnsatisfiableArchitectureSpecError(UnsatisfiableSpecError): - """Raised when a spec architecture conflicts with package constraints.""" +class UnsatisfiableTargetSpecError(UnsatisfiableSpecError): + """Raised when a spec target conflicts with package constraints.""" def __init__(self, provided, required): - super(UnsatisfiableArchitectureSpecError, self).__init__( - provided, required, "architecture") + super(UnsatisfiableTargetSpecError, self).__init__( + provided, required, "target") class UnsatisfiableProviderSpecError(UnsatisfiableSpecError): diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 3a7474799a..32b16feeb1 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -6,13 +6,14 @@ import os import platform import spack from spack.architecture import * -from spack.architectures.cray import Cray -from spack.architectures.linux import Linux -from spack.architectures.bgq import Bgq +from spack.platforms.cray import Cray +from spack.platforms.linux import Linux +from spack.platforms.bgq import Bgq +from spack.platforms.darwin import Darwin class ArchitectureTest(unittest.TestCase): - def test_Architecture_class_and_compiler_strategies(self): + def test_platform_class_and_compiler_strategies(self): a = Cray() t = a.target('default') self.assertEquals(t.compiler_strategy, 'MODULES') @@ -21,15 +22,15 @@ class ArchitectureTest(unittest.TestCase): self.assertEquals(s.compiler_strategy, 'PATH') def test_sys_type(self): - output_arch_class = sys_type() + output_platform_class = sys_type() my_arch_class = None if os.path.exists('/opt/cray/craype'): - my_arch_class = Cray() + my_platform_class = Cray() elif os.path.exists('/bgsys'): - my_arch_class = Bgq() + my_platform_class = Bgq() elif 'Linux' in platform.system(): - my_arch_class = Linux() -# elif 'Darwin' in platform.system(): -# my_arch_class = Darwin() + my_platform_class = Linux() + elif 'Darwin' in platform.system(): + my_platform_class = Darwin() - self.assertEqual(str(output_arch_class), str(my_arch_class)) + self.assertEqual(str(output_platform_class), str(my_platform_class)) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index f81a2f5af8..2403719134 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -46,8 +46,8 @@ class ConcretizeTest(MockPackagesTest): if abstract.compiler and abstract.compiler.concrete: self.assertEqual(abstract.compiler, concrete.compiler) - if abstract.architecture and abstract.architecture.concrete: - self.assertEqual(abstract.architecture, concrete.architecture) + if abstract.target and abstract.target.concrete: + self.assertEqual(abstract.target, concrete.target) def check_concretize(self, abstract_spec): diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index cd5d9e625e..741dc96072 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -91,21 +91,21 @@ class MultiMethodTest(MockPackagesTest): self.assertEqual(pkg.has_a_default(), 'default') - def test_architecture_match(self): + def test_target_match(self): pkg = spack.db.get('multimethod=x86_64') - self.assertEqual(pkg.different_by_architecture(), 'x86_64') + self.assertEqual(pkg.different_by_target(), 'x86_64') pkg = spack.db.get('multimethod=ppc64') - self.assertEqual(pkg.different_by_architecture(), 'ppc64') + self.assertEqual(pkg.different_by_target(), 'ppc64') pkg = spack.db.get('multimethod=ppc32') - self.assertEqual(pkg.different_by_architecture(), 'ppc32') + self.assertEqual(pkg.different_by_target(), 'ppc32') pkg = spack.db.get('multimethod=arm64') - self.assertEqual(pkg.different_by_architecture(), 'arm64') + self.assertEqual(pkg.different_by_target(), 'arm64') pkg = spack.db.get('multimethod=macos') - self.assertRaises(NoSuchMethodError, pkg.different_by_architecture) + self.assertRaises(NoSuchMethodError, pkg.different_by_target) def test_dependency_match(self): diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 549f829d3e..6a2dd6140f 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -240,10 +240,10 @@ class SpecDagTest(MockPackagesTest): self.assertRaises(spack.spec.UnsatisfiableCompilerSpecError, spec.normalize) - def test_unsatisfiable_architecture(self): + def test_unsatisfiable_target(self): set_pkg_dep('mpileaks', 'mpich=bgqos_0') spec = Spec('mpileaks ^mpich=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') - self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) + self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) def test_invalid_dep(self): diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 6666dbbb52..b8b4fb951c 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -110,7 +110,7 @@ class SpecSematicsTest(MockPackagesTest): self.check_unsatisfiable('foo %gcc@4.7', '%gcc@4.7.3') - def test_satisfies_architecture(self): + def test_satisfies_target(self): self.check_satisfies('foo=chaos_5_x86_64_ib', '=chaos_5_x86_64_ib') self.check_satisfies('foo=bgqos_0', '=bgqos_0') @@ -266,7 +266,7 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain('libelf+debug~foo', 'libelf+debug', 'libelf+debug~foo') - def test_constrain_arch(self): + def test_constrain_target(self): self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0') self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0') diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index 75b1606ffc..f78ef3bb3d 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -101,22 +101,22 @@ class Multimethod(Package): # - # Make sure we can switch methods on different architectures + # Make sure we can switch methods on different target # @when('=x86_64') - def different_by_architecture(self): + def different_by_target(self): return 'x86_64' @when('=ppc64') - def different_by_architecture(self): + def different_by_target(self): return 'ppc64' @when('=ppc32') - def different_by_architecture(self): + def different_by_target(self): return 'ppc32' @when('=arm64') - def different_by_architecture(self): + def different_by_target(self): return 'arm64' -- cgit v1.2.3-60-g2f50 From 83917c4c302851a0d4ff91ef652fdd1b26fb1e08 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 7 Jan 2016 12:43:39 -0800 Subject: Improved target specification --- lib/spack/spack/architecture.py | 2 ++ lib/spack/spack/concretize.py | 38 ++++++++++++++++++++++++++++++++++---- lib/spack/spack/platforms/bgq.py | 2 +- 3 files changed, 37 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index a269767fab..6b9ae33b3e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -93,6 +93,8 @@ class Target(object): return self.__str__() def __str__(self): + if self.platform_name: + return self.platform_name + '-' + self.name return self.name @key_ordering diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index fcd23a6055..ce86786004 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -33,11 +33,13 @@ or user preferences. TODO: make this customizable and allow users to configure concretization policies. """ +from llnl.util.filesystem import join_path import spack import spack.spec import spack.compilers import spack.architecture import spack.error +from spack.util.naming import mod_to_class from spack.version import * from functools import partial from spec import DependencyMap @@ -207,6 +209,36 @@ class DefaultConcretizer(object): return True # Things changed + def class_from_platform_name(self, platform_name): + file_path = join_path(spack.platform_path, platform_name) + platform_mod = imp.load_source('spack.platforms', file_path + '.py') + cls = getattr(platform_mod, mod_to_class(platform_name)) + + return cls + + def spec_add_target_from_string(self, spec, target): + """If only a target is provided, spack will assume the default architecture. + A platform-target pair can be input delimited by a '-'. If either portion of + a platform-target pair is empty, spack will supply a default, in the case of + a blank target the default will be dependent on the platform. + E.g. x86_64 -> 64 bit x86 + bgq- -> default bgq target (back end/powerpc) + cray-hawswell -> haswell target on cray platform + """ + if '-' in target: + platform, target = target.split('-') + else: + platform = '' + + if platform != '': + cls = self.class_from_platform_name(platform) + platform = cls() + else: + platform = spack.architecture.sys_type() + if target != '': + spec.target = platform.target(target) + else: + spec.target = platform.target('default') def concretize_target(self, spec): """If the spec already has an target and it is a an target type, @@ -219,16 +251,14 @@ class DefaultConcretizer(object): if isinstance(spec.target,spack.architecture.Target): return False else: - platform = spack.architecture.sys_type() - spec.target = platform.target(spec.target) + self.spec_add_target_from_string(spec, spec.target) return True #changed if spec.root.target: if isinstance(spec.root.target,spack.architecture.Target): spec.target = spec.root.target else: - platform = spack.architecture.sys_type() - spec.target = platform.target(spec.root.target) + self.spec_add_target_from_string(spec, spec.root.target) else: platform = spack.architecture.sys_type() spec.target = platform.target('default') diff --git a/lib/spack/spack/platforms/bgq.py b/lib/spack/spack/platforms/bgq.py index 988e67023c..6e872d2e72 100644 --- a/lib/spack/spack/platforms/bgq.py +++ b/lib/spack/spack/platforms/bgq.py @@ -9,7 +9,7 @@ class Bgq(Platform): default = 'powerpc' def __init__(self): - super(Bgq, self).__init__('cray') + super(Bgq, self).__init__('bgq') self.add_target(self.front_end, Target(self.front_end, 'PATH')) self.add_target(self.back_end, Target(self.back_end, 'PATH')) -- cgit v1.2.3-60-g2f50 From 53d4f82ce1863d0871f53a7e20b75d73b8d4fad6 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 7 Jan 2016 13:01:05 -0800 Subject: Improved cray architecture class --- lib/spack/spack/platforms/cray.py | 40 -------------------------------- lib/spack/spack/platforms/cray_xc.py | 45 ++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 40 deletions(-) delete mode 100644 lib/spack/spack/platforms/cray.py create mode 100644 lib/spack/spack/platforms/cray_xc.py (limited to 'lib') diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py deleted file mode 100644 index 815a9aea98..0000000000 --- a/lib/spack/spack/platforms/cray.py +++ /dev/null @@ -1,40 +0,0 @@ -import os - -from spack.architecture import Platform, Target - -class Cray(Platform): - priority = 20 - front_end = 'sandybridge' - back_end = 'ivybridge' - default = 'ivybridge' - - def __init__(self): - ''' Since cori doesn't have ivybridge as a front end it's better - if we use CRAY_CPU_TARGET as the default. This will ensure - that if we're on a XC-40 or XC-30 then we can detect the target - ''' - super(Cray, self).__init__('cray') - - # Handle the default here so we can check for a key error - if 'CRAY_CPU_TARGET' in os.environ: - self.default = os.environ['CRAY_CPU_TARGET'] - - # Change the defaults to haswell if we're on an XC40 - if self.default == 'haswell': - self.front_end = self.default - self.back_end = self.default - - # Could switch to use modules and fe targets for front end - # Currently using compilers by path for front end. - self.add_target(self.front_end, Target(self.front_end, 'PATH')) - # Back End compiler needs the proper target module loaded. -# self.add_target(self.back_end, Target(self.front_end, 'MODULES', 'craype-'+ self.back_end)) - self.add_target(self.default, Target(self.default, 'MODULES', 'craype-' + self.default)) - # This is kludgy and the order matters when the targets are all haswell - # This is because the last one overwrites the others when they have the - # same name. - - @classmethod - def detect(self): - return os.path.exists('/opt/cray/craype') - diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py new file mode 100644 index 0000000000..e3adb182ea --- /dev/null +++ b/lib/spack/spack/platforms/cray_xc.py @@ -0,0 +1,45 @@ +import os + +from spack.architecture import Platform, Target + +class CrayXc(Platform): + priority = 20 + front_end = 'sandybridge' + back_end = 'ivybridge' + default = 'ivybridge' + + def __init__(self): + ''' Since cori doesn't have ivybridge as a front end it's better + if we use CRAY_CPU_TARGET as the default. This will ensure + that if we're on a XC-40 or XC-30 then we can detect the target + ''' + super(CrayXc, self).__init__('cray_xc') + + # Handle the default here so we can check for a key error + if 'CRAY_CPU_TARGET' in os.environ: + self.default = os.environ['CRAY_CPU_TARGET'] + + # Change the defaults to haswell if we're on an XC40 + if self.default == 'haswell': + self.front_end = self.default + self.back_end = self.default + + + # Could switch to use modules and fe targets for front end + # Currently using compilers by path for front end. + self.add_target('sandybridge', Target('sandybridge', 'PATH')) + self.add_target('ivybridge', Target('ivybridge', 'MODULES', 'craype-ivybridge')) + self.add_target('haswell', Target('haswell', 'MODULES', 'craype-haswell')) + +# self.add_target(self.front_end, Target(self.front_end, 'PATH')) + # Back End compiler needs the proper target module loaded. +# self.add_target(self.back_end, Target(self.front_end, 'MODULES', 'craype-'+ self.back_end)) +# self.add_target(self.default, Target(self.default, 'MODULES', 'craype-' + self.default)) + # This is kludgy and the order matters when the targets are all haswell + # This is because the last one overwrites the others when they have the + # same name. + + @classmethod + def detect(self): + return os.path.exists('/opt/cray/craype') + -- cgit v1.2.3-60-g2f50 From ba63111f453e258a0231a285dcbb11a876cccc19 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 7 Jan 2016 13:05:58 -0800 Subject: brought the architecture test up to date with new changes --- lib/spack/spack/test/architecture.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 32b16feeb1..cf7938f5d6 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -6,7 +6,7 @@ import os import platform import spack from spack.architecture import * -from spack.platforms.cray import Cray +from spack.platforms.cray_xc import CrayXc from spack.platforms.linux import Linux from spack.platforms.bgq import Bgq from spack.platforms.darwin import Darwin @@ -14,7 +14,7 @@ from spack.platforms.darwin import Darwin class ArchitectureTest(unittest.TestCase): def test_platform_class_and_compiler_strategies(self): - a = Cray() + a = CrayXc() t = a.target('default') self.assertEquals(t.compiler_strategy, 'MODULES') b = Linux() @@ -25,7 +25,7 @@ class ArchitectureTest(unittest.TestCase): output_platform_class = sys_type() my_arch_class = None if os.path.exists('/opt/cray/craype'): - my_platform_class = Cray() + my_platform_class = CrayXc() elif os.path.exists('/bgsys'): my_platform_class = Bgq() elif 'Linux' in platform.system(): -- cgit v1.2.3-60-g2f50 From d3d37ad0cebe4d42d48f5f5f5a53077ae6126631 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 14:29:01 -0800 Subject: removed the submodule as part of reversion of extraneous commits --- lib/spack/spack/util/python_recipe_parser | 1 - 1 file changed, 1 deletion(-) delete mode 160000 lib/spack/spack/util/python_recipe_parser (limited to 'lib') diff --git a/lib/spack/spack/util/python_recipe_parser b/lib/spack/spack/util/python_recipe_parser deleted file mode 160000 index 437a62abb3..0000000000 --- a/lib/spack/spack/util/python_recipe_parser +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 437a62abb3df7212e3ee20269c0089a0a9766fe0 -- cgit v1.2.3-60-g2f50 From aa28e4e81f80a1a388aabe589ca23955ebd9721b Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 15:13:48 -0800 Subject: Improved error messages for compiler_for_spec when either zero or multiple compilers returned. --- lib/spack/spack/compilers/__init__.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 87106282cf..3a0a88b5e3 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -219,7 +219,10 @@ def compiler_for_spec(compiler_spec, target): compilers = [c for c in compilers if c.modules is None] elif target.compiler_strategy == "MODULES": compilers = [c for c in compilers if c.modules is not None] - assert(len(compilers) == 1) + if len(compilers) < 1: + raise NoCompilerForSpecError(compiler_spec, target) + if len(compilers) > 1: + raise CompilerSpecInsufficientlySpecificError(compiler_spec) return compilers[0] @@ -253,3 +256,13 @@ class InvalidCompilerConfigurationError(spack.error.SpackError): class NoCompilersError(spack.error.SpackError): def __init__(self): super(NoCompilersError, self).__init__("Spack could not find any compilers!") + +class NoCompilerForSpecError(spack.error.SpackError): + def __init__(self, compiler_spec, target): + super(NoCompilerForSpecError, self).__init__("No compilers for target %s satisfy spec %s", + compiler_spec, target) + +class CompilerSpecInsufficientlySpecificError(spack.error.SpackError): + def __init__(self, compiler_spec): + super(CompilerSpecInsufficientlySpecificError, self).__init__("Multiple compilers satisfy spec %s", + compiler_spec) -- cgit v1.2.3-60-g2f50 From 2b4dd8b9af427e2fc3c4a743f4f058a26c7d583f Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 16:30:27 -0800 Subject: Fixed target satisfaction and updated tests accordingly --- lib/spack/spack/concretize.py | 35 ++-------------------- lib/spack/spack/spec.py | 35 ++++++++++++++++++++++ lib/spack/spack/test/multimethod.py | 22 +++++--------- lib/spack/spack/test/spec_dag.py | 10 +++++-- lib/spack/spack/test/spec_semantics.py | 41 +++++++++++++++++--------- var/spack/mock_packages/multimethod/package.py | 23 +++++---------- 6 files changed, 86 insertions(+), 80 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index ce86786004..43637ed468 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -209,37 +209,6 @@ class DefaultConcretizer(object): return True # Things changed - def class_from_platform_name(self, platform_name): - file_path = join_path(spack.platform_path, platform_name) - platform_mod = imp.load_source('spack.platforms', file_path + '.py') - cls = getattr(platform_mod, mod_to_class(platform_name)) - - return cls - - def spec_add_target_from_string(self, spec, target): - """If only a target is provided, spack will assume the default architecture. - A platform-target pair can be input delimited by a '-'. If either portion of - a platform-target pair is empty, spack will supply a default, in the case of - a blank target the default will be dependent on the platform. - E.g. x86_64 -> 64 bit x86 - bgq- -> default bgq target (back end/powerpc) - cray-hawswell -> haswell target on cray platform - """ - if '-' in target: - platform, target = target.split('-') - else: - platform = '' - - if platform != '': - cls = self.class_from_platform_name(platform) - platform = cls() - else: - platform = spack.architecture.sys_type() - if target != '': - spec.target = platform.target(target) - else: - spec.target = platform.target('default') - def concretize_target(self, spec): """If the spec already has an target and it is a an target type, return. Otherwise, if it has a target that is a string type, generate a @@ -251,14 +220,14 @@ class DefaultConcretizer(object): if isinstance(spec.target,spack.architecture.Target): return False else: - self.spec_add_target_from_string(spec, spec.target) + spec.add_target_from_string(spec, spec.target) return True #changed if spec.root.target: if isinstance(spec.root.target,spack.architecture.Target): spec.target = spec.root.target else: - self.spec_add_target_from_string(spec, spec.root.target) + spec.add_target_from_string(spec, spec.root.target) else: platform = spack.architecture.sys_type() spec.target = platform.target('default') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 9e011bfb9f..8f90cc0d7f 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1228,6 +1228,36 @@ class Spec(object): return parse_anonymous_spec(spec_like, self.name) + def add_target_from_string(self, target): + """If only a target is provided, spack will assume the default architecture. + A platform-target pair can be input delimited by a '-'. If either portion of + a platform-target pair is empty, spack will supply a default, in the case of + a blank target the default will be dependent on the platform. + E.g. x86_64 -> 64 bit x86 + bgq- -> default bgq target (back end/powerpc) + cray-hawswell -> haswell target on cray platform + """ + if target is None: + return + if '-' in target: + platform, target = target.split('-') + else: + platform = '' + + if platform != '': + # Find the class for the platform name given + file_path = join_path(spack.platform_path, platform_name) + platform_mod = imp.load_source('spack.platforms', file_path + '.py') + cls = getattr(platform_mod, mod_to_class(platform_name)) + platform = cls() + else: + platform = spack.architecture.sys_type() + if target != '': + self.target = platform.target(target) + else: + self.target = platform.target('default') + + def satisfies(self, other, deps=True, strict=False): """Determine if this spec satisfies all constraints of another. @@ -1275,6 +1305,11 @@ class Spec(object): # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. + if not isinstance(self.target, spack.architecture.Target): + self.add_target_from_string(self.target) + if not isinstance(other.target, spack.architecture.Target): + other.add_target_from_string(other.target) + if self.target and other.target: if self.target != other.target: return False diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 741dc96072..c651b7ad4a 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -92,22 +92,16 @@ class MultiMethodTest(MockPackagesTest): def test_target_match(self): - pkg = spack.db.get('multimethod=x86_64') - self.assertEqual(pkg.different_by_target(), 'x86_64') - - pkg = spack.db.get('multimethod=ppc64') - self.assertEqual(pkg.different_by_target(), 'ppc64') - - pkg = spack.db.get('multimethod=ppc32') - self.assertEqual(pkg.different_by_target(), 'ppc32') - - pkg = spack.db.get('multimethod=arm64') - self.assertEqual(pkg.different_by_target(), 'arm64') - - pkg = spack.db.get('multimethod=macos') + platform = spack.architecture.sys_type() + targets = platform.targets.values() + for target in targets[:-1]: + print target + pkg = spack.db.get('multimethod='+target.name) + self.assertEqual(pkg.different_by_target(), target.name) + + pkg = spack.db.get('multimethod='+targets[-1].name) self.assertRaises(NoSuchMethodError, pkg.different_by_target) - def test_dependency_match(self): pkg = spack.db.get('multimethod^zmpi') self.assertEqual(pkg.different_by_dep(), 'zmpi') diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 6a2dd6140f..d116454d5c 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -241,9 +241,13 @@ class SpecDagTest(MockPackagesTest): def test_unsatisfiable_target(self): - set_pkg_dep('mpileaks', 'mpich=bgqos_0') - spec = Spec('mpileaks ^mpich=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') - self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) + platform = spack.architecture.sys_type() + if len(platform.targets) > 1: + first = platform.targets.values()[0].name + second = platform.targets.values()[1].name + set_pkg_dep('mpileaks', 'mpich='+first) + spec = Spec('mpileaks ^mpich='+ second +' ^callpath ^dyninst ^libelf ^libdwarf') + self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) def test_invalid_dep(self): diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index b8b4fb951c..ef4db3fe65 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -111,12 +111,13 @@ class SpecSematicsTest(MockPackagesTest): def test_satisfies_target(self): - self.check_satisfies('foo=chaos_5_x86_64_ib', '=chaos_5_x86_64_ib') - self.check_satisfies('foo=bgqos_0', '=bgqos_0') - - self.check_unsatisfiable('foo=bgqos_0', '=chaos_5_x86_64_ib') - self.check_unsatisfiable('foo=chaos_5_x86_64_ib', '=bgqos_0') + platform = spack.architecture.sys_type() + targets = platform.targets.values() + for target in targets: + self.check_satisfies('foo='+target.name, '='+target.name) + for i in range(1,len(targets)): + self.check_unsatisfiable('foo='+targets[i-1].name, '='+targets[i].name) def test_satisfies_dependencies(self): self.check_satisfies('mpileaks^mpich', '^mpich') @@ -267,13 +268,15 @@ class SpecSematicsTest(MockPackagesTest): def test_constrain_target(self): - self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0') - self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0') + platform = spack.architecture.sys_type() + target = platform.target('default').name + self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) + self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) def test_constrain_compiler(self): - self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0') - self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0') + self.check_constrain('libelf%intel', 'libelf%intel', 'libelf%intel') + self.check_constrain('libelf%intel', 'libelf', 'libelf%intel') def test_invalid_constraint(self): @@ -283,7 +286,10 @@ class SpecSematicsTest(MockPackagesTest): self.check_invalid_constraint('libelf+debug', 'libelf~debug') self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') - self.check_invalid_constraint('libelf=bgqos_0', 'libelf=x86_54') + platform = spack.architecture.sys_type() + targets = platform.targets.values() + if len(targets) > 1: + self.check_invalid_constraint('libelf='+targets[0].name, 'libelf='+targets[1].name) def test_constrain_changed(self): @@ -293,7 +299,8 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_changed('libelf%gcc', '%gcc@4.5') self.check_constrain_changed('libelf', '+debug') self.check_constrain_changed('libelf', '~debug') - self.check_constrain_changed('libelf', '=bgqos_0') + platform = spack.architecture.sys_type() + self.check_constrain_changed('libelf', '='+platform.target('default').name) def test_constrain_not_changed(self): @@ -304,7 +311,9 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_not_changed('libelf%gcc@4.5', '%gcc@4.5') self.check_constrain_not_changed('libelf+debug', '+debug') self.check_constrain_not_changed('libelf~debug', '~debug') - self.check_constrain_not_changed('libelf=bgqos_0', '=bgqos_0') + platform = spack.architecture.sys_type() + default = platform.target('default').name + self.check_constrain_not_changed('libelf='+default, '='+default) self.check_constrain_not_changed('libelf^foo', 'libelf^foo') self.check_constrain_not_changed('libelf^foo^bar', 'libelf^foo^bar') @@ -316,7 +325,9 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_changed('libelf^foo%gcc', 'libelf^foo%gcc@4.5') self.check_constrain_changed('libelf^foo', 'libelf^foo+debug') self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') - self.check_constrain_changed('libelf^foo', 'libelf^foo=bgqos_0') + platform = spack.architecture.sys_type() + default = platform.target('default').name + self.check_constrain_changed('libelf^foo', 'libelf^foo='+default) def test_constrain_dependency_not_changed(self): @@ -326,5 +337,7 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_not_changed('libelf^foo%gcc@4.5', 'libelf^foo%gcc@4.5') self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') - self.check_constrain_not_changed('libelf^foo=bgqos_0', 'libelf^foo=bgqos_0') + platform = spack.architecture.sys_type() + default = platform.target('default').name + self.check_constrain_not_changed('libelf^foo='+default, 'libelf^foo='+default) diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index f78ef3bb3d..0a1e991d37 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * - +import spack.architecture class Multimethod(Package): """This package is designed for use with Spack's multimethod test. @@ -103,21 +103,12 @@ class Multimethod(Package): # # Make sure we can switch methods on different target # - @when('=x86_64') - def different_by_target(self): - return 'x86_64' - - @when('=ppc64') - def different_by_target(self): - return 'ppc64' - - @when('=ppc32') - def different_by_target(self): - return 'ppc32' - - @when('=arm64') - def different_by_target(self): - return 'arm64' + platform = spack.architecture.sys_type() + targets = platform.targets.values() + for target in targets[:-1]: + @when('='+target.name) + def different_by_target(self): + return self.spec.target.name # -- cgit v1.2.3-60-g2f50 From 2b2d4bae4e48151ae17c0f2f0514873b1247ff15 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 17:38:37 -0800 Subject: Fixing multimethod test for new platforms --- lib/spack/spack/test/multimethod.py | 6 +++++- var/spack/mock_packages/multimethod/package.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index c651b7ad4a..116cb2d2ff 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -100,7 +100,11 @@ class MultiMethodTest(MockPackagesTest): self.assertEqual(pkg.different_by_target(), target.name) pkg = spack.db.get('multimethod='+targets[-1].name) - self.assertRaises(NoSuchMethodError, pkg.different_by_target) + if len(targets) == 1: + self.assertEqual(pkg.different_by_target(), targets[-1].name) + else: + self.assertRaises(NoSuchMethodError, pkg.different_by_target) + def test_dependency_match(self): pkg = spack.db.get('multimethod^zmpi') diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index 0a1e991d37..4e9a2438a7 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -22,6 +22,9 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import imp +from llnl.util.filesystem import join_path +from spack.util.naming import mod_to_class from spack import * import spack.architecture @@ -103,8 +106,15 @@ class Multimethod(Package): # # Make sure we can switch methods on different target # +# for platform_name in ['cray_xc', 'darwin', 'linux']: +# file_path = join_path(spack.platform_path, platform_name) +# platform_mod = imp.load_source('spack.platforms', file_path + '.py') +# cls = getattr(platform_mod, mod_to_class(platform_name)) + +# platform = cls() platform = spack.architecture.sys_type() targets = platform.targets.values() + for target in targets[:-1]: @when('='+target.name) def different_by_target(self): -- cgit v1.2.3-60-g2f50 From 6e5238d037aebe4e610a16f200d860cee1a1355d Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 17:44:16 -0800 Subject: Fixing the fix and removing debug printing --- lib/spack/spack/test/multimethod.py | 1 - var/spack/mock_packages/multimethod/package.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 116cb2d2ff..9d9f39e87d 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -95,7 +95,6 @@ class MultiMethodTest(MockPackagesTest): platform = spack.architecture.sys_type() targets = platform.targets.values() for target in targets[:-1]: - print target pkg = spack.db.get('multimethod='+target.name) self.assertEqual(pkg.different_by_target(), target.name) diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index 4e9a2438a7..b37db9f1eb 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -114,8 +114,10 @@ class Multimethod(Package): # platform = cls() platform = spack.architecture.sys_type() targets = platform.targets.values() + if len(targets) > 1: + targets = targets[:-1] - for target in targets[:-1]: + for target in targets: @when('='+target.name) def different_by_target(self): return self.spec.target.name -- cgit v1.2.3-60-g2f50 From 41046499c69150a7a81518f2ffba3405de0d6577 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 18:51:38 -0800 Subject: minor bug chasing --- lib/spack/env/cc | 16 ---------------- lib/spack/spack/compilers/__init__.py | 4 ++-- lib/spack/spack/concretize.py | 4 ++-- 3 files changed, 4 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/spack/env/cc b/lib/spack/env/cc index b6c6e03e42..1704ae3280 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -71,22 +71,6 @@ for param in $parameters; do done -# -# Cray module environment-related stuff. -# -if [ ! -z "$SPACK_CRAYPE" ]; then - cur_pe=$(module list 2>&1 | grep PrgEnv | grep -o 'PrgEnv-[^/]*') - if [ ! -z "$cur_pe" ]; then - module swap $cur_pe $SPACK_CRAYPE - else - module load $SPACK_CRAYPE - fi -fi - -if [ ! -z "$SPACK_COMP_MODULE" ]; then - module load $SPACK_COMP_MODULE -fi - # # Figure out the type of compiler, the language, and the mode so that # the compiler script knows what to do. diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 3a0a88b5e3..e38ef949d7 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -259,8 +259,8 @@ class NoCompilersError(spack.error.SpackError): class NoCompilerForSpecError(spack.error.SpackError): def __init__(self, compiler_spec, target): - super(NoCompilerForSpecError, self).__init__("No compilers for target %s satisfy spec %s", - compiler_spec, target) + super(NoCompilerForSpecError, self).__init__("No compilers for target %s satisfy spec %s" % ( + target, compiler_spec)) class CompilerSpecInsufficientlySpecificError(spack.error.SpackError): def __init__(self, compiler_spec): diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 43637ed468..b8057cd46c 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -220,14 +220,14 @@ class DefaultConcretizer(object): if isinstance(spec.target,spack.architecture.Target): return False else: - spec.add_target_from_string(spec, spec.target) + spec.add_target_from_string(spec.target) return True #changed if spec.root.target: if isinstance(spec.root.target,spack.architecture.Target): spec.target = spec.root.target else: - spec.add_target_from_string(spec, spec.root.target) + spec.add_target_from_string(spec.root.target) else: platform = spack.architecture.sys_type() spec.target = platform.target('default') -- cgit v1.2.3-60-g2f50 From 9615efd940c425b049a142f8c22545c9a04124df Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 18:53:19 -0800 Subject: cleanup: Removing an unnecessary line in an otherwise unchanged file --- lib/spack/env/cc | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 1704ae3280..fa85bb595e 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -70,7 +70,6 @@ for param in $parameters; do fi done - # # Figure out the type of compiler, the language, and the mode so that # the compiler script knows what to do. -- cgit v1.2.3-60-g2f50 From aab1a67d056bf9f0979f2cebfff607d05ac8b7e9 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 9 Jan 2016 09:42:36 -0800 Subject: Adding needed import modules. Changed platform_name to platform to prevent syntax errors --- lib/spack/spack/spec.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 8f90cc0d7f..264e0c0506 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -91,6 +91,7 @@ specs to avoid ambiguity. Both are provided because ~ can cause shell expansion when it is the first character in an id typed on the command line. """ import sys +import imp import itertools import hashlib import base64 @@ -100,6 +101,7 @@ from external import yaml from external.yaml.error import MarkedYAMLError import llnl.util.tty as tty +from llnl.util.filesystem import join_path from llnl.util.lang import * from llnl.util.tty.color import * @@ -109,6 +111,7 @@ import spack.error import spack.compilers as compilers from spack.version import * +from spack.util.naming import mod_to_class from spack.util.string import * from spack.util.prefix import Prefix from spack.virtual import ProviderIndex @@ -1246,9 +1249,9 @@ class Spec(object): if platform != '': # Find the class for the platform name given - file_path = join_path(spack.platform_path, platform_name) + file_path = join_path(spack.platform_path, platform) platform_mod = imp.load_source('spack.platforms', file_path + '.py') - cls = getattr(platform_mod, mod_to_class(platform_name)) + cls = getattr(platform_mod, mod_to_class(platform)) platform = cls() else: platform = spack.architecture.sys_type() -- cgit v1.2.3-60-g2f50 From 15713219e57a97048f4e3b085f9c75e188a67787 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 20 Jan 2016 10:32:56 -0800 Subject: Better regular expression searching. Tested on edison was finding test compilers and then spack was giving multiple compiler match errors --- lib/spack/spack/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 225c65917a..1a5be3c3fe 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -283,7 +283,7 @@ class Compiler(object): modulecmd = which('modulecmd') modulecmd.add_default_arg('python') output = modulecmd('avail', cls.PrgEnv_compiler, return_oe=True) - matches = re.findall(r'(%s)/(\d+[\.\d]+)' % cls.PrgEnv_compiler, output) + matches = re.findall(r'(%s)/([\d\.]+[\d$])' % cls.PrgEnv_compiler, output) for name, version in matches: v = version -- cgit v1.2.3-60-g2f50 From 840b41c450267764d021456875dbe87af52c2176 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 20 Jan 2016 10:36:15 -0800 Subject: Removed unneccessary $. Still learning regexp --- lib/spack/spack/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 1a5be3c3fe..46f2dfaec7 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -283,7 +283,7 @@ class Compiler(object): modulecmd = which('modulecmd') modulecmd.add_default_arg('python') output = modulecmd('avail', cls.PrgEnv_compiler, return_oe=True) - matches = re.findall(r'(%s)/([\d\.]+[\d$])' % cls.PrgEnv_compiler, output) + matches = re.findall(r'(%s)/([\d\.]+[\d])' % cls.PrgEnv_compiler, output) for name, version in matches: v = version -- cgit v1.2.3-60-g2f50 From 03585225338e2ccd61174800b42242c7f256f533 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 6 Feb 2016 15:41:22 -0800 Subject: new branch and also pushing some architecture changes where os is detected by linux and darwin and manually set by cray and bgq --- lib/spack/spack/architecture.py | 136 +++++++++++++++++++++++++--------------- 1 file changed, 84 insertions(+), 52 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 6b9ae33b3e..4f36a3fa92 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -39,20 +39,25 @@ from external import yaml class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, self).__init__( + "Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") + super(NoSysTypeError, self).__init__( + "Could not determine sys_type for this machine.") @key_ordering class Target(object): - """ Target is the processor of the host machine. The host machine may have different front-end - and back-end targets, especially if it is a Cray machine. The target will have a name and - also the module_name (e.g craype-compiler). Targets will also recognize which platform - they came from using the set_platform method. Targets will have compiler finding strategies + """ Target is the processor of the host machine. + The host machine may have different front-end + and back-end targets, especially if it is a Cray machine. + The target will have a name and module_name (e.g craype-compiler). + Targets will also recognize which platform + they came from using the set_platform method. + Targets will have compiler finding strategies """ def __init__(self, name, compiler_strategy, module_name=None): @@ -60,10 +65,13 @@ class Target(object): self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge - # Sets only the platform name to avoid recursiveness + # Sets only the platform name to avoid recursiveness def set_platform(self, platform): self.platform_name = platform.name + def set_operating_system(self, operating_sys): + self.platform_os = operating_sys + def to_dict(self): d = {} d['name'] = self.name @@ -71,6 +79,7 @@ class Target(object): d['module_name'] = self.module_name if self.platform_name: d['platform'] = self.platform_name + return d @staticmethod @@ -87,14 +96,16 @@ class Target(object): def _cmp_key(self): - return (self.name, self.compiler_strategy, self.module_name) + return (self.name, self.compiler_strategy, + self.module_name, self.platform_os) def __repr__(self): return self.__str__() def __str__(self): - if self.platform_name: - return self.platform_name + '-' + self.name + if self.platform_name and self.platform_os: + return (self.platform_name + '-' + + self.platform_os + '-' + self.name) return self.name @key_ordering @@ -105,26 +116,38 @@ class Platform(object): """ priority = None # Subclass needs to set this number. This controls order in which platform is detected. + front_end = None back_end = None default = None # The default back end target. On cray ivybridge + + front_os = None + back_os = None + default_os = None def __init__(self, name): self.targets = {} self.name = name def add_target(self, name, target): - """Used by the platform specific subclass to list available targets. Raises an error - if the platform specifies a name that is reserved by spack as an alias. + """Used by the platform specific subclass to list available targets. + Raises an error if the platform specifies a name + that is reserved by spack as an alias. """ if name in ['front_end', 'fe', 'back_end', 'be', 'default']: - raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) + raise ValueError( + "%s is a spack reserved" \ + "alias and cannot be the name of a target" + % name) + + target.set_operating_system(self.platform_os()) target.set_platform(self) self.targets[name] = target def target(self, name): - """This is a getter method for the target dictionary that handles defaulting based - on the values provided by default, front-end, and back-end. This can be overwritten + """This is a getter method for the target dictionary that + handles defaulting based on the values provided by default, + front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ if name == 'default': @@ -135,6 +158,51 @@ class Platform(object): name = self.back_end return self.targets[name] + + def _detect_linux_os(self): + """ If it is one a linux machine use the python method platform.dist() + """ + os_name = py_platform.dist()[0] + version = py_platform.dist()[1] + a return os_name + version + + def _detect_mac_os(self): + """If it is on a mac machine then use the python method platform.mac_ver + """ + mac_releases = {'10.6' : 'snowleopard', '10.7' : 'lion', + '10.8' : 'mountainlion', '10.9' : 'mavericks', + '10.10' : 'yosemite', '10.11' : 'elcapitan'} + + mac_ver = py_platform.mac_ver() + try: + os_name = mac_releases[mac_ver] + mac_ver = Version(mac_ver) + + except KeyError: + os_name = 'mac_os' + + return os_name + + def set_os(self): + """ Set the OS according to the platform it is on. Darwin and Linux + will simply be an auto-detected linux distro or mac release. The + special cases will be for Cray and BGQ machines which have two + different OS for login and compute nodes. The client should provide + the name and major version of the operating system + """ + if self.name == 'darwin': + self.default_os = self._detect_mac_os() + else: + self.default_os = self._detect_linux_os() + + def platform_os(self, name=None): + """ Get the platform operating system from the platform """ + if name == 'front_os': + return self.front_os + elif name == 'back_os': + return self.back_os + else: + return self.default_os @classmethod def detect(self): @@ -144,6 +212,7 @@ class Platform(object): """ raise NotImplementedError() + def __repr__(self): return self.__str__() @@ -153,43 +222,6 @@ class Platform(object): def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values())) -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" - if not hasattr(spack, "sys_type"): - return None - elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() #If in __init__.py there is a sys_type() then call that - else: - return spack.sys_type # Else use the attributed which defaults to None - - -# This is livermore dependent. Hard coded for livermore -#def get_sys_type_from_environment(): -# """Return $SYS_TYPE or None if it's not defined.""" -# return os.environ.get('SYS_TYPE') - - -def get_mac_sys_type(): - """Return a Mac OS SYS_TYPE or None if this isn't a mac. - Front-end config - """ - mac_ver = py_platform.mac_ver()[0] - if not mac_ver: - return None - return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine()) - - -def get_sys_type_from_uname(): - """ Returns a sys_type from the uname argument - Front-end config - """ - try: - platform_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) - platform, _ = platform_proc.communicate() - return platform.strip() - except: - return None - @memoized def all_platforms(): modules = [] -- cgit v1.2.3-60-g2f50 From 21a5a3404134c1676a7aa498ec201ed911e80029 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 10:59:16 -0800 Subject: Revert "new branch and also pushing some architecture changes where os is detected by linux and darwin and manually set by cray and bgq" This reverts commit 70088be24b2ed34076f7f5292a2a465a8655a886. --- lib/spack/spack/architecture.py | 136 +++++++++++++++------------------------- 1 file changed, 52 insertions(+), 84 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 4f36a3fa92..6b9ae33b3e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -39,25 +39,20 @@ from external import yaml class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__( - "Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__( - "Could not determine sys_type for this machine.") + super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") @key_ordering class Target(object): - """ Target is the processor of the host machine. - The host machine may have different front-end - and back-end targets, especially if it is a Cray machine. - The target will have a name and module_name (e.g craype-compiler). - Targets will also recognize which platform - they came from using the set_platform method. - Targets will have compiler finding strategies + """ Target is the processor of the host machine. The host machine may have different front-end + and back-end targets, especially if it is a Cray machine. The target will have a name and + also the module_name (e.g craype-compiler). Targets will also recognize which platform + they came from using the set_platform method. Targets will have compiler finding strategies """ def __init__(self, name, compiler_strategy, module_name=None): @@ -65,13 +60,10 @@ class Target(object): self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge - # Sets only the platform name to avoid recursiveness + # Sets only the platform name to avoid recursiveness def set_platform(self, platform): self.platform_name = platform.name - def set_operating_system(self, operating_sys): - self.platform_os = operating_sys - def to_dict(self): d = {} d['name'] = self.name @@ -79,7 +71,6 @@ class Target(object): d['module_name'] = self.module_name if self.platform_name: d['platform'] = self.platform_name - return d @staticmethod @@ -96,16 +87,14 @@ class Target(object): def _cmp_key(self): - return (self.name, self.compiler_strategy, - self.module_name, self.platform_os) + return (self.name, self.compiler_strategy, self.module_name) def __repr__(self): return self.__str__() def __str__(self): - if self.platform_name and self.platform_os: - return (self.platform_name + '-' + - self.platform_os + '-' + self.name) + if self.platform_name: + return self.platform_name + '-' + self.name return self.name @key_ordering @@ -116,38 +105,26 @@ class Platform(object): """ priority = None # Subclass needs to set this number. This controls order in which platform is detected. - front_end = None back_end = None default = None # The default back end target. On cray ivybridge - - front_os = None - back_os = None - default_os = None def __init__(self, name): self.targets = {} self.name = name def add_target(self, name, target): - """Used by the platform specific subclass to list available targets. - Raises an error if the platform specifies a name - that is reserved by spack as an alias. + """Used by the platform specific subclass to list available targets. Raises an error + if the platform specifies a name that is reserved by spack as an alias. """ if name in ['front_end', 'fe', 'back_end', 'be', 'default']: - raise ValueError( - "%s is a spack reserved" \ - "alias and cannot be the name of a target" - % name) - - target.set_operating_system(self.platform_os()) + raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) target.set_platform(self) self.targets[name] = target def target(self, name): - """This is a getter method for the target dictionary that - handles defaulting based on the values provided by default, - front-end, and back-end. This can be overwritten + """This is a getter method for the target dictionary that handles defaulting based + on the values provided by default, front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ if name == 'default': @@ -158,51 +135,6 @@ class Platform(object): name = self.back_end return self.targets[name] - - def _detect_linux_os(self): - """ If it is one a linux machine use the python method platform.dist() - """ - os_name = py_platform.dist()[0] - version = py_platform.dist()[1] - a return os_name + version - - def _detect_mac_os(self): - """If it is on a mac machine then use the python method platform.mac_ver - """ - mac_releases = {'10.6' : 'snowleopard', '10.7' : 'lion', - '10.8' : 'mountainlion', '10.9' : 'mavericks', - '10.10' : 'yosemite', '10.11' : 'elcapitan'} - - mac_ver = py_platform.mac_ver() - try: - os_name = mac_releases[mac_ver] - mac_ver = Version(mac_ver) - - except KeyError: - os_name = 'mac_os' - - return os_name - - def set_os(self): - """ Set the OS according to the platform it is on. Darwin and Linux - will simply be an auto-detected linux distro or mac release. The - special cases will be for Cray and BGQ machines which have two - different OS for login and compute nodes. The client should provide - the name and major version of the operating system - """ - if self.name == 'darwin': - self.default_os = self._detect_mac_os() - else: - self.default_os = self._detect_linux_os() - - def platform_os(self, name=None): - """ Get the platform operating system from the platform """ - if name == 'front_os': - return self.front_os - elif name == 'back_os': - return self.back_os - else: - return self.default_os @classmethod def detect(self): @@ -212,7 +144,6 @@ class Platform(object): """ raise NotImplementedError() - def __repr__(self): return self.__str__() @@ -222,6 +153,43 @@ class Platform(object): def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values())) +def get_sys_type_from_spack_globals(): + """Return the SYS_TYPE from spack globals, or None if it isn't set.""" + if not hasattr(spack, "sys_type"): + return None + elif hasattr(spack.sys_type, "__call__"): + return spack.sys_type() #If in __init__.py there is a sys_type() then call that + else: + return spack.sys_type # Else use the attributed which defaults to None + + +# This is livermore dependent. Hard coded for livermore +#def get_sys_type_from_environment(): +# """Return $SYS_TYPE or None if it's not defined.""" +# return os.environ.get('SYS_TYPE') + + +def get_mac_sys_type(): + """Return a Mac OS SYS_TYPE or None if this isn't a mac. + Front-end config + """ + mac_ver = py_platform.mac_ver()[0] + if not mac_ver: + return None + return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine()) + + +def get_sys_type_from_uname(): + """ Returns a sys_type from the uname argument + Front-end config + """ + try: + platform_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + platform, _ = platform_proc.communicate() + return platform.strip() + except: + return None + @memoized def all_platforms(): modules = [] -- cgit v1.2.3-60-g2f50 From 2650c60374a0857679d52f98fc5f94b23c7d6660 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 16:14:50 -0800 Subject: Added operating system class prototype and some autodetect features for operating system --- lib/spack/spack/architecture.py | 61 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 6b9ae33b3e..2914d36ba2 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -109,8 +109,13 @@ class Platform(object): back_end = None default = None # The default back end target. On cray ivybridge + front_os = None + back_os = None + default_os = None + def __init__(self, name): self.targets = {} + self.operating_sys = {} self.name = name def add_target(self, name, target): @@ -136,10 +141,49 @@ class Platform(object): return self.targets[name] + def _detect_linux_os(self): + return OperatingSystem(py_platform.dist()[0], py_platform.dist()[1]) + + def _detect_mac_os(self): + mac_releases = {'10.6': "snowleopard", + "10.7": "lion", + "10.8": "mountainlion", + "10.9": "mavericks", + "10.10": "yosemite", + "10.11": "elcapitan"} + mac_ver = py_platform.mac_ver()[:-2] + try: + os_name = mac_releases[mac_ver] + return OperatingSystem(os_name, mac_ver) + except KeyError: + os_name = "mac_os" + return OperatingSystem(os_name, mac_ver) + + def add_operating_system(self, name=None, operating_system=None): + if self.name == 'linux': + linux_os = self._detect_linux_os() + self.operating_sys[linux_os.name] = linux_os + elif self.name == 'darwin': + mac_os = self._detect_mac_os() + self.operating_sys[mac_os.name] = mac_os + else: + self.operating_sys[name] = operating_system + + def operating_system(self, name): + if name == 'default': + name = self.default_os + if name == 'front_os': + name = self.front_os + if name == 'back_os': + name = self.back_os + + return self.operating_sys[name] + @classmethod def detect(self): """ Subclass is responsible for implementing this method. - Returns True if the Platform class detects that it is the current platform + Returns True if the Platform class detects that + it is the current platform and False if it's not. """ raise NotImplementedError() @@ -153,6 +197,21 @@ class Platform(object): def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values())) + +class OperatingSystem(object): + """ The operating system will contain a name and version. It will + also represent itself simple with it's name + """ + def __init__(self, name, version): + self.name = name + self.version = version + + def __str__(self): + return self.name + + def __repr__(self): + return self.__str__() + def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): -- cgit v1.2.3-60-g2f50 From d0ae6dd40159c1e3b043db254f56c8b7ca22294c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 16:15:25 -0800 Subject: Changed add_target_from_string to include methods to concretize operating_system --- lib/spack/spack/spec.py | 64 ++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 264e0c0506..503f80e1e6 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -90,6 +90,7 @@ thing. Spack uses ~variant in directory names and in the canonical form of specs to avoid ambiguity. Both are provided because ~ can cause shell expansion when it is the first character in an id typed on the command line. """ +from collections import namedtuple import sys import imp import itertools @@ -459,7 +460,7 @@ class Spec(object): """Called by the parser to set the target.""" if self.target: raise DuplicateTargetError( "Spec for '%s' cannot have two targets." % self.name) - self.target = target + self.target = target # a string can be set def _add_dependency(self, spec): @@ -1231,7 +1232,7 @@ class Spec(object): return parse_anonymous_spec(spec_like, self.name) - def add_target_from_string(self, target): + def add_target_from_string(self, arch): """If only a target is provided, spack will assume the default architecture. A platform-target pair can be input delimited by a '-'. If either portion of a platform-target pair is empty, spack will supply a default, in the case of @@ -1240,31 +1241,28 @@ class Spec(object): bgq- -> default bgq target (back end/powerpc) cray-hawswell -> haswell target on cray platform """ - if target is None: + Arch = namedtuple("Arch", "arch_os target") + platform = spack.architecture.sys_type() + + if arch is None: return - if '-' in target: - platform, target = target.split('-') - else: - platform = '' - - if platform != '': - # Find the class for the platform name given - file_path = join_path(spack.platform_path, platform) - platform_mod = imp.load_source('spack.platforms', file_path + '.py') - cls = getattr(platform_mod, mod_to_class(platform)) - platform = cls() - else: - platform = spack.architecture.sys_type() - if target != '': - self.target = platform.target(target) - else: - self.target = platform.target('default') + if '-' in arch: + os_name, target = arch.split('-') + self.target = Arch(arch_os=platform.operating_system(os_name), + target = platform.target(target)) + elif arch in platform.targets: + self.target = Arch(arch_os=platform.operating_system('default'), + target=platform.target(target)) + else: + os_name = arch # Odd naming here, definitely need to change + self.target = Arch(arch_os=platform.operating_system(os_name), + target=platform.target('default')) def satisfies(self, other, deps=True, strict=False): - """Determine if this spec satisfies all constraints of another. + """determine if this spec satisfies all constraints of another. - There are two senses for satisfies: + there are two senses for satisfies: * `loose` (default): the absence of a constraint in self implies that it *could* be satisfied by other, so we only @@ -1276,32 +1274,32 @@ class Spec(object): """ other = self._autospec(other) - # A concrete provider can satisfy a virtual dependency. + # a concrete provider can satisfy a virtual dependency. if not self.virtual and other.virtual: pkg = spack.db.get(self.name) if pkg.provides(other.name): for provided, when_spec in pkg.provided.items(): - if self.satisfies(when_spec, deps=False, strict=strict): + if self.satisfies(when_spec, deps=false, strict=strict): if provided.satisfies(other): - return True - return False + return True + return False - # Otherwise, first thing we care about is whether the name matches + # otherwise, first thing we care about is whether the name matches if self.name != other.name: - return False + return False if self.versions and other.versions: if not self.versions.satisfies(other.versions, strict=strict): - return False + return False elif strict and (self.versions or other.versions): - return False + return False - # None indicates no constraints when not strict. + # none indicates no constraints when not strict. if self.compiler and other.compiler: if not self.compiler.satisfies(other.compiler, strict=strict): - return False + return False elif strict and (other.compiler and not self.compiler): - return False + return True if not self.variants.satisfies(other.variants, strict=strict): return False -- cgit v1.2.3-60-g2f50 From fb234205c210930112f7fcd2c7c360e0c950d042 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 16:16:11 -0800 Subject: Added method to concretize target and os pair --- lib/spack/spack/concretize.py | 79 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index b8057cd46c..d78c102d93 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -33,6 +33,7 @@ or user preferences. TODO: make this customizable and allow users to configure concretization policies. """ +import collections from llnl.util.filesystem import join_path import spack import spack.spec @@ -209,6 +210,22 @@ class DefaultConcretizer(object): return True # Things changed + def _concretize_operating_system(self, arch, platform): + """ Future method for concretizing operating system """ + if isinstance(arch.arch_os, OperatingSystem): + return False + else: + arch.arch_os = platform.operating_system('default') + return True + + + def _concretize_arch_target(self, arch, platform): + if isinstance(arch.target, spack.architecture.Target): + return False + else: + arch.target = platform.target('default') + return True + def concretize_target(self, spec): """If the spec already has an target and it is a an target type, return. Otherwise, if it has a target that is a string type, generate a @@ -216,23 +233,53 @@ class DefaultConcretizer(object): DAG has an target, then use that. Otherwise, take the system's default target. """ - if spec.target is not None: - if isinstance(spec.target,spack.architecture.Target): - return False - else: - spec.add_target_from_string(spec.target) - return True #changed - - if spec.root.target: - if isinstance(spec.root.target,spack.architecture.Target): - spec.target = spec.root.target - else: - spec.add_target_from_string(spec.root.target) + platform = spack.architecture.sys_type() + + if spec.target is None: + # Create an empty tuple + Arch = collections.namedtuple("Arch", "arch_os target") + spec.target = Arch(arch_os=platform.operating_system('default'), + target=platform.target('default')) + + return True + # If there is a target and it is a tuple and has both filled return + # False + if not isinstance(spec.target, basestring): + return any((self._concretize_operating_system(spec.target, platform), + self._concretize_arch_target(spec.target, plarform))) else: - platform = spack.architecture.sys_type() - spec.target = platform.target('default') - - return True #changed + spec.add_target_from_string(spec.target) + + if spec.root.target and \ + not isinstance(spec.root.target, basestring): + bool_flag = any( + (self._concretize_operating_system(spec.root.target, platform), + self._concretize_arch_target(spec.root.target, platform))) + spec.target =spec.root.target + return bool_flag + else: + spec.add_target_from_string(spec.root.target) + + + # if there is no target specified used the defaults + + #if spec.target is not None: + # if isinstance(spec.target,spack.architecture.Target): + # return False + # else: + # spec.add_target_from_string(spec.target) + # return True #changed + + #if spec.root.target: + # if isinstance(spec.root.target,spack.architecture.Target): + # spec.target = spec.root.target + # else: + # spec.add_target_from_string(spec.root.target) + #else: + # platform = spack.architecture.sys_type() + # spec.target = platform.target('default') + + #return True #changed def concretize_variants(self, spec): -- cgit v1.2.3-60-g2f50 From cb8d5ab1837cfddefa31bd09592ca4ac999692dc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 16:16:35 -0800 Subject: Added new add_os method --- lib/spack/spack/platforms/cray_xc.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index e3adb182ea..b37d36a62c 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -1,6 +1,6 @@ import os -from spack.architecture import Platform, Target +from spack.architecture import Platform, Target, OperatingSystem class CrayXc(Platform): priority = 20 @@ -8,6 +8,10 @@ class CrayXc(Platform): back_end = 'ivybridge' default = 'ivybridge' + front_os = "SuSE11" + back_os = "GNU/Linux" + default_os = "GNU/Linux" + def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better if we use CRAY_CPU_TARGET as the default. This will ensure @@ -30,6 +34,9 @@ class CrayXc(Platform): self.add_target('sandybridge', Target('sandybridge', 'PATH')) self.add_target('ivybridge', Target('ivybridge', 'MODULES', 'craype-ivybridge')) self.add_target('haswell', Target('haswell', 'MODULES', 'craype-haswell')) + + self.add_operating_system('SuSE11', OperatingSystem('SuSE11', '11')) + self.add_operating_system('GNU/Linux', OperatingSystem('GNU/Linux', '10')) # self.add_target(self.front_end, Target(self.front_end, 'PATH')) # Back End compiler needs the proper target module loaded. -- cgit v1.2.3-60-g2f50 From 77e93e1b791f1c264a999de3b59f2e6396e13391 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 16:16:58 -0800 Subject: Added add_os method to platform subclasses --- lib/spack/spack/platforms/darwin.py | 1 + lib/spack/spack/platforms/linux.py | 1 + 2 files changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 52284826b1..cfa1d957a1 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -10,6 +10,7 @@ class Darwin(Platform): def __init__(self): super(Darwin, self).__init__('darwin') self.add_target(self.default, Target(self.default, 'PATH')) + self.add_operating_system() @classmethod def detect(self): diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 7f94d80c34..dcb08569a8 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -10,6 +10,7 @@ class Linux(Platform): def __init__(self): super(Linux, self).__init__('linux') self.add_target(self.default, Target(self.default, 'PATH')) + self.add_operating_system() @classmethod def detect(self): -- cgit v1.2.3-60-g2f50 From b9d09202c9856bf173c4bbabd4b73573d355d408 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 11 Feb 2016 11:15:19 -0800 Subject: changed some variables to account for the fact that target is now a tuple --- lib/spack/spack/concretize.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index d78c102d93..8886b3cd93 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -212,7 +212,7 @@ class DefaultConcretizer(object): def _concretize_operating_system(self, arch, platform): """ Future method for concretizing operating system """ - if isinstance(arch.arch_os, OperatingSystem): + if isinstance(arch.arch_os, spack.architecture.OperatingSystem): return False else: arch.arch_os = platform.operating_system('default') @@ -240,13 +240,12 @@ class DefaultConcretizer(object): Arch = collections.namedtuple("Arch", "arch_os target") spec.target = Arch(arch_os=platform.operating_system('default'), target=platform.target('default')) - - return True + return True # If there is a target and it is a tuple and has both filled return # False if not isinstance(spec.target, basestring): return any((self._concretize_operating_system(spec.target, platform), - self._concretize_arch_target(spec.target, plarform))) + self._concretize_arch_target(spec.target, platform))) else: spec.add_target_from_string(spec.target) @@ -316,9 +315,9 @@ class DefaultConcretizer(object): # Should think whether this can be more efficient def _proper_compiler_style(cspec, target): compilers = spack.compilers.compilers_for_spec(cspec) - if target.compiler_strategy == 'PATH': + if target.target.compiler_strategy == 'PATH': filter(lambda c: not c.modules, compilers) - if target.compiler_strategy == 'MODULES': + if target.target.compiler_strategy == 'MODULES': filter(lambda c: c.modules, compilers) return compilers -- cgit v1.2.3-60-g2f50 From 3e1be63b0f8e745a04e041288a7583333d31673c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 11 Feb 2016 11:15:50 -0800 Subject: Changed some variables to account for the fact that target is now a tuple --- lib/spack/spack/spec.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 503f80e1e6..3b2d54d047 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -566,7 +566,7 @@ class Spec(object): in the traversal. root [=True] - If false, this won't yield the root node, just its descendents. + If False, this won't yield the root node, just its descendents. direction [=children|parents] If 'children', does a traversal of this spec's children. If @@ -661,7 +661,7 @@ class Spec(object): for d in sorted(self.dependencies)) } if self.target: - d['target'] = self.target.to_dict() + d['target'] = self.target.target.to_dict() else: d['target'] = None if self.compiler: @@ -755,7 +755,6 @@ class Spec(object): if self.name in presets: changed |= self.constrain(presets[self.name]) - else: # Concretize virtual dependencies last. Because they're added # to presets below, their constraints will all be merged, but we'll @@ -830,6 +829,7 @@ class Spec(object): changed = True force = False + # Loops forever here in my implementation while changed: changes = (self.normalize(force=force), self._expand_virtual_packages(), @@ -1279,7 +1279,7 @@ class Spec(object): pkg = spack.db.get(self.name) if pkg.provides(other.name): for provided, when_spec in pkg.provided.items(): - if self.satisfies(when_spec, deps=false, strict=strict): + if self.satisfies(when_spec, deps=False, strict=strict): if provided.satisfies(other): return True return False @@ -1306,9 +1306,9 @@ class Spec(object): # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. - if not isinstance(self.target, spack.architecture.Target): + if isinstance(self.target, basestring): self.add_target_from_string(self.target) - if not isinstance(other.target, spack.architecture.Target): + if isinstance(other.target, basestring): other.add_target_from_string(other.target) if self.target and other.target: @@ -1380,7 +1380,7 @@ class Spec(object): Options: dependencies[=True] - Whether deps should be copied too. Set to false to copy a + Whether deps should be copied too. Set to False to copy a spec but not its dependencies. """ -- cgit v1.2.3-60-g2f50 From 8e8c63bd679b57a969274287d402ea9843ecd1ae Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 11 Feb 2016 11:47:39 -0800 Subject: Using pylint, fixed some of the indentation and spacing errors --- lib/spack/spack/architecture.py | 94 +++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 61 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 2914d36ba2..c4ef0805ac 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,7 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os import imp import platform as py_platform import inspect @@ -34,31 +33,34 @@ import llnl.util.tty as tty import spack from spack.util.naming import mod_to_class import spack.error as serr -from spack.version import Version -from external import yaml + class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, self).__init__( + "Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") + super(NoSysTypeError, self).__init__( + "Could not determine sys_type for this machine.") @key_ordering class Target(object): - """ Target is the processor of the host machine. The host machine may have different front-end - and back-end targets, especially if it is a Cray machine. The target will have a name and - also the module_name (e.g craype-compiler). Targets will also recognize which platform - they came from using the set_platform method. Targets will have compiler finding strategies - """ + """ Target is the processor of the host machine. + The host machine may have different front-end and back-end targets, + especially if it is a Cray machine. The target will have a name and + also the module_name (e.g craype-compiler). Targets will also + recognize which platform they came from using the set_platform method. + Targets will have compiler finding strategies + """ def __init__(self, name, compiler_strategy, module_name=None): - self.name = name # case of cray "ivybridge" but if it's x86_64 + self.name = name # case of cray "ivybridge" but if it's x86_64 self.compiler_strategy = compiler_strategy - self.module_name = module_name # craype-ivybridge + self.module_name = module_name # craype-ivybridge # Sets only the platform name to avoid recursiveness def set_platform(self, platform): @@ -85,7 +87,6 @@ class Target(object): target.platform_name = d['platform'] return target - def _cmp_key(self): return (self.name, self.compiler_strategy, self.module_name) @@ -97,6 +98,7 @@ class Target(object): return self.platform_name + '-' + self.name return self.name + @key_ordering class Platform(object): """ Abstract class that each type of Platform will subclass. @@ -104,10 +106,10 @@ class Platform(object): is returned """ - priority = None # Subclass needs to set this number. This controls order in which platform is detected. + priority = None # Subclass needs to set this number. This controls order in which platform is detected. front_end = None back_end = None - default = None # The default back end target. On cray ivybridge + default = None # The default back end target. On cray ivybridge front_os = None back_os = None @@ -119,17 +121,21 @@ class Platform(object): self.name = name def add_target(self, name, target): - """Used by the platform specific subclass to list available targets. Raises an error - if the platform specifies a name that is reserved by spack as an alias. + """Used by the platform specific subclass to list available targets. + Raises an error if the platform specifies a name + that is reserved by spack as an alias. """ if name in ['front_end', 'fe', 'back_end', 'be', 'default']: - raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) + raise ValueError( + "%s is a spack reserved alias " + "and cannot be the name of a target" % name) target.set_platform(self) self.targets[name] = target def target(self, name): - """This is a getter method for the target dictionary that handles defaulting based - on the values provided by default, front-end, and back-end. This can be overwritten + """This is a getter method for the target dictionary + that handles defaulting based on the values provided by default, + front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ if name == 'default': @@ -150,7 +156,7 @@ class Platform(object): "10.8": "mountainlion", "10.9": "mavericks", "10.10": "yosemite", - "10.11": "elcapitan"} + "10.11": "elcapitan"} mac_ver = py_platform.mac_ver()[:-2] try: os_name = mac_releases[mac_ver] @@ -199,8 +205,10 @@ class Platform(object): class OperatingSystem(object): - """ The operating system will contain a name and version. It will - also represent itself simple with it's name + """ Operating System class. It will include the name and version. + The biggest attribute to this class will be the compiler finding + strategy. At the moment the compiler finding strategy is going to + be a string. """ def __init__(self, name, version): self.name = name @@ -212,42 +220,6 @@ class OperatingSystem(object): def __repr__(self): return self.__str__() -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" - if not hasattr(spack, "sys_type"): - return None - elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() #If in __init__.py there is a sys_type() then call that - else: - return spack.sys_type # Else use the attributed which defaults to None - - -# This is livermore dependent. Hard coded for livermore -#def get_sys_type_from_environment(): -# """Return $SYS_TYPE or None if it's not defined.""" -# return os.environ.get('SYS_TYPE') - - -def get_mac_sys_type(): - """Return a Mac OS SYS_TYPE or None if this isn't a mac. - Front-end config - """ - mac_ver = py_platform.mac_ver()[0] - if not mac_ver: - return None - return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine()) - - -def get_sys_type_from_uname(): - """ Returns a sys_type from the uname argument - Front-end config - """ - try: - platform_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) - platform, _ = platform_proc.communicate() - return platform.strip() - except: - return None @memoized def all_platforms(): @@ -267,6 +239,7 @@ def all_platforms(): return modules + @memoized def sys_type(): """ Gather a list of all available subclasses of platforms. @@ -276,9 +249,8 @@ def sys_type(): """ # Try to create a Platform object using the config file FIRST platform_list = all_platforms() - platform_list.sort(key = lambda a: a.priority) + platform_list.sort(key=lambda a: a.priority) for platform in platform_list: if platform.detect(): return platform() - -- cgit v1.2.3-60-g2f50 From 7732f375d24c7089d571cbb1659cbd1d60250888 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 13 Feb 2016 14:36:01 -0800 Subject: Changed operating system getter so that linux and darwin autodetected operating systems are set as class default_os --- lib/spack/spack/architecture.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index c4ef0805ac..0e006bdf40 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -168,9 +168,11 @@ class Platform(object): def add_operating_system(self, name=None, operating_system=None): if self.name == 'linux': linux_os = self._detect_linux_os() + self.default_os = linux_os.name self.operating_sys[linux_os.name] = linux_os elif self.name == 'darwin': mac_os = self._detect_mac_os() + self.default_os = mac_os.name self.operating_sys[mac_os.name] = mac_os else: self.operating_sys[name] = operating_system -- cgit v1.2.3-60-g2f50 From c7993010116e02426c73dc32d8cfb74b8d542868 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 13 Feb 2016 14:37:07 -0800 Subject: Changed the method in which architecture is converted from string to namedtuple. Also added some TODO comments to later consider changing how architecture is handled --- lib/spack/spack/spec.py | 168 ++++++++++++++++++++++++++++++------------------ 1 file changed, 105 insertions(+), 63 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 3b2d54d047..9b5df9a825 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -63,7 +63,7 @@ line is a spec for a particular installation of the mpileaks package. if it comes immediately after the compiler name. Otherwise it will be associated with the current package spec. -6. The target to build with. This is needed on machines where +6. The architecture to build with. This is needed on machines where cross-compilation is required Here is the EBNF grammar for a spec:: @@ -72,9 +72,9 @@ Here is the EBNF grammar for a spec:: dep_list = { ^ spec } spec = id [ options ] options = { @version-list | +variant | -variant | ~variant | - %compiler | =target } + %compiler | =architecture } variant = id - target = id + architecture = id compiler = id [ version-list ] version-list = version [ { , version } ] version = id | id: | :id | id:id @@ -107,6 +107,7 @@ from llnl.util.lang import * from llnl.util.tty.color import * import spack +import spack.architecture import spack.parse import spack.error import spack.compilers as compilers @@ -123,7 +124,7 @@ identifier_re = r'\w[\w-]*' # Convenient names for color formats so that other things can use them compiler_color = '@g' version_color = '@c' -target_color = '@m' +architecture_color = '@m' enabled_variant_color = '@B' disabled_variant_color = '@r' dependency_color = '@.' @@ -134,7 +135,7 @@ hash_color = '@K' See spack.color for descriptions of the color codes. """ color_formats = {'%' : compiler_color, '@' : version_color, - '=' : target_color, + '=' : architecture_color, '+' : enabled_variant_color, '~' : disabled_variant_color, '^' : dependency_color, @@ -411,7 +412,7 @@ class Spec(object): self.name = other.name self.dependents = other.dependents self.versions = other.versions - self.target = other.target + self.architecture = other.architecture self.compiler = other.compiler self.dependencies = other.dependencies self.variants = other.variants @@ -456,11 +457,11 @@ class Spec(object): self.compiler = compiler - def _set_target(self, target): - """Called by the parser to set the target.""" - if self.target: raise DuplicateTargetError( - "Spec for '%s' cannot have two targets." % self.name) - self.target = target # a string can be set + def _set_architecture(self, architecture): + """Called by the parser to set the architecture.""" + if self.architecture: raise DuplicateArchitectureError( + "Spec for '%s' cannot have two architectures." % self.name) + self.architecture = architecture # a string can be set def _add_dependency(self, spec): @@ -516,7 +517,7 @@ class Spec(object): @property def concrete(self): """A spec is concrete if it can describe only ONE build of a package. - If any of the name, version, target, compiler, + If any of the name, version, architecture, compiler, variants, or depdenencies are ambiguous,then it is not concrete. """ if self._concrete: @@ -525,7 +526,7 @@ class Spec(object): self._concrete = bool(not self.virtual and self.versions.concrete and self.variants.concrete - and self.target + and self.architecture and self.compiler and self.compiler.concrete and self.dependencies.concrete) return self._concrete @@ -660,10 +661,12 @@ class Spec(object): 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } - if self.target: - d['target'] = self.target.target.to_dict() + if self.architecture: + # TODO: Fix the target.to_dict to account for the tuple + # Want it to be a dict of dicts + d['architecture'] = self.architecture.target.to_dict() else: - d['target'] = None + d['architecture'] = None if self.compiler: d.update(self.compiler.to_dict()) else: @@ -689,7 +692,9 @@ class Spec(object): spec = Spec(name) spec.versions = VersionList.from_dict(node) - spec.target = spack.architecture.Target.from_dict(node['target']) + # TODO: Need to fix the architecture.Target.from_dict + spec.architecture = spack.architecture.Target.from_dict( + node['architecture']) if node['compiler'] is None: spec.compiler = None @@ -760,7 +765,7 @@ class Spec(object): # to presets below, their constraints will all be merged, but we'll # still need to select a concrete package later. changed |= any( - (spack.concretizer.concretize_target(self), + (spack.concretizer.concretize_architecture(self), spack.concretizer.concretize_compiler(self), spack.concretizer.concretize_version(self), spack.concretizer.concretize_variants(self))) @@ -1149,10 +1154,11 @@ class Spec(object): raise UnsatisfiableVariantSpecError(self.variants[v], other.variants[v]) - if self.target is not None and other.target is not None: - if self.target != other.target: - raise UnsatisfiableTargetSpecError(self.target, - other.target) + # TODO: Check out the logic here + if self.architecture is not None and other.architecture is not None: + if self.architecture != other.architecture: + raise UnsatisfiableTargetSpecError(self.architecture, + other.architecture) changed = False if self.compiler is not None and other.compiler is not None: @@ -1164,9 +1170,9 @@ class Spec(object): changed |= self.versions.intersect(other.versions) changed |= self.variants.constrain(other.variants) - old = self.target - self.target = self.target or other.target - changed |= (self.target != old) + old = self.architecture + self.architecture = self.architecture or other.architecture + changed |= (self.architecture != old) if deps: changed |= self._constrain_dependencies(other) @@ -1231,34 +1237,67 @@ class Spec(object): except SpecError: return parse_anonymous_spec(spec_like, self.name) + def _is_valid_platform(self, platform, platform_list): + if platform in platform_names: + return True + return False + + def _is_valid_target(self, target, platform): + if target in platform.targets: + return True + return False + + def add_architecture_from_string(self, arch): + """ The user is able to provide a architecture string of the form + platform-os-target. This string will be parsed by this function + and turn the architecture string into an architecture tuple of + platform, operating system and target processor classes. + The platform-os-target triplet can be delimited by a '-'. If any + portion of the architecture triplet is empty, spack will supply + the default. If the architecture is blank then defaults will + be provided based off of the platform + + e.g + =linux-ubuntu10-x84_64 -> (linux, ubuntu10, x86_64) + + =cray_xc-SuSE11-haswell -> (cray_xc, SuSE11, haswell) + + =bgq -> (bgq, + default_os, + default_target) + + =elcapitan -> (darwin, elcapitan, x86_64) - def add_target_from_string(self, arch): - """If only a target is provided, spack will assume the default architecture. - A platform-target pair can be input delimited by a '-'. If either portion of - a platform-target pair is empty, spack will supply a default, in the case of - a blank target the default will be dependent on the platform. - E.g. x86_64 -> 64 bit x86 - bgq- -> default bgq target (back end/powerpc) - cray-hawswell -> haswell target on cray platform + =x86_64 -> (autodetected platform, + default_os, + x86_64) """ - Arch = namedtuple("Arch", "arch_os target") - platform = spack.architecture.sys_type() - + + platform_list = spack.architecture.all_platforms() + platform_names = [plat.__name__.lower() for plat in platform_list] if arch is None: return + # Get all the platforms to check whether string is valid + Arch = namedtuple("Arch", "platform platform_os target") + arch_list = arch.split("-") + platform = spack.architecture.sys_type() + for entry in arch_list: + if _is_valid_platform(entry, platform_names): + platform = entry() + elif _is_valid_target(entry, platform): + target = entry + else: + platform_os = entry - if '-' in arch: - os_name, target = arch.split('-') - self.target = Arch(arch_os=platform.operating_system(os_name), - target = platform.target(target)) - elif arch in platform.targets: - self.target = Arch(arch_os=platform.operating_system('default'), - target=platform.target(target)) - else: - os_name = arch # Odd naming here, definitely need to change - self.target = Arch(arch_os=platform.operating_system(os_name), - target=platform.target('default')) + if target is None: + target = platform.target('default') + if platform_os is None: + platform_os = platform.operating_system('default') + self.architecture = Arch(platform=platform, + platform_os=platform_os, + target=target) + def satisfies(self, other, deps=True, strict=False): """determine if this spec satisfies all constraints of another. @@ -1306,15 +1345,16 @@ class Spec(object): # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. - if isinstance(self.target, basestring): - self.add_target_from_string(self.target) - if isinstance(other.target, basestring): - other.add_target_from_string(other.target) - - if self.target and other.target: - if self.target != other.target: + if isinstance(self.architecture, basestring): + self.add_architecture_from_string(self.architecture) + if isinstance(other.architecture, basestring): + other.add_architecture_from_string(other.architecture) + + # TODO: Need to make sure that comparisons can be made via classes + if self.architecture and other.architecture: + if self.architecture != other.architecture: return False - elif strict and (other.target and not self.target): + elif strict and (other.architecture and not self.architecture): return False # If we need to descend into dependencies, do it, otherwise we're done. @@ -1384,11 +1424,12 @@ class Spec(object): spec but not its dependencies. """ + # TODO: Check if comparisons for tuple are valid # We don't count dependencies as changes here changed = True if hasattr(self, 'name'): changed = (self.name != other.name and self.versions != other.versions and \ - self.target != other.target and self.compiler != other.compiler and \ + self.architecture != other.architecture and self.compiler != other.compiler and \ self.variants != other.variants and self._normal != other._normal and \ self.concrete != other.concrete and self.external != other.external and \ self.external_module != other.external_module) @@ -1396,7 +1437,7 @@ class Spec(object): # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() - self.target = other.target + self.architecture = other.architecture self.compiler = other.compiler.copy() if other.compiler else None if kwargs.get('cleardeps', True): self.dependents = DependencyMap() @@ -1526,7 +1567,7 @@ class Spec(object): def _cmp_node(self): """Comparison key for just *this node* and not its deps.""" return (self.name, self.versions, self.variants, - self.target, self.compiler) + self.architecture, self.compiler) def eq_node(self, other): @@ -1585,7 +1626,7 @@ class Spec(object): Anything else is copied verbatim into the output stream. *Example:* ``$_$@$+`` translates to the name, version, and options - of the package, but no dependencies, target, or compiler. + of the package, but no dependencies, architecture, or compiler. TODO: allow, e.g., $6# to customize short hash length TODO: allow, e.g., $## for full hash. @@ -1628,9 +1669,10 @@ class Spec(object): elif c == '+': if self.variants: write(fmt % str(self.variants), c) + # TODO: Check string methods here elif c == '=': - if self.target: - write(fmt % (c + str(self.target)), c) + if self.architecture: + write(fmt % (c + str(self.architecture)), c) elif c == '#': out.write('-' + fmt % (self.dag_hash(7))) elif c == '$': @@ -2036,10 +2078,10 @@ class UnknownVariantError(SpecError): "Package %s has no variant %s!" % (pkg, variant)) -class DuplicateTargetError(SpecError): +class DuplicateArchitectureError(SpecError): """Raised when the same target occurs in a spec twice.""" def __init__(self, message): - super(DuplicateTargetError, self).__init__(message) + super(DuplicateArchitectureError, self).__init__(message) class InconsistentSpecError(SpecError): -- cgit v1.2.3-60-g2f50 From 35a602baaf39fb5056f3dbb7eccbd638e126159f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 16 Feb 2016 15:17:57 -0800 Subject: Changed target checking to architecture checking for abi compatible --- lib/spack/spack/abi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index 1dd49d6b2c..0f07feda5f 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -34,9 +34,10 @@ class ABI(object): """This class provides methods to test ABI compatibility between specs. The current implementation is rather rough and could be improved.""" - def target_compatible(self, parent, child): + def architecture_compatible(self, parent, child): """Returns true iff the parent and child specs have ABI compatible targets.""" - return not parent.target or not child.target or parent.target == child.target + return not parent.architecture or not \ + child.achitecture or parent.architecture == child.architecture @memoized @@ -123,6 +124,6 @@ class ABI(object): def compatible(self, parent, child, **kwargs): """Returns true iff a parent and child spec are ABI compatible""" loosematch = kwargs.get('loose', False) - return self.target_compatible(parent, child) and \ + return self.architecture_compatible(parent, child) and \ self.compiler_compatible(parent, child, loose=loosematch) -- cgit v1.2.3-60-g2f50 From 19dca4bcc9dcdd902fb5490b4ee49b9dc92aa5ba Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 16 Feb 2016 15:21:36 -0800 Subject: Added some _cmp_key methods to OperatingSystem class, and also changed the way Platforms are compared. Created a mini inherited class named Arch that inherits from namedtuple. The reason for this is to override the namedtuple __str__ method to make arch tuples look prettier when printed out. Rather than Arch(platform= ... ) it prints to platform-os-target which makes directory paths to packages look cleaner. --- lib/spack/spack/architecture.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 0e006bdf40..3cbce5dc52 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,6 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +from collections import namedtuple import imp import platform as py_platform import inspect @@ -178,7 +179,7 @@ class Platform(object): self.operating_sys[name] = operating_system def operating_system(self, name): - if name == 'default': + if name == 'default_os': name = self.default_os if name == 'front_os': name = self.front_os @@ -203,9 +204,10 @@ class Platform(object): return self.name def _cmp_key(self): - return (self.name, (_cmp_key(t) for t in self.targets.values())) - + return (self.name, (_cmp_key(t) for t in self.targets.values()), + (_cmp_key(o) for o in self.operating_sys.values())) +@key_ordering class OperatingSystem(object): """ Operating System class. It will include the name and version. The biggest attribute to this class will be the compiler finding @@ -222,6 +224,19 @@ class OperatingSystem(object): def __repr__(self): return self.__str__() + def _cmp_key(self): + return (self.name, self.version) + + +class Arch(namedtuple("Arch", "platform platform_os target")): + """ namedtuple for Architecture. Will have it's own __str__ method + to make printing out the tuple easier and also won't make directory + paths look odd """ + __slots__ = () + + def __str__(self): + return (self.platform.name +"-"+ + self.platform_os.name + "-" + self.target.name) @memoized def all_platforms(): -- cgit v1.2.3-60-g2f50 From 31ab2383067830e47d54788a2cac01f949e77798 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 16 Feb 2016 15:22:23 -0800 Subject: Beginning attemps to fix concretization method to account for the new tuple architecture --- lib/spack/spack/concretize.py | 73 +++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 27 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 8886b3cd93..1c373679a2 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -210,55 +210,74 @@ class DefaultConcretizer(object): return True # Things changed + def _concretize_platform(self, arch, platform): + if issubclass(arch.platform.__class__, spack.architecture.Platform): + return True + else: + arch.platform = platform + def _concretize_operating_system(self, arch, platform): """ Future method for concretizing operating system """ - if isinstance(arch.arch_os, spack.architecture.OperatingSystem): + if isinstance(arch.platform_os, spack.architecture.OperatingSystem): return False else: - arch.arch_os = platform.operating_system('default') + arch.arch_os = platform.operating_system('default_os') return True - def _concretize_arch_target(self, arch, platform): + def _concretize_target(self, arch, platform): if isinstance(arch.target, spack.architecture.Target): return False else: arch.target = platform.target('default') return True - def concretize_target(self, spec): - """If the spec already has an target and it is a an target type, - return. Otherwise, if it has a target that is a string type, generate a - target based on that type. If it has no target and the root of the - DAG has an target, then use that. Otherwise, take the system's default - target. + def concretize_architecture(self, spec): + """If the spec is empty provide the defaults of the platform. If the + architecture is not a basestring, then check if either the platform, + target or operating system are concretized. If any of the fields are + changed then return True. If everything is concretized (i.e the + architecture attribute is a namedtuple of classes) then return False. + If the target is a string type, then convert the string into a + concretized architecture. If it has no architecture and the root of the + DAG has an architecture, then use the root otherwise use the defaults + on the platform. """ - platform = spack.architecture.sys_type() - if spec.target is None: + platform = spack.architecture.sys_type() + import ipdb;ipdb.set_trace() + if spec.architecture is None: # Create an empty tuple - Arch = collections.namedtuple("Arch", "arch_os target") - spec.target = Arch(arch_os=platform.operating_system('default'), - target=platform.target('default')) - return True - # If there is a target and it is a tuple and has both filled return - # False - if not isinstance(spec.target, basestring): - return any((self._concretize_operating_system(spec.target, platform), - self._concretize_arch_target(spec.target, platform))) + Arch = spack.architecture.Arch + # Set the architecture to all defaults + spec.architecture = Arch(platform=platform, + platform_os=platform.operating_system('default_os'), + target=platform.target('default')) + return True + #If there is a target and it is a tuple and has both filled return + #False + if not isinstance(spec.architecture, basestring): + return any(( + self._concretize_platform(spec.architecture, platform), + self._concretize_operating_system(spec.architecture, platform), + self._concretize_arch_target(spec.architecture, platform))) else: spec.add_target_from_string(spec.target) - if spec.root.target and \ - not isinstance(spec.root.target, basestring): - bool_flag = any( - (self._concretize_operating_system(spec.root.target, platform), - self._concretize_arch_target(spec.root.target, platform))) - spec.target =spec.root.target + # Does not look pretty at all!!! + if spec.root.architecture and \ + not isinstance(spec.root.architecture, basestring): + bool_flag = any(( + self._concretize_platform(spec.root.architecture, platform), + self._concretize_operating_system(spec.root.architecture, + platform), + self._concretize_target(spec.root.target, platform))) + spec.architecture =spec.root.architecture return bool_flag else: - spec.add_target_from_string(spec.root.target) + spec.add_architecture_from_string(spec.root.architecture) + return True # if there is no target specified used the defaults -- cgit v1.2.3-60-g2f50 From 62b0293963bd21ff734993e5ac577650af0faa3f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 16 Feb 2016 15:23:38 -0800 Subject: Cleaned up the file. Got rid of commented out sections of code since they weren't going to be used anyway --- lib/spack/spack/platforms/cray_xc.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index b37d36a62c..d48b6ffcfa 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -17,7 +17,7 @@ class CrayXc(Platform): if we use CRAY_CPU_TARGET as the default. This will ensure that if we're on a XC-40 or XC-30 then we can detect the target ''' - super(CrayXc, self).__init__('cray_xc') + super(CrayXc, self).__init__('crayxc') # Handle the default here so we can check for a key error if 'CRAY_CPU_TARGET' in os.environ: @@ -28,23 +28,21 @@ class CrayXc(Platform): self.front_end = self.default self.back_end = self.default - # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. self.add_target('sandybridge', Target('sandybridge', 'PATH')) - self.add_target('ivybridge', Target('ivybridge', 'MODULES', 'craype-ivybridge')) - self.add_target('haswell', Target('haswell', 'MODULES', 'craype-haswell')) - - self.add_operating_system('SuSE11', OperatingSystem('SuSE11', '11')) - self.add_operating_system('GNU/Linux', OperatingSystem('GNU/Linux', '10')) - -# self.add_target(self.front_end, Target(self.front_end, 'PATH')) - # Back End compiler needs the proper target module loaded. -# self.add_target(self.back_end, Target(self.front_end, 'MODULES', 'craype-'+ self.back_end)) -# self.add_target(self.default, Target(self.default, 'MODULES', 'craype-' + self.default)) - # This is kludgy and the order matters when the targets are all haswell - # This is because the last one overwrites the others when they have the - # same name. + + self.add_target('ivybridge', + Target('ivybridge', 'MODULES', 'craype-ivybridge')) + + self.add_target('haswell', + Target('haswell', 'MODULES', 'craype-haswell')) + + self.add_operating_system('SuSE11', + OperatingSystem('SuSE11', '11')) + + self.add_operating_system('GNU/Linux', + OperatingSystem('GNU/Linux', '10')) @classmethod def detect(self): -- cgit v1.2.3-60-g2f50 From a3039c4c67b376d2e8577c2227cffac8873bdc74 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 16 Feb 2016 15:26:07 -0800 Subject: Changed add_architecture_from_string, it now loops through the string and checks whether each piece of string is a valid platform, operating system and target. If the operating system or target are none it will use the defaults. Updated the documentation for that method. One thing that bothers me is how to account for the multitude of operating systems when cross compiling. If someone wants to compile with another operating system not found on current platform. How can spack check to see if it is valid? --- lib/spack/spack/spec.py | 64 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 9b5df9a825..91e9e432eb 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1238,14 +1238,18 @@ class Spec(object): return parse_anonymous_spec(spec_like, self.name) def _is_valid_platform(self, platform, platform_list): - if platform in platform_names: + if platform in platform_list: return True return False def _is_valid_target(self, target, platform): if target in platform.targets: return True - return False + return False + def _is_valid_os(self, os_string, platform): + if os_string in platform.operating_sys: + return True + return False def add_architecture_from_string(self, arch): """ The user is able to provide a architecture string of the form @@ -1254,8 +1258,9 @@ class Spec(object): platform, operating system and target processor classes. The platform-os-target triplet can be delimited by a '-'. If any portion of the architecture triplet is empty, spack will supply - the default. If the architecture is blank then defaults will - be provided based off of the platform + the default. If the entire architecture field is blank then + defaults will be provided based off of the platform. + This happens in the concretize_architecture method in concretize.py e.g =linux-ubuntu10-x84_64 -> (linux, ubuntu10, x86_64) @@ -1272,27 +1277,38 @@ class Spec(object): default_os, x86_64) """ - + if arch is None: return + platform_list = spack.architecture.all_platforms() - platform_names = [plat.__name__.lower() for plat in platform_list] - if arch is None: - return - # Get all the platforms to check whether string is valid - Arch = namedtuple("Arch", "platform platform_os target") + platform_names = {plat.__name__.lower():plat for plat in platform_list} + Arch = spack.architecture.Arch arch_list = arch.split("-") - platform = spack.architecture.sys_type() + + # Create instance of current platform, gets overwritten if user + # provided a platform spec. + platform = spack.architecture.sys_type() + target = None + platform_os = None + for entry in arch_list: - if _is_valid_platform(entry, platform_names): - platform = entry() - elif _is_valid_target(entry, platform): - target = entry + if self._is_valid_platform(entry, platform_names): + if entry != platform.name: + platform = platform_dict[entry]() # Create instance of platform + elif self._is_valid_target(entry, platform): + target = platform.target(entry) + # Need to figure out if we're supporting arbitrary os's and how + # to account for them + # Not really a good implementation since the user can add + # gibberish and spack will see it as an os + elif self._is_valid_os(entry, platform): + platform_os = platform.operating_system(entry) else: - platform_os = entry + raise UnknownArchitectureSpecError(entry) if target is None: target = platform.target('default') if platform_os is None: - platform_os = platform.operating_system('default') + platform_os = platform.operating_system('default_os') self.architecture = Arch(platform=platform, platform_os=platform_os, @@ -1854,7 +1870,6 @@ class SpecParser(spack.parse.Parser): def do_parse(self): specs = [] - try: while self.next: if self.accept(ID): @@ -1889,7 +1904,7 @@ class SpecParser(spack.parse.Parser): spec.name = self.token.value spec.versions = VersionList() spec.variants = VariantMap(spec) - spec.target = None + spec.architecture = None spec.compiler = None spec.external = None spec.external_module = None @@ -1920,7 +1935,7 @@ class SpecParser(spack.parse.Parser): spec._set_compiler(self.compiler()) elif self.accept(EQ): - spec._set_target(self.target()) + spec._set_architecture(self.architecture()) else: break @@ -1938,7 +1953,7 @@ class SpecParser(spack.parse.Parser): return self.token.value - def target(self): + def architecture(self): self.expect(ID) return self.token.value @@ -2077,6 +2092,13 @@ class UnknownVariantError(SpecError): super(UnknownVariantError, self).__init__( "Package %s has no variant %s!" % (pkg, variant)) +class UnknownArchitectureSpecError(SpecError): + """ Raised when an entry in a string field is neither a platform, + operating system or a target. """ + def __init__(self, architecture_spec_entry): + super(UnknownArchitectureSpecError, self).__init__( + "Architecture spec %s is not a valid spec entry" % ( + architecture_spec_entry)) class DuplicateArchitectureError(SpecError): """Raised when the same target occurs in a spec twice.""" -- cgit v1.2.3-60-g2f50 From 2a79537ba1120620f79d39bd4081a6a11b7862c4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:44:38 -0800 Subject: Fixed silly typo that gave errors in tests --- lib/spack/spack/abi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index 0f07feda5f..bd9bc30fd1 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -36,8 +36,8 @@ class ABI(object): def architecture_compatible(self, parent, child): """Returns true iff the parent and child specs have ABI compatible targets.""" - return not parent.architecture or not \ - child.achitecture or parent.architecture == child.architecture + return not parent.architecture or not child.architecture \ + or parent.architecture == child.architecture @memoized -- cgit v1.2.3-60-g2f50 From 2c20fc2ebf8e9b360f3935a1867d9b95c69513ac Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:45:53 -0800 Subject: Changed module loading to use arch tuple instead of regular target.module --- lib/spack/spack/build_environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index d849b4b56c..6e03394777 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -226,8 +226,8 @@ def set_build_environment_variables(pkg): pkg_config_dirs.append(pcdir) path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) - if pkg.spec.target.module_name: - load_module(pkg.spec.target.module_name) + if pkg.spec.architecture.target.module_name: + load_module(pkg.spec.architecture.target.module_name) def set_module_variables_for_package(pkg): """Populate the module scope of install() with some useful functions. -- cgit v1.2.3-60-g2f50 From 524e9b372ee85508abad1163368c7a9fdc09fcc3 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:47:29 -0800 Subject: From now on all targets are now architecture. Architecture is a named tuple so any time we need access to target we do architecture.target. --- lib/spack/spack/concretize.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 1c373679a2..6c2aaeda01 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -180,7 +180,7 @@ class DefaultConcretizer(object): # If there are known available versions, return the most recent # version that satisfies the spec - pkg = spec.package + pkg = spec.package # Gives error here with dynist cmp_versions = partial(spack.pkgsort.version_compare, spec.name) valid_versions = sorted( [v for v in pkg.versions @@ -212,9 +212,10 @@ class DefaultConcretizer(object): def _concretize_platform(self, arch, platform): if issubclass(arch.platform.__class__, spack.architecture.Platform): - return True + return False else: arch.platform = platform + return True def _concretize_operating_system(self, arch, platform): """ Future method for concretizing operating system """ @@ -245,7 +246,6 @@ class DefaultConcretizer(object): """ platform = spack.architecture.sys_type() - import ipdb;ipdb.set_trace() if spec.architecture is None: # Create an empty tuple Arch = spack.architecture.Arch @@ -260,9 +260,9 @@ class DefaultConcretizer(object): return any(( self._concretize_platform(spec.architecture, platform), self._concretize_operating_system(spec.architecture, platform), - self._concretize_arch_target(spec.architecture, platform))) + self._concretize_target(spec.architecture, platform))) else: - spec.add_target_from_string(spec.target) + spec.add_architecture_from_string(spec.target) # Does not look pretty at all!!! if spec.root.architecture and \ @@ -325,18 +325,18 @@ class DefaultConcretizer(object): link to this one, to maximize compatibility. """ # Pass on concretizing the compiler if the target is not yet determined - if not spec.target: + if not spec.architecture.target: #Although this usually means changed, this means awaiting other changes return True # Only use a matching compiler if it is of the proper style # Takes advantage of the proper logic already existing in compiler_for_spec # Should think whether this can be more efficient - def _proper_compiler_style(cspec, target): + def _proper_compiler_style(cspec, architecture): compilers = spack.compilers.compilers_for_spec(cspec) - if target.target.compiler_strategy == 'PATH': + if architecture.target.compiler_strategy == 'PATH': filter(lambda c: not c.modules, compilers) - if target.target.compiler_strategy == 'MODULES': + if architecture.target.compiler_strategy == 'MODULES': filter(lambda c: c.modules, compilers) return compilers @@ -369,7 +369,7 @@ class DefaultConcretizer(object): # copy concrete version into other_compiler index = len(matches)-1 - while not _proper_compiler_style(matches[index], spec.target): + while not _proper_compiler_style(matches[index], spec.architecture): index -= 1 if index == 0: raise NoValidVersionError(spec) -- cgit v1.2.3-60-g2f50 From ab4006ee2c4e7ecbf256a9de56e54e8573b37e01 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:48:53 -0800 Subject: Changed so that directory layout uses the platform-os-target string version of the arch tuple --- lib/spack/spack/directory_layout.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index da3441fa13..435ecfdf4d 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -200,8 +200,7 @@ class YamlDirectoryLayout(DirectoryLayout): spec.version, spec.dag_hash(self.hash_len)) - path = join_path( - spec.target, + path = join_path(spec.architecture, "%s-%s" % (spec.compiler.name, spec.compiler.version), dir_name) -- cgit v1.2.3-60-g2f50 From 1d484dbe5fb70885875dcbb2e795c4d517646e77 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:49:36 -0800 Subject: Changed so that modules use correct file path with spec.architecture instead of spec.target --- lib/spack/spack/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index f84ac75c77..56a61adefb 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -193,7 +193,7 @@ class Dotkit(EnvModule): @property def file_name(self): - return join_path(Dotkit.path, self.spec.target, + return join_path(Dotkit.path, self.spec.architecture, self.spec.format('$_$@$%@$+$#.dk')) @property @@ -230,7 +230,7 @@ class TclModule(EnvModule): @property def file_name(self): - return join_path(TclModule.path, self.spec.target, self.use_name) + return join_path(TclModule.path, self.spec.architecture, self.use_name) @property -- cgit v1.2.3-60-g2f50 From 54042e399b5cfb4344643d8447c4274d34842643 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:50:10 -0800 Subject: Changed target to architecture.target --- lib/spack/spack/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 1148e4ac3d..b0d84863de 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -588,11 +588,13 @@ class Package(object): @property + #TODO: Change this to architecture def compiler(self): """Get the spack.compiler.Compiler object used to build this package.""" if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") - return spack.compilers.compiler_for_spec(self.spec.compiler, self.spec.target) + return spack.compilers.compiler_for_spec(self.spec.compiler, + self.spec.architecture.target) def url_version(self, version): -- cgit v1.2.3-60-g2f50 From d9e8bf18078bc4df381b05820b14365f6e0ae01f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:51:14 -0800 Subject: Added ARCHITECTURE field when showing specs --- lib/spack/spack/spec.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 91e9e432eb..0b42498910 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1735,9 +1735,9 @@ class Spec(object): elif named_str == 'OPTIONS': if self.variants: write(fmt % str(self.variants), '+') - elif named_str == 'TARGET': - if self.target: - write(fmt % str(self.target), '=') + elif named_str == 'ARCHITECTURE': + if self.architecture: + write(fmt % self.architecture, '=') elif named_str == 'SHA1': if self.dependencies: out.write(fmt % str(self.dep_hash(8))) -- cgit v1.2.3-60-g2f50 From 93a92a00eeaf6e3e721b2150fbc44bd1b388ee66 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:53:09 -0800 Subject: Changed abstract.target to abstract.architecture.target and abstract.architecture.target.concrete --- lib/spack/spack/test/concretize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 2403719134..c12aada076 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -46,7 +46,7 @@ class ConcretizeTest(MockPackagesTest): if abstract.compiler and abstract.compiler.concrete: self.assertEqual(abstract.compiler, concrete.compiler) - if abstract.target and abstract.target.concrete: + if abstract.architecture and abstract.architecture.target.concrete: self.assertEqual(abstract.target, concrete.target) -- cgit v1.2.3-60-g2f50 From 28218755a5b11dd02c3c7e4c4a503a110ade94e2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 18 Feb 2016 13:06:55 -0800 Subject: Changed spec.target to spec.architecture --- lib/spack/spack/concretize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 6c2aaeda01..0e7bf53b44 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -262,7 +262,7 @@ class DefaultConcretizer(object): self._concretize_operating_system(spec.architecture, platform), self._concretize_target(spec.architecture, platform))) else: - spec.add_architecture_from_string(spec.target) + spec.add_architecture_from_string(spec.architecture) # Does not look pretty at all!!! if spec.root.architecture and \ -- cgit v1.2.3-60-g2f50 From 18ddbae60e0fa91cbbc73a1b501c489c3f7fc8d2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:49:53 -0800 Subject: Added new module path to operating system file --- lib/spack/spack/__init__.py | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index e1ef094f17..6ad9b87ca2 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -39,6 +39,7 @@ build_env_path = join_path(lib_path, "env") module_path = join_path(lib_path, "spack") platform_path = join_path(module_path, 'platforms') compilers_path = join_path(module_path, "compilers") +operating_system_path = join_path(module_path, 'operating_system') test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") var_path = join_path(prefix, "var", "spack") -- cgit v1.2.3-60-g2f50 From a8de45ce28b8277c0ca1740aa53e1e4e334596d1 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:50:51 -0800 Subject: Got rid of methods used to detect mac osx and linux osx. Now those methods are used my operating system subclasses --- lib/spack/spack/architecture.py | 90 +++++++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 35 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3cbce5dc52..3ba45f3d82 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -148,35 +148,29 @@ class Platform(object): return self.targets[name] - def _detect_linux_os(self): - return OperatingSystem(py_platform.dist()[0], py_platform.dist()[1]) - - def _detect_mac_os(self): - mac_releases = {'10.6': "snowleopard", - "10.7": "lion", - "10.8": "mountainlion", - "10.9": "mavericks", - "10.10": "yosemite", - "10.11": "elcapitan"} - mac_ver = py_platform.mac_ver()[:-2] - try: - os_name = mac_releases[mac_ver] - return OperatingSystem(os_name, mac_ver) - except KeyError: - os_name = "mac_os" - return OperatingSystem(os_name, mac_ver) - - def add_operating_system(self, name=None, operating_system=None): - if self.name == 'linux': - linux_os = self._detect_linux_os() - self.default_os = linux_os.name - self.operating_sys[linux_os.name] = linux_os - elif self.name == 'darwin': - mac_os = self._detect_mac_os() - self.default_os = mac_os.name - self.operating_sys[mac_os.name] = mac_os - else: - self.operating_sys[name] = operating_system + #def _detect_linux_os(self): + # return OperatingSystem(py_platform.dist()[0], py_platform.dist()[1]) + + #def _detect_mac_os(self): + # mac_releases = {'10.6': "snowleopard", + # "10.7": "lion", + # "10.8": "mountainlion", + # "10.9": "mavericks", + # "10.10": "yosemite", + # "10.11": "elcapitan"} + # mac_ver = py_platform.mac_ver()[:-2] + # try: + # os_name = mac_releases[mac_ver] + # return OperatingSystem(os_name, mac_ver) + # except KeyError: + # os_name = "mac_os" + # return OperatingSystem(os_name, mac_ver) + + def add_operating_system(self, name, os_class): + """ Add the operating_system class object into the + platform.operating_sys dictionary + """ + self.operating_sys[name] = os_class def operating_system(self, name): if name == 'default_os': @@ -207,27 +201,37 @@ class Platform(object): return (self.name, (_cmp_key(t) for t in self.targets.values()), (_cmp_key(o) for o in self.operating_sys.values())) + @key_ordering class OperatingSystem(object): - """ Operating System class. It will include the name and version. - The biggest attribute to this class will be the compiler finding - strategy. At the moment the compiler finding strategy is going to - be a string. + """ Operating System will be like a class similar to platform extended + by subclasses for the specifics. Operating System will contain the + compiler finding logic. Instead of calling two separate methods to + find compilers we call find_compilers method for each operating system """ def __init__(self, name, version): self.name = name self.version = version def __str__(self): - return self.name + return self.name + self.version def __repr__(self): return self.__str__() + + def compiler_strategy(self): + """ The compiler strategy will be overwritten by the subclass. + Depending on where it comes from it will either use compilers + based off of MODULES search method or the PATH search method + """ + raise NotImplementedError() def _cmp_key(self): return (self.name, self.version) + - +#NOTE: Key error caused because Architecture has no comparison method +@key_ordering class Arch(namedtuple("Arch", "platform platform_os target")): """ namedtuple for Architecture. Will have it's own __str__ method to make printing out the tuple easier and also won't make directory @@ -238,6 +242,22 @@ class Arch(namedtuple("Arch", "platform platform_os target")): return (self.platform.name +"-"+ self.platform_os.name + "-" + self.target.name) + def _cmp_key(self): + return (self.platform.name, self.platform_os.name, self.target.name) + + @staticmethod + def to_dict(platform, os_name, target): + """ This function will handle all the converting of dictionaries + for each of the architecture classes. This is better than having + each class have a to_dict method when creating the dict of dicts + arch class + """ + d = {} + d['platform'] = platform + d['operating_system'] = os_name + d['target'] = target.to_dict() + return d + @memoized def all_platforms(): modules = [] -- cgit v1.2.3-60-g2f50 From 725d6d5fce56aa7d0ae401ec71d1565835248250 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:52:09 -0800 Subject: import linux distro operating system subclass, set front-end os to LinuxDistro() --- lib/spack/spack/platforms/cray_xc.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index d48b6ffcfa..b58c84d3eb 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -1,6 +1,7 @@ import os -from spack.architecture import Platform, Target, OperatingSystem +from spack.architecture import Platform, Target +from spack.operating_system.linux_distro import LinuxDistro class CrayXc(Platform): priority = 20 @@ -9,8 +10,8 @@ class CrayXc(Platform): default = 'ivybridge' front_os = "SuSE11" - back_os = "GNU/Linux" - default_os = "GNU/Linux" + back_os = "CNL" + default_os = "CNL" def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better @@ -31,18 +32,12 @@ class CrayXc(Platform): # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. self.add_target('sandybridge', Target('sandybridge', 'PATH')) - self.add_target('ivybridge', Target('ivybridge', 'MODULES', 'craype-ivybridge')) - self.add_target('haswell', - Target('haswell', 'MODULES', 'craype-haswell')) - - self.add_operating_system('SuSE11', - OperatingSystem('SuSE11', '11')) + Target('haswell', 'MODULES', 'craype-haswell')) - self.add_operating_system('GNU/Linux', - OperatingSystem('GNU/Linux', '10')) + self.add_operating_system('SuSE11', LinuxDistro()) @classmethod def detect(self): -- cgit v1.2.3-60-g2f50 From 1a7d6ed49a8a3fd18fd3deb8ddc035ce0063766d Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:52:52 -0800 Subject: import MacOSX subclass and added add_operating_system method to use subclass --- lib/spack/spack/platforms/darwin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index cfa1d957a1..5b797a78db 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,5 +1,6 @@ import subprocess from spack.architecture import Platform, Target +from spack.operating_system.mac_osx import MacOSX class Darwin(Platform): priority = 89 @@ -10,7 +11,9 @@ class Darwin(Platform): def __init__(self): super(Darwin, self).__init__('darwin') self.add_target(self.default, Target(self.default, 'PATH')) - self.add_operating_system() + mac_os = MacOSX() + self.default_os = mac_os.name + self.add_operating_system(mac_os.name, mac_os) @classmethod def detect(self): -- cgit v1.2.3-60-g2f50 From 22bf4bc08020898106ef2c8043f102ceeb83ec5c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:53:16 -0800 Subject: import linux distro subclass. Added method to add operating system to platform dictionary --- lib/spack/spack/platforms/linux.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index dcb08569a8..220f1bbaf4 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -1,5 +1,6 @@ import subprocess from spack.architecture import Platform, Target +from spack.operating_system.linux_distro import LinuxDistro class Linux(Platform): priority = 90 @@ -10,7 +11,9 @@ class Linux(Platform): def __init__(self): super(Linux, self).__init__('linux') self.add_target(self.default, Target(self.default, 'PATH')) - self.add_operating_system() + linux_dist = LinuxDistro() + self.default_os = str(linux_dist) + self.add_operating_system(str(linux_dist), linux_dist) @classmethod def detect(self): -- cgit v1.2.3-60-g2f50 From 1367ccab93e6263e11477abe7ea05f1ad74d88a2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:53:55 -0800 Subject: New folder that will hold operating system subclasses --- lib/spack/spack/operating_system/__init__.py | 0 lib/spack/spack/operating_system/linux_distro.py | 25 +++++++++++++ lib/spack/spack/operating_system/mac_osx.py | 45 ++++++++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 lib/spack/spack/operating_system/__init__.py create mode 100644 lib/spack/spack/operating_system/linux_distro.py create mode 100644 lib/spack/spack/operating_system/mac_osx.py (limited to 'lib') diff --git a/lib/spack/spack/operating_system/__init__.py b/lib/spack/spack/operating_system/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/spack/spack/operating_system/linux_distro.py b/lib/spack/spack/operating_system/linux_distro.py new file mode 100644 index 0000000000..30a85fe61c --- /dev/null +++ b/lib/spack/spack/operating_system/linux_distro.py @@ -0,0 +1,25 @@ +import platform as py_platform +import spack +from spack.architecture import Platform, OperatingSystem + +class LinuxDistro(OperatingSystem): + """ This class will represent the autodetected operating system + for a Linux System. Since there are many different flavors of + Linux, this class will attempt to encompass them all through + autodetection using the python module platform and the method + platform.dist() + """ + def __init__(self): + def detect_operating_system(): + name = py_platform.dist()[0] + version = py_platform.dist()[1] + return name, version + + name, version = detect_operating_system() + + super(LinuxDistro, self).__init__(name, version) + + @property + def compiler_strategy(self): + return "PATH" + diff --git a/lib/spack/spack/operating_system/mac_osx.py b/lib/spack/spack/operating_system/mac_osx.py new file mode 100644 index 0000000000..0b939a5546 --- /dev/null +++ b/lib/spack/spack/operating_system/mac_osx.py @@ -0,0 +1,45 @@ +""" This class represents the MAC_OSX operating system. This will be auto + detected using the python platform.mac_ver. The MAC_OSX platform + will be represented using the major version operating system name, i.e + el capitan, yosemite...etc. +""" + +import spack +import os +import platform as py_platform +from spack.architecture import Platform, OperatingSystem + +class MacOSX(OperatingSystem): + def __init__(self): + """ Autodetects the mac version from a dictionary. Goes back as + far as 10.6 snowleopard. If the user has an older mac then + the version will just be a generic mac_os. + """ + + def get_mac_release(): + mac_releases = {'10.6': "snowleopard", + "10.7": "lion", + "10.8": "mountainlion", + "10.9": "mavericks", + "10.10": "yosemite", + "10.11": "elcapitan"} + + mac_ver = py_platform.mac_ver()[0][:-2] + try: + name = mac_releases[mac_ver] + return name, mac_ver + except KeyError: + name = "mac_os" + return name, mac_ver + + name, version = get_mac_release() + + super(MacOSX, self).__init__(name, version) + + @property + def compiler_strategy(self): + return "PATH" + + + + -- cgit v1.2.3-60-g2f50 From a385dae1aee6b7624707f42af8a6c52b9d9889bd Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:54:30 -0800 Subject: Unit tests to test operating system subclass creation and whether compiler strategy is set correctly --- lib/spack/spack/test/operating_system.py | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 lib/spack/spack/test/operating_system.py (limited to 'lib') diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py new file mode 100644 index 0000000000..9d6850bfa6 --- /dev/null +++ b/lib/spack/spack/test/operating_system.py @@ -0,0 +1,41 @@ +""" Test checks if the operating_system class is created correctly and that +the functions are using the correct operating_system. Also checks whether +the operating_system correctly uses the compiler_strategy +""" + +import unittest +import os +import platform +from spack.platforms.cray_xc import CrayXc +from spack.platforms.linux import Linux +from spack.platforms.darwin import Darwin +from spack.operating_system.linux_distro import LinuxDistro +from spack.operating_system.mac_osx import MacOSX + +class TestOperatingSystem(unittest.TestCase): + + def setUp(self): + cray_xc = CrayXc() + linux = Linux() + darwin = Darwin() + self.cray_operating_sys = cray_xc.operating_system('front_os') + self.darwin_operating_sys = darwin.operating_system('default_os') + self.linux_operating_sys = linux.operating_system('default_os') + + def test_cray_front_end_operating_system(self): + self.assertIsInstance(self.cray_operating_sys, LinuxDistro) + + def test_cray_front_end_compiler_strategy(self): + self.assertEquals(self.cray_operating_sys.compiler_strategy, "PATH") + + def test_linux_operating_system(self): + print self.linux_operating_sys + self.assertIsInstance(self.linux_operating_sys, LinuxDistro) + + def test_linux_compiler_strategy(self): + self.assertEquals(self.linux_operating_sys.compiler_strategy, "PATH") + + + + + -- cgit v1.2.3-60-g2f50 From ebc5e26c2d95971f414378631089fee270beeb51 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 16:16:20 -0800 Subject: Added compiler strategy field to Operating System class --- lib/spack/spack/architecture.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3ba45f3d82..574076bae4 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -209,26 +209,28 @@ class OperatingSystem(object): compiler finding logic. Instead of calling two separate methods to find compilers we call find_compilers method for each operating system """ - def __init__(self, name, version): + + def __init__(self, name, version, compiler_strategy): self.name = name self.version = version + self.compiler_strategy = compiler_strategy def __str__(self): return self.name + self.version def __repr__(self): return self.__str__() - - def compiler_strategy(self): - """ The compiler strategy will be overwritten by the subclass. - Depending on where it comes from it will either use compilers - based off of MODULES search method or the PATH search method - """ - raise NotImplementedError() def _cmp_key(self): return (self.name, self.version) + def compiler_strategy(self): + """ Operating Systems will now be in charge of the compiler finding + strategy. They will each have their own find_compilers method + which depending on their compiler strategy will find the compilers + using a specific method (i.e PATH vs MODULES) + """ + raise NotImplementedError() #NOTE: Key error caused because Architecture has no comparison method @key_ordering -- cgit v1.2.3-60-g2f50 From 3c87d137a3f0823289c52ffff87e58b8b1d7e271 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 16:17:33 -0800 Subject: Added compiler strategy entry and also added new operating system subclass called CNL for compute node linux (name will probably change because I can't find docs on the EXACT name) --- lib/spack/spack/operating_system/cnl.py | 20 ++++++++++++++++++++ lib/spack/spack/operating_system/linux_distro.py | 10 +++++++--- lib/spack/spack/operating_system/mac_osx.py | 9 ++++++--- 3 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 lib/spack/spack/operating_system/cnl.py (limited to 'lib') diff --git a/lib/spack/spack/operating_system/cnl.py b/lib/spack/spack/operating_system/cnl.py new file mode 100644 index 0000000000..e52052dfa8 --- /dev/null +++ b/lib/spack/spack/operating_system/cnl.py @@ -0,0 +1,20 @@ +import spack +from spack.architecture import OperatingSystem + +class ComputeNodeLinux(OperatingSystem): + """ Compute Node Linux (CNL) is the operating system used for the Cray XC + series super computers. It is a very stripped down version of GNU/Linux. + Any compilers found through this operating system will be used with + modules. If updated, user must make sure that version and name are + updated to indicate that OS has been upgraded (or downgraded) + """ + def __init__(self): + name = 'CNL' + version = '10' + super(ComputeNodeLinux, self).__init__(name, version, "MODULES") + + def compiler_strategy(self): + return self.compiler_strategy + + def find_compilers(self): + pass diff --git a/lib/spack/spack/operating_system/linux_distro.py b/lib/spack/spack/operating_system/linux_distro.py index 30a85fe61c..b11c7a88fa 100644 --- a/lib/spack/spack/operating_system/linux_distro.py +++ b/lib/spack/spack/operating_system/linux_distro.py @@ -17,9 +17,13 @@ class LinuxDistro(OperatingSystem): name, version = detect_operating_system() - super(LinuxDistro, self).__init__(name, version) + super(LinuxDistro, self).__init__(name, version, "PATH") - @property def compiler_strategy(self): - return "PATH" + return self.compiler_strategy + + def find_compilers(self): + pass + + diff --git a/lib/spack/spack/operating_system/mac_osx.py b/lib/spack/spack/operating_system/mac_osx.py index 0b939a5546..f52cdd6bc7 100644 --- a/lib/spack/spack/operating_system/mac_osx.py +++ b/lib/spack/spack/operating_system/mac_osx.py @@ -34,11 +34,14 @@ class MacOSX(OperatingSystem): name, version = get_mac_release() - super(MacOSX, self).__init__(name, version) + super(MacOSX, self).__init__(name, version, "PATH") - @property def compiler_strategy(self): - return "PATH" + return self.compiler_strategy + + def find_compilers(self): + pass + -- cgit v1.2.3-60-g2f50 From 083b7b46d92605269df05a9267657acf5e4b4677 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 16:17:54 -0800 Subject: Added more tests --- lib/spack/spack/test/operating_system.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py index 9d6850bfa6..a4327dde83 100644 --- a/lib/spack/spack/test/operating_system.py +++ b/lib/spack/spack/test/operating_system.py @@ -11,6 +11,7 @@ from spack.platforms.linux import Linux from spack.platforms.darwin import Darwin from spack.operating_system.linux_distro import LinuxDistro from spack.operating_system.mac_osx import MacOSX +from spack.operating_system.cnl import ComputeNodeLinux class TestOperatingSystem(unittest.TestCase): @@ -19,6 +20,8 @@ class TestOperatingSystem(unittest.TestCase): linux = Linux() darwin = Darwin() self.cray_operating_sys = cray_xc.operating_system('front_os') + self.cray_default_os = cray_xc.operating_system('default_os') + self.cray_back_os = cray_xc.operating_system('back_os') self.darwin_operating_sys = darwin.operating_system('default_os') self.linux_operating_sys = linux.operating_system('default_os') @@ -28,6 +31,12 @@ class TestOperatingSystem(unittest.TestCase): def test_cray_front_end_compiler_strategy(self): self.assertEquals(self.cray_operating_sys.compiler_strategy, "PATH") + def test_cray_back_end_operating_system(self): + self.assertIsInstance(self.cray_back_os,ComputeNodeLinux) + + def test_cray_back_end_compiler_strategy(self): + self.assertEquals(self.cray_back_os.compiler_strategy, "MODULES") + def test_linux_operating_system(self): print self.linux_operating_sys self.assertIsInstance(self.linux_operating_sys, LinuxDistro) @@ -35,7 +44,16 @@ class TestOperatingSystem(unittest.TestCase): def test_linux_compiler_strategy(self): self.assertEquals(self.linux_operating_sys.compiler_strategy, "PATH") - + + def test_cray_front_end_compiler_list(self): + """ Operating systems will now be in charge of finding compilers. + So, depending on which operating system you want to build for + or which operating system you are on, then you could detect + compilers in a certain way. Cray linux environment on the front + end is just a regular linux distro whereas the Cray linux compute + node is a stripped down version which modules are important + """ + self.assertEquals(True, False) -- cgit v1.2.3-60-g2f50 From cfa7c4feb8419f82a864f9d1afcab69965e9c996 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 16:18:25 -0800 Subject: Added CNL10 as back_os and default_os --- lib/spack/spack/platforms/cray_xc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index b58c84d3eb..dffa2e4031 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -2,6 +2,7 @@ import os from spack.architecture import Platform, Target from spack.operating_system.linux_distro import LinuxDistro +from spack.operating_system.cnl import ComputeNodeLinux class CrayXc(Platform): priority = 20 @@ -10,8 +11,8 @@ class CrayXc(Platform): default = 'ivybridge' front_os = "SuSE11" - back_os = "CNL" - default_os = "CNL" + back_os = "CNL10" + default_os = "CNL10" def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better @@ -38,6 +39,7 @@ class CrayXc(Platform): Target('haswell', 'MODULES', 'craype-haswell')) self.add_operating_system('SuSE11', LinuxDistro()) + self.add_operating_system('CNL10', ComputeNodeLinux()) @classmethod def detect(self): -- cgit v1.2.3-60-g2f50 From 5d5d3c5858a8882b615fb97fb07e335dc6c17f14 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 12:15:52 -0800 Subject: Added to dictionary method --- lib/spack/spack/architecture.py | 95 ++++++++++++++++++++++++++++++++--------- 1 file changed, 74 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 574076bae4..109e599470 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -89,7 +89,7 @@ class Target(object): return target def _cmp_key(self): - return (self.name, self.compiler_strategy, self.module_name) + return (self.name, self.module_name) def __repr__(self): return self.__str__() @@ -200,7 +200,6 @@ class Platform(object): def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values()), (_cmp_key(o) for o in self.operating_sys.values())) - @key_ordering class OperatingSystem(object): @@ -221,17 +220,19 @@ class OperatingSystem(object): def __repr__(self): return self.__str__() - def _cmp_key(self): - return (self.name, self.version) def compiler_strategy(self): """ Operating Systems will now be in charge of the compiler finding strategy. They will each have their own find_compilers method which depending on their compiler strategy will find the compilers - using a specific method (i.e PATH vs MODULES) + using a specific method (i.e PATH vs MODULES). """ raise NotImplementedError() + def _cmp_key(self): + return (self.name, self.version, self.compiler_strategy) + + #NOTE: Key error caused because Architecture has no comparison method @key_ordering class Arch(namedtuple("Arch", "platform platform_os target")): @@ -241,24 +242,77 @@ class Arch(namedtuple("Arch", "platform platform_os target")): __slots__ = () def __str__(self): - return (self.platform.name +"-"+ - self.platform_os.name + "-" + self.target.name) - + return (self.platform.name +"-"+ self.platform_os.name + "-" + self.target.name) + def _cmp_key(self): return (self.platform.name, self.platform_os.name, self.target.name) - @staticmethod - def to_dict(platform, os_name, target): - """ This function will handle all the converting of dictionaries - for each of the architecture classes. This is better than having - each class have a to_dict method when creating the dict of dicts - arch class - """ - d = {} - d['platform'] = platform - d['operating_system'] = os_name - d['target'] = target.to_dict() - return d + +def _helper_to_dict(arch_field_dict, arch_field_name, *args): + """ General method to turn each class in architecture into a + dictionary. Takes as argument the class dictionary, + """ + d = {} + d[arch_field_name] = {} + for items in args: + d[arch_field_name][items] = arch_field_dict[items] + return d + +def to_dict(arch): + d = {} + d['architecture'] = {} + + platform = arch.platform.__dict__ + platform_os = arch.platform_os.__dict__ + target = arch.target.__dict__ + + platform_dict = _helper_to_dict(platform,'platform','name') + os_dict = _helper_to_dict(platform_os, 'platform_os', 'name','version', + 'compiler_strategy') + target_dict = _helper_to_dict(target,'target', 'name', + 'module_name','platform_name') + + d['architecture'].update(platform_dict) + d['architecture'].update(os_dict) + d['architecture'].update(target_dict) + + return d + +def _platform_from_dict(platform): + platform_list = all_platforms() + platform_names = {plat.__name__.lower():plat for plat in platform_list} + return platform_names[platform['name']]() + + +def _target_from_dict(target): + target = Target.__new__(Target) + target.name = d['name'] + target.compiler_strategy = d['compiler_strategy'] + target.module_name = d['module_name'] + if 'platform' in d: + target.platform_name = d['platform'] + return target + +def _operating_system_from_dict(os_dict): + pass + +def arch_from_dict(d): + if d is None: + return None + arch = Arch + platform_dict = d['platform'] + platform_os_dict = d['platform_os'] + target_dict = d['target'] + + platform = _platform_from_dict(platform_dict) + platform_os = _operating_system_from_dict(platform_os_dict) + target = _target_from_dict(target_dict) + + arch.platform = platform + arch.platform_os = platform_os + arch.target = target + + return arch @memoized def all_platforms(): @@ -278,7 +332,6 @@ def all_platforms(): return modules - @memoized def sys_type(): """ Gather a list of all available subclasses of platforms. -- cgit v1.2.3-60-g2f50 From 339f4bfd23b097d009009b6dabd524f8bc831449 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 12:16:09 -0800 Subject: Added unit testing for to_dict method --- lib/spack/spack/test/architecture.py | 43 ++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index cf7938f5d6..5bc78d1b65 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -6,19 +6,58 @@ import os import platform import spack from spack.architecture import * +import spack.spec from spack.platforms.cray_xc import CrayXc from spack.platforms.linux import Linux from spack.platforms.bgq import Bgq from spack.platforms.darwin import Darwin class ArchitectureTest(unittest.TestCase): + + def setUp(self): + zlib = spack.spec.Spec("zlib") + zlib.concretize() + self.architecture = zlib.architecture + self.platform = sys_type() + self.platform_os = self.platform.operating_system('default_os') + self.target = self.platform.target('default') + + #def test_to_dict_function_with_target(self): + # d = spack.architecture.to_dict(self.architecture) + # print d['target'] + # self.assertEquals(d['target'], {'name': self.target.name, + # 'module_name' : self.target.module_name, + # 'platform_name' : self.target.platform_name, + # 'compiler_strategy': 'MODULES' + # }) + + def test_to_dict_function_with_architecture(self): + d = spack.architecture.to_dict(self.architecture) + self.assertEquals(d, {'architecture': + {'platform' : {'name': 'crayxc'}, + 'platform_os': { + 'compiler_strategy': 'MODULES', + 'name':'CNL', + 'version':'10'}, + 'target' : {'platform_name' :'crayxc', + 'module_name': 'craype-haswell', + 'name':'haswell'}}}) + + #def test_to_dict_function_with_operating_system(self): + # d = spack.architecture.to_dict(self.architecture) + # self.assertEquals(d['platform_os'], {'name': self.platform_os.name, + # 'version': self.platform_os.version, + # 'compiler_strategy': self.platform_os.compiler_strategy}) + + def test_architecture_from_dict(self): + pass def test_platform_class_and_compiler_strategies(self): a = CrayXc() - t = a.target('default') + t = a.operating_system('default_os') self.assertEquals(t.compiler_strategy, 'MODULES') b = Linux() - s = b.target('default') + s = b.operating_system('default_os') self.assertEquals(s.compiler_strategy, 'PATH') def test_sys_type(self): -- cgit v1.2.3-60-g2f50 From 77799741430796974809fce0f08d099303ec6da6 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:02:40 -0800 Subject: arch_from_dict worked on --- lib/spack/spack/architecture.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 109e599470..f0fc00242c 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -294,7 +294,11 @@ def _target_from_dict(target): return target def _operating_system_from_dict(os_dict): - pass + name = os_dict['name'] + os_list = all_platforms(True) + os_classes = {o.__name__.lower():o for o in os_list} + return os_classes[name]() + def arch_from_dict(d): if d is None: @@ -315,11 +319,19 @@ def arch_from_dict(d): return arch @memoized -def all_platforms(): +def all_platforms(operating_system=False): modules = [] - for name in list_modules(spack.platform_path): - mod_name = 'spack.platformss' + name - path = join_path(spack.platform_path, name) + ".py" + + if operating_system: + mod_path = spack.operating_system_path + mod_string = "spack.operating_system." + else: + mod_path = spack.platform_path + mod_string = "spack.platformss" + + for name in list_modules(mod_path): + mod_name = mod_string + name + path = join_path(mod_path, name) + ".py" mod = imp.load_source(mod_name, path) class_name = mod_to_class(name) if not hasattr(mod, class_name): -- cgit v1.2.3-60-g2f50 From 9e844d974c2bce42b3e6e647c16b10aa122e54ce Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:30:20 -0800 Subject: Added comments. Need to figure out how to make arch_tuple from_dict --- lib/spack/spack/architecture.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index f0fc00242c..4b24680501 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -250,7 +250,8 @@ class Arch(namedtuple("Arch", "platform platform_os target")): def _helper_to_dict(arch_field_dict, arch_field_name, *args): """ General method to turn each class in architecture into a - dictionary. Takes as argument the class dictionary, + dictionary. Takes as argument the class dictionary, the field name + (platform, platform_os, target) and then any attribute args """ d = {} d[arch_field_name] = {} @@ -294,10 +295,11 @@ def _target_from_dict(target): return target def _operating_system_from_dict(os_dict): + #TODO: Have a way to recognize the OS subclasses name = os_dict['name'] os_list = all_platforms(True) - os_classes = {o.__name__.lower():o for o in os_list} - return os_classes[name]() + os_classes = {o.__name__:o for o in os_list} + def arch_from_dict(d): @@ -318,6 +320,7 @@ def arch_from_dict(d): return arch +#TODO: Haven't changed name here but all_platforms should now pull os class list @memoized def all_platforms(operating_system=False): modules = [] -- cgit v1.2.3-60-g2f50 From 2de81cfc62ca456b1f303d64198590eb003fdcd3 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:31:21 -0800 Subject: Changed name to appropriate camelcase --- lib/spack/spack/operating_system/cnl.py | 4 +-- lib/spack/spack/operating_system/linux_distro.py | 10 ++---- lib/spack/spack/operating_system/mac_osx.py | 42 ++++++++++-------------- 3 files changed, 22 insertions(+), 34 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/operating_system/cnl.py b/lib/spack/spack/operating_system/cnl.py index e52052dfa8..07221630f2 100644 --- a/lib/spack/spack/operating_system/cnl.py +++ b/lib/spack/spack/operating_system/cnl.py @@ -1,7 +1,7 @@ import spack from spack.architecture import OperatingSystem -class ComputeNodeLinux(OperatingSystem): +class Cnl(OperatingSystem): """ Compute Node Linux (CNL) is the operating system used for the Cray XC series super computers. It is a very stripped down version of GNU/Linux. Any compilers found through this operating system will be used with @@ -11,7 +11,7 @@ class ComputeNodeLinux(OperatingSystem): def __init__(self): name = 'CNL' version = '10' - super(ComputeNodeLinux, self).__init__(name, version, "MODULES") + super(Cnl, self).__init__(name, version, "MODULES") def compiler_strategy(self): return self.compiler_strategy diff --git a/lib/spack/spack/operating_system/linux_distro.py b/lib/spack/spack/operating_system/linux_distro.py index b11c7a88fa..1345b7d618 100644 --- a/lib/spack/spack/operating_system/linux_distro.py +++ b/lib/spack/spack/operating_system/linux_distro.py @@ -1,6 +1,6 @@ import platform as py_platform import spack -from spack.architecture import Platform, OperatingSystem +from spack.architecture import OperatingSystem class LinuxDistro(OperatingSystem): """ This class will represent the autodetected operating system @@ -10,13 +10,9 @@ class LinuxDistro(OperatingSystem): platform.dist() """ def __init__(self): - def detect_operating_system(): - name = py_platform.dist()[0] - version = py_platform.dist()[1] - return name, version + name = py_platform.dist()[0] + version = py_platform.dist()[1] - name, version = detect_operating_system() - super(LinuxDistro, self).__init__(name, version, "PATH") def compiler_strategy(self): diff --git a/lib/spack/spack/operating_system/mac_osx.py b/lib/spack/spack/operating_system/mac_osx.py index f52cdd6bc7..6fb6fef138 100644 --- a/lib/spack/spack/operating_system/mac_osx.py +++ b/lib/spack/spack/operating_system/mac_osx.py @@ -3,38 +3,30 @@ will be represented using the major version operating system name, i.e el capitan, yosemite...etc. """ - -import spack -import os import platform as py_platform -from spack.architecture import Platform, OperatingSystem +import spack +from spack.architecture import OperatingSystem -class MacOSX(OperatingSystem): +class MacOsx(OperatingSystem): def __init__(self): """ Autodetects the mac version from a dictionary. Goes back as far as 10.6 snowleopard. If the user has an older mac then the version will just be a generic mac_os. """ - - def get_mac_release(): - mac_releases = {'10.6': "snowleopard", - "10.7": "lion", - "10.8": "mountainlion", - "10.9": "mavericks", - "10.10": "yosemite", - "10.11": "elcapitan"} - - mac_ver = py_platform.mac_ver()[0][:-2] - try: - name = mac_releases[mac_ver] - return name, mac_ver - except KeyError: - name = "mac_os" - return name, mac_ver - - name, version = get_mac_release() - - super(MacOSX, self).__init__(name, version, "PATH") + mac_releases = {'10.6': "snowleopard", + "10.7": "lion", + "10.8": "mountainlion", + "10.9": "mavericks", + "10.10": "yosemite", + "10.11": "elcapitan"} + + mac_ver = py_platform.mac_ver()[0][:-2] + try: + name = mac_releases[mac_ver] + except KeyError: + name = "mac_os" + + super(MacOsx, self).__init__(name, mac_ver, "PATH") def compiler_strategy(self): return self.compiler_strategy -- cgit v1.2.3-60-g2f50 From 234681306cdfb9aea9772781fcb68c4045bdc315 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:32:23 -0800 Subject: Changed to appropriate class names for operating system --- lib/spack/spack/platforms/cray_xc.py | 5 ++--- lib/spack/spack/platforms/darwin.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index dffa2e4031..da90726fef 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -1,8 +1,7 @@ import os - from spack.architecture import Platform, Target from spack.operating_system.linux_distro import LinuxDistro -from spack.operating_system.cnl import ComputeNodeLinux +from spack.operating_system.cnl import Cnl class CrayXc(Platform): priority = 20 @@ -39,7 +38,7 @@ class CrayXc(Platform): Target('haswell', 'MODULES', 'craype-haswell')) self.add_operating_system('SuSE11', LinuxDistro()) - self.add_operating_system('CNL10', ComputeNodeLinux()) + self.add_operating_system('CNL10', Cnl()) @classmethod def detect(self): diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 5b797a78db..4512857371 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,6 +1,6 @@ import subprocess from spack.architecture import Platform, Target -from spack.operating_system.mac_osx import MacOSX +from spack.operating_system.mac_osx import MacOsx class Darwin(Platform): priority = 89 @@ -11,7 +11,7 @@ class Darwin(Platform): def __init__(self): super(Darwin, self).__init__('darwin') self.add_target(self.default, Target(self.default, 'PATH')) - mac_os = MacOSX() + mac_os = MacOsx() self.default_os = mac_os.name self.add_operating_system(mac_os.name, mac_os) -- cgit v1.2.3-60-g2f50 From 5c3c6e7f066a3f95f9113c46640546516d7a8257 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:34:32 -0800 Subject: Got rid of print statements --- lib/spack/spack/test/operating_system.py | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py index a4327dde83..205408db3f 100644 --- a/lib/spack/spack/test/operating_system.py +++ b/lib/spack/spack/test/operating_system.py @@ -38,7 +38,6 @@ class TestOperatingSystem(unittest.TestCase): self.assertEquals(self.cray_back_os.compiler_strategy, "MODULES") def test_linux_operating_system(self): - print self.linux_operating_sys self.assertIsInstance(self.linux_operating_sys, LinuxDistro) def test_linux_compiler_strategy(self): -- cgit v1.2.3-60-g2f50 From bd7c189c123154e88e092eadbbd93b00ce43cc6a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:37:50 -0800 Subject: Added more comments --- lib/spack/spack/spec.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 0b42498910..c8c114a8b1 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -661,12 +661,14 @@ class Spec(object): 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } + if self.architecture: # TODO: Fix the target.to_dict to account for the tuple # Want it to be a dict of dicts - d['architecture'] = self.architecture.target.to_dict() + d['architecture'] = spack.architecture.to_dict(self.architecture) else: d['architecture'] = None + if self.compiler: d.update(self.compiler.to_dict()) else: @@ -693,8 +695,7 @@ class Spec(object): spec = Spec(name) spec.versions = VersionList.from_dict(node) # TODO: Need to fix the architecture.Target.from_dict - spec.architecture = spack.architecture.Target.from_dict( - node['architecture']) + spec.architecture = spack.architecture.from_dict(node['architecture']) if node['compiler'] is None: spec.compiler = None @@ -1292,17 +1293,17 @@ class Spec(object): for entry in arch_list: if self._is_valid_platform(entry, platform_names): + # If entry is different from platform name then create it. + # else just keep the already instantiated platform class if entry != platform.name: platform = platform_dict[entry]() # Create instance of platform elif self._is_valid_target(entry, platform): target = platform.target(entry) - # Need to figure out if we're supporting arbitrary os's and how - # to account for them - # Not really a good implementation since the user can add - # gibberish and spack will see it as an os + # check if os is valid by checking platform operating sys dict elif self._is_valid_os(entry, platform): platform_os = platform.operating_system(entry) else: + # throw error since entry is unknown raise UnknownArchitectureSpecError(entry) if target is None: -- cgit v1.2.3-60-g2f50 From 8d1c06d14125d86c24584c7e6f0a9ab0d300d10f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:38:33 -0800 Subject: Changed to dict method. Didn't realize this was being assigned to a dictionary already so got rid of d['architecture'] --- lib/spack/spack/architecture.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 4b24680501..e76e411ae0 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -261,8 +261,7 @@ def _helper_to_dict(arch_field_dict, arch_field_name, *args): def to_dict(arch): d = {} - d['architecture'] = {} - + platform = arch.platform.__dict__ platform_os = arch.platform_os.__dict__ target = arch.target.__dict__ @@ -273,9 +272,9 @@ def to_dict(arch): target_dict = _helper_to_dict(target,'target', 'name', 'module_name','platform_name') - d['architecture'].update(platform_dict) - d['architecture'].update(os_dict) - d['architecture'].update(target_dict) + d.update(platform_dict) + d.update(os_dict) + d.update(target_dict) return d -- cgit v1.2.3-60-g2f50 From 863a5bc4fa7d6b880769192e4fcb7a461a6a0063 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 10:34:55 -0800 Subject: Fix for 'Parent module spack.operating_systems not found while handling absolute import' --- lib/spack/spack/architecture.py | 42 ++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e76e411ae0..5827427003 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -67,26 +67,26 @@ class Target(object): def set_platform(self, platform): self.platform_name = platform.name - def to_dict(self): - d = {} - d['name'] = self.name - d['compiler_strategy'] = self.compiler_strategy - d['module_name'] = self.module_name - if self.platform_name: - d['platform'] = self.platform_name - return d - - @staticmethod - def from_dict(d): - if d is None: - return None - target = Target.__new__(Target) - target.name = d['name'] - target.compiler_strategy = d['compiler_strategy'] - target.module_name = d['module_name'] - if 'platform' in d: - target.platform_name = d['platform'] - return target + #def to_dict(self): + # d = {} + # d['name'] = self.name + # d['compiler_strategy'] = self.compiler_strategy + # d['module_name'] = self.module_name + # if self.platform_name: + # d['platform'] = self.platform_name + # return d + + #@staticmethod + #def from_dict(d): + # if d is None: + # return None + # target = Target.__new__(Target) + # target.name = d['name'] + # target.compiler_strategy = d['compiler_strategy'] + # target.module_name = d['module_name'] + # if 'platform' in d: + # target.platform_name = d['platform'] + # return target def _cmp_key(self): return (self.name, self.module_name) @@ -326,7 +326,7 @@ def all_platforms(operating_system=False): if operating_system: mod_path = spack.operating_system_path - mod_string = "spack.operating_system." + mod_string = "spack.operating_systems" else: mod_path = spack.platform_path mod_string = "spack.platformss" -- cgit v1.2.3-60-g2f50 From 34be473b7cfbb23d4deb5c3f9774bc1acea22574 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 10:35:58 -0800 Subject: Deleted files --- lib/spack/spack/operating_systems/cnl.py | 19 +++++++++++ lib/spack/spack/operating_systems/linux_distro.py | 24 ++++++++++++++ lib/spack/spack/operating_systems/mac_osx.py | 40 +++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 lib/spack/spack/operating_systems/cnl.py create mode 100644 lib/spack/spack/operating_systems/linux_distro.py create mode 100644 lib/spack/spack/operating_systems/mac_osx.py (limited to 'lib') diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py new file mode 100644 index 0000000000..20cb4c215b --- /dev/null +++ b/lib/spack/spack/operating_systems/cnl.py @@ -0,0 +1,19 @@ +from spack.architecture import OperatingSystem + +class Cnl(OperatingSystem): + """ Compute Node Linux (CNL) is the operating system used for the Cray XC + series super computers. It is a very stripped down version of GNU/Linux. + Any compilers found through this operating system will be used with + modules. If updated, user must make sure that version and name are + updated to indicate that OS has been upgraded (or downgraded) + """ + def __init__(self): + name = 'CNL' + version = '10' + super(Cnl, self).__init__(name, version, "MODULES") + + def compiler_strategy(self): + return self.compiler_strategy + + def find_compilers(self): + pass diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py new file mode 100644 index 0000000000..c5547749d8 --- /dev/null +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -0,0 +1,24 @@ +import platform as py_platform +from spack.architecture import OperatingSystem + +class LinuxDistro(OperatingSystem): + """ This class will represent the autodetected operating system + for a Linux System. Since there are many different flavors of + Linux, this class will attempt to encompass them all through + autodetection using the python module platform and the method + platform.dist() + """ + def __init__(self): + name = py_platform.dist()[0] + version = py_platform.dist()[1] + + super(LinuxDistro, self).__init__(name, version, "PATH") + + def compiler_strategy(self): + return self.compiler_strategy + + def find_compilers(self): + pass + + + diff --git a/lib/spack/spack/operating_systems/mac_osx.py b/lib/spack/spack/operating_systems/mac_osx.py new file mode 100644 index 0000000000..555c5cfb14 --- /dev/null +++ b/lib/spack/spack/operating_systems/mac_osx.py @@ -0,0 +1,40 @@ +import platform as py_platform +from spack.architecture import OperatingSystem + +class MacOsx(OperatingSystem): + """ This class represents the MAC_OSX operating system. This will be auto + detected using the python platform.mac_ver. The MAC_OSX platform + will be represented using the major version operating system name, i.e + el capitan, yosemite...etc. + """ + + def __init__(self): + """ Autodetects the mac version from a dictionary. Goes back as + far as 10.6 snowleopard. If the user has an older mac then + the version will just be a generic mac_os. + """ + mac_releases = {'10.6': "snowleopard", + "10.7": "lion", + "10.8": "mountainlion", + "10.9": "mavericks", + "10.10": "yosemite", + "10.11": "elcapitan"} + + mac_ver = py_platform.mac_ver()[0][:-2] + try: + name = mac_releases[mac_ver] + except KeyError: + name = "mac_os" + + super(MacOsx, self).__init__(name, mac_ver, "PATH") + + def compiler_strategy(self): + return self.compiler_strategy + + def find_compilers(self): + pass + + + + + -- cgit v1.2.3-60-g2f50 From f1616099292251f208ec0cfdec3d1b092d4441b9 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 10:36:12 -0800 Subject: Renamed operating_system to operating_systems --- lib/spack/spack/operating_systems/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lib/spack/spack/operating_systems/__init__.py (limited to 'lib') diff --git a/lib/spack/spack/operating_systems/__init__.py b/lib/spack/spack/operating_systems/__init__.py new file mode 100644 index 0000000000..e69de29bb2 -- cgit v1.2.3-60-g2f50 From b43a5498a159c634ca88341d3010bc1c98197e24 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 10:36:47 -0800 Subject: Changed import path to operating_systems --- lib/spack/spack/platforms/bgq.py | 1 - lib/spack/spack/platforms/cray_xc.py | 4 ++-- lib/spack/spack/platforms/darwin.py | 2 +- lib/spack/spack/platforms/linux.py | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/platforms/bgq.py b/lib/spack/spack/platforms/bgq.py index 6e872d2e72..b550f1d5cb 100644 --- a/lib/spack/spack/platforms/bgq.py +++ b/lib/spack/spack/platforms/bgq.py @@ -1,5 +1,4 @@ import os - from spack.architecture import Platform, Target class Bgq(Platform): diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index da90726fef..ea3ae0e477 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -1,7 +1,7 @@ import os from spack.architecture import Platform, Target -from spack.operating_system.linux_distro import LinuxDistro -from spack.operating_system.cnl import Cnl +from spack.operating_systems.linux_distro import LinuxDistro +from spack.operating_systems.cnl import Cnl class CrayXc(Platform): priority = 20 diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 4512857371..c48f7d17d2 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,6 +1,6 @@ import subprocess from spack.architecture import Platform, Target -from spack.operating_system.mac_osx import MacOsx +from spack.operating_systems.mac_osx import MacOsx class Darwin(Platform): priority = 89 diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 220f1bbaf4..1b22b69048 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -1,6 +1,6 @@ import subprocess from spack.architecture import Platform, Target -from spack.operating_system.linux_distro import LinuxDistro +from spack.operating_systems.linux_distro import LinuxDistro class Linux(Platform): priority = 90 -- cgit v1.2.3-60-g2f50 From 9ac2556285ae38dd6d661d9bba811a5e007cb551 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 10:41:28 -0800 Subject: Deleted old operating_system (without the s) file --- lib/spack/spack/operating_system/__init__.py | 0 lib/spack/spack/operating_system/cnl.py | 20 ------------ lib/spack/spack/operating_system/linux_distro.py | 25 --------------- lib/spack/spack/operating_system/mac_osx.py | 40 ------------------------ 4 files changed, 85 deletions(-) delete mode 100644 lib/spack/spack/operating_system/__init__.py delete mode 100644 lib/spack/spack/operating_system/cnl.py delete mode 100644 lib/spack/spack/operating_system/linux_distro.py delete mode 100644 lib/spack/spack/operating_system/mac_osx.py (limited to 'lib') diff --git a/lib/spack/spack/operating_system/__init__.py b/lib/spack/spack/operating_system/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lib/spack/spack/operating_system/cnl.py b/lib/spack/spack/operating_system/cnl.py deleted file mode 100644 index 07221630f2..0000000000 --- a/lib/spack/spack/operating_system/cnl.py +++ /dev/null @@ -1,20 +0,0 @@ -import spack -from spack.architecture import OperatingSystem - -class Cnl(OperatingSystem): - """ Compute Node Linux (CNL) is the operating system used for the Cray XC - series super computers. It is a very stripped down version of GNU/Linux. - Any compilers found through this operating system will be used with - modules. If updated, user must make sure that version and name are - updated to indicate that OS has been upgraded (or downgraded) - """ - def __init__(self): - name = 'CNL' - version = '10' - super(Cnl, self).__init__(name, version, "MODULES") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass diff --git a/lib/spack/spack/operating_system/linux_distro.py b/lib/spack/spack/operating_system/linux_distro.py deleted file mode 100644 index 1345b7d618..0000000000 --- a/lib/spack/spack/operating_system/linux_distro.py +++ /dev/null @@ -1,25 +0,0 @@ -import platform as py_platform -import spack -from spack.architecture import OperatingSystem - -class LinuxDistro(OperatingSystem): - """ This class will represent the autodetected operating system - for a Linux System. Since there are many different flavors of - Linux, this class will attempt to encompass them all through - autodetection using the python module platform and the method - platform.dist() - """ - def __init__(self): - name = py_platform.dist()[0] - version = py_platform.dist()[1] - - super(LinuxDistro, self).__init__(name, version, "PATH") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass - - - diff --git a/lib/spack/spack/operating_system/mac_osx.py b/lib/spack/spack/operating_system/mac_osx.py deleted file mode 100644 index 6fb6fef138..0000000000 --- a/lib/spack/spack/operating_system/mac_osx.py +++ /dev/null @@ -1,40 +0,0 @@ -""" This class represents the MAC_OSX operating system. This will be auto - detected using the python platform.mac_ver. The MAC_OSX platform - will be represented using the major version operating system name, i.e - el capitan, yosemite...etc. -""" -import platform as py_platform -import spack -from spack.architecture import OperatingSystem - -class MacOsx(OperatingSystem): - def __init__(self): - """ Autodetects the mac version from a dictionary. Goes back as - far as 10.6 snowleopard. If the user has an older mac then - the version will just be a generic mac_os. - """ - mac_releases = {'10.6': "snowleopard", - "10.7": "lion", - "10.8": "mountainlion", - "10.9": "mavericks", - "10.10": "yosemite", - "10.11": "elcapitan"} - - mac_ver = py_platform.mac_ver()[0][:-2] - try: - name = mac_releases[mac_ver] - except KeyError: - name = "mac_os" - - super(MacOsx, self).__init__(name, mac_ver, "PATH") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass - - - - - -- cgit v1.2.3-60-g2f50 From 550df4cdd6ce1760370e18a858da624167a3139a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 11:47:27 -0800 Subject: Some cleaning up. Finally got arch_from_dict working successfully. --- lib/spack/spack/architecture.py | 81 ++++++++--------------------------------- 1 file changed, 15 insertions(+), 66 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 5827427003..7bf5afcf63 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -67,27 +67,6 @@ class Target(object): def set_platform(self, platform): self.platform_name = platform.name - #def to_dict(self): - # d = {} - # d['name'] = self.name - # d['compiler_strategy'] = self.compiler_strategy - # d['module_name'] = self.module_name - # if self.platform_name: - # d['platform'] = self.platform_name - # return d - - #@staticmethod - #def from_dict(d): - # if d is None: - # return None - # target = Target.__new__(Target) - # target.name = d['name'] - # target.compiler_strategy = d['compiler_strategy'] - # target.module_name = d['module_name'] - # if 'platform' in d: - # target.platform_name = d['platform'] - # return target - def _cmp_key(self): return (self.name, self.module_name) @@ -147,24 +126,6 @@ class Platform(object): name = self.back_end return self.targets[name] - - #def _detect_linux_os(self): - # return OperatingSystem(py_platform.dist()[0], py_platform.dist()[1]) - - #def _detect_mac_os(self): - # mac_releases = {'10.6': "snowleopard", - # "10.7": "lion", - # "10.8": "mountainlion", - # "10.9": "mavericks", - # "10.10": "yosemite", - # "10.11": "elcapitan"} - # mac_ver = py_platform.mac_ver()[:-2] - # try: - # os_name = mac_releases[mac_ver] - # return OperatingSystem(os_name, mac_ver) - # except KeyError: - # os_name = "mac_os" - # return OperatingSystem(os_name, mac_ver) def add_operating_system(self, name, os_class): """ Add the operating_system class object into the @@ -242,7 +203,8 @@ class Arch(namedtuple("Arch", "platform platform_os target")): __slots__ = () def __str__(self): - return (self.platform.name +"-"+ self.platform_os.name + "-" + self.target.name) + return (self.platform.name +"-"+ + self.platform_os.name + "-" + self.target.name) def _cmp_key(self): return (self.platform.name, self.platform_os.name, self.target.name) @@ -284,52 +246,39 @@ def _platform_from_dict(platform): return platform_names[platform['name']]() -def _target_from_dict(target): +def _target_from_dict(target_dict): target = Target.__new__(Target) - target.name = d['name'] - target.compiler_strategy = d['compiler_strategy'] - target.module_name = d['module_name'] - if 'platform' in d: - target.platform_name = d['platform'] + target.name = target_dict['name'] + #target.compiler_strategy = target_dict['compiler_strategy'] + target.module_name = target_dict['module_name'] + if 'platform_name' in target_dict: + target.platform_name = target_dict['platform_name'] return target -def _operating_system_from_dict(os_dict): - #TODO: Have a way to recognize the OS subclasses +def _operating_system_from_dict(os_dict, platform_class): name = os_dict['name'] - os_list = all_platforms(True) - os_classes = {o.__name__:o for o in os_list} - + return platform_class.operating_system(name) def arch_from_dict(d): if d is None: return None - arch = Arch platform_dict = d['platform'] platform_os_dict = d['platform_os'] target_dict = d['target'] platform = _platform_from_dict(platform_dict) - platform_os = _operating_system_from_dict(platform_os_dict) + platform_os = _operating_system_from_dict(platform_os_dict, platform) target = _target_from_dict(target_dict) - arch.platform = platform - arch.platform_os = platform_os - arch.target = target - - return arch + return Arch(platform, platform_os, target) -#TODO: Haven't changed name here but all_platforms should now pull os class list @memoized -def all_platforms(operating_system=False): +def all_platforms(): modules = [] - if operating_system: - mod_path = spack.operating_system_path - mod_string = "spack.operating_systems" - else: - mod_path = spack.platform_path - mod_string = "spack.platformss" + mod_path = spack.platform_path + mod_string = "spack.platformss" for name in list_modules(mod_path): mod_name = mod_string + name -- cgit v1.2.3-60-g2f50 From 5e4d3e7b8280e633bb9b9dd28510dc394feb2000 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 11:48:43 -0800 Subject: Changed naming convention of operating_system, makes it easier to create instance from dictionary just using name without the version attached. Object will stil display as name+version --- lib/spack/spack/platforms/cray_xc.py | 10 +++++----- lib/spack/spack/platforms/linux.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index ea3ae0e477..b71481e46e 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -9,9 +9,9 @@ class CrayXc(Platform): back_end = 'ivybridge' default = 'ivybridge' - front_os = "SuSE11" - back_os = "CNL10" - default_os = "CNL10" + front_os = "SuSE" + back_os = "CNL" + default_os = "CNL" def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better @@ -37,8 +37,8 @@ class CrayXc(Platform): self.add_target('haswell', Target('haswell', 'MODULES', 'craype-haswell')) - self.add_operating_system('SuSE11', LinuxDistro()) - self.add_operating_system('CNL10', Cnl()) + self.add_operating_system('SuSE', LinuxDistro()) + self.add_operating_system('CNL', Cnl()) @classmethod def detect(self): diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 1b22b69048..d9e33511a1 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -12,8 +12,8 @@ class Linux(Platform): super(Linux, self).__init__('linux') self.add_target(self.default, Target(self.default, 'PATH')) linux_dist = LinuxDistro() - self.default_os = str(linux_dist) - self.add_operating_system(str(linux_dist), linux_dist) + self.default_os = linux_dist.name + self.add_operating_system(linux_dist.name, linux_dist) @classmethod def detect(self): -- cgit v1.2.3-60-g2f50 From 5989e3f65d83afdd497059bba9442576e2df0509 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 11:53:02 -0800 Subject: Changed from_dict to arch_from_dict method --- lib/spack/spack/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index c8c114a8b1..abe17a8f2b 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -695,7 +695,7 @@ class Spec(object): spec = Spec(name) spec.versions = VersionList.from_dict(node) # TODO: Need to fix the architecture.Target.from_dict - spec.architecture = spack.architecture.from_dict(node['architecture']) + spec.architecture = spack.architecture.arch_from_dict(node['architecture']) if node['compiler'] is None: spec.compiler = None -- cgit v1.2.3-60-g2f50 From ac55ce989d0b5a6e7a1160dc32ac7fb12f805f76 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 18:47:41 -0800 Subject: Changed compiler finding strategy to come from operating system instead of target --- lib/spack/spack/concretize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 0e7bf53b44..3434ca66f7 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -334,9 +334,9 @@ class DefaultConcretizer(object): # Should think whether this can be more efficient def _proper_compiler_style(cspec, architecture): compilers = spack.compilers.compilers_for_spec(cspec) - if architecture.target.compiler_strategy == 'PATH': + if architecture.platform_os.compiler_strategy == 'PATH': filter(lambda c: not c.modules, compilers) - if architecture.target.compiler_strategy == 'MODULES': + if architecture.platform_os.compiler_strategy == 'MODULES': filter(lambda c: c.modules, compilers) return compilers -- cgit v1.2.3-60-g2f50 From f0149faf88b97b63635ba451efe008846478d115 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 18:48:00 -0800 Subject: Added doc strings to functions in architecture.py --- lib/spack/spack/architecture.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 7bf5afcf63..14a8c56793 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -222,6 +222,12 @@ def _helper_to_dict(arch_field_dict, arch_field_name, *args): return d def to_dict(arch): + """ Convert the Arch tuple into a dictionary for yaml dumping. This + uses the _helper_to_dict method to create the dictionary from the + provided architecture field. Can assign the architecture + field name (either platform, platform_os or target) and any + attributes that make up that architecture field, + """ d = {} platform = arch.platform.__dict__ @@ -241,12 +247,19 @@ def to_dict(arch): return d def _platform_from_dict(platform): + """Creates all the platform class module names into a dictionary of + name : key-value pairs. From there we can construct the + platform subclass + """ platform_list = all_platforms() platform_names = {plat.__name__.lower():plat for plat in platform_list} return platform_names[platform['name']]() def _target_from_dict(target_dict): + """ Creates new instance of target and assigns all the attributes of + that target from the dictionary + """ target = Target.__new__(Target) target.name = target_dict['name'] #target.compiler_strategy = target_dict['compiler_strategy'] @@ -256,11 +269,19 @@ def _target_from_dict(target_dict): return target def _operating_system_from_dict(os_dict, platform_class): + """ uses platform's operating system method to grab the constructed + operating systems that are valid on the platform. + """ +# NOTE: Might need a better way to create operating system objects name = os_dict['name'] return platform_class.operating_system(name) def arch_from_dict(d): + """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict + helper methods to recreate the arch tuple from the dictionary read from + a yaml file + """ if d is None: return None platform_dict = d['platform'] -- cgit v1.2.3-60-g2f50 From eb96f3829865492c3c484de0d89e14cba578a37f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 18:48:25 -0800 Subject: Got rid of debug comment --- lib/spack/spack/spec.py | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index abe17a8f2b..0a55ec5d76 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -835,7 +835,6 @@ class Spec(object): changed = True force = False - # Loops forever here in my implementation while changed: changes = (self.normalize(force=force), self._expand_virtual_packages(), -- cgit v1.2.3-60-g2f50 From ccd4a79b39b56a8408980ed27b0c5de5a1995fbc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 18:49:27 -0800 Subject: changed architecture.target to architecture.platform_os, so that compiler_for_spec uses operating system strategy --- lib/spack/spack/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b0d84863de..5d922f6e58 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -594,7 +594,7 @@ class Package(object): if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") return spack.compilers.compiler_for_spec(self.spec.compiler, - self.spec.architecture.target) + self.spec.architecture.platform_os) def url_version(self, version): -- cgit v1.2.3-60-g2f50 From b591d2b5015fdde5d551dd81a8906ac0b2a51260 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 18:49:52 -0800 Subject: Changed operating_system path --- lib/spack/spack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 6ad9b87ca2..48fa9edb67 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -39,7 +39,7 @@ build_env_path = join_path(lib_path, "env") module_path = join_path(lib_path, "spack") platform_path = join_path(module_path, 'platforms') compilers_path = join_path(module_path, "compilers") -operating_system_path = join_path(module_path, 'operating_system') +operating_system_path = join_path(module_path, 'operating_systems') test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") var_path = join_path(prefix, "var", "spack") -- cgit v1.2.3-60-g2f50 From 4601c36f572ed7cf66f539c11ebe0835976655dc Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 09:55:44 -0800 Subject: cleaned up concretize architecture --- lib/spack/spack/concretize.py | 93 ++++++++++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 3434ca66f7..04f932aa5d 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -218,20 +218,55 @@ class DefaultConcretizer(object): return True def _concretize_operating_system(self, arch, platform): - """ Future method for concretizing operating system """ - if isinstance(arch.platform_os, spack.architecture.OperatingSystem): - return False + if spec.architecture.platform_os is not None: + if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): + return False + else: + spec.add_operating_system_from_string(spec.architecture.platform_os) + return True #changed + if spec.root.architecture.platform_os: + if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): + spec.architecture.platform_os = spec.root.architecture.platform_os + else: + spec.add_operating_system_from_string(spec.root.architecture.platform_os) else: - arch.arch_os = platform.operating_system('default_os') - return True + platform = spack.architecture.sys_type() + spec.architecture.platform_os = platform.operating_system('default_os') + + return True #changed + +# """ Future method for concretizing operating system """ +# if isinstance(arch.platform_os, spack.architecture.OperatingSystem): +# return False +# else: +# arch.arch_os = platform.operating_system('default_os') +# return True def _concretize_target(self, arch, platform): - if isinstance(arch.target, spack.architecture.Target): - return False + if spec.target is not None: + if isinstance(spec.target,spack.architecture.Target): + return False + else: + spec.add_target_from_string(spec.target) + return True #changed + + if spec.root.target: + if isinstance(spec.root.target,spack.architecture.Target): + spec.target = spec.root.target + else: + spec.add_target_from_string(spec.root.target) else: - arch.target = platform.target('default') - return True + platform = spack.architecture.sys_type() + spec.target = platform.target('default') + + return True #changed + +# if isinstance(arch.target, spack.architecture.Target): +# return False +# else: +# arch.target = platform.target('default') +# return True def concretize_architecture(self, spec): """If the spec is empty provide the defaults of the platform. If the @@ -251,33 +286,35 @@ class DefaultConcretizer(object): Arch = spack.architecture.Arch # Set the architecture to all defaults spec.architecture = Arch(platform=platform, - platform_os=platform.operating_system('default_os'), - target=platform.target('default')) + None, + None) return True #If there is a target and it is a tuple and has both filled return #False - if not isinstance(spec.architecture, basestring): - return any(( +# if isinstance(spec.architecture, basestring): +# spec.split_architecture_string(spec.architecture) + + + ret = any(( self._concretize_platform(spec.architecture, platform), self._concretize_operating_system(spec.architecture, platform), self._concretize_target(spec.architecture, platform))) - else: - spec.add_architecture_from_string(spec.architecture) - # Does not look pretty at all!!! - if spec.root.architecture and \ - not isinstance(spec.root.architecture, basestring): - bool_flag = any(( - self._concretize_platform(spec.root.architecture, platform), - self._concretize_operating_system(spec.root.architecture, - platform), - self._concretize_target(spec.root.target, platform))) - spec.architecture =spec.root.architecture - return bool_flag - else: - spec.add_architecture_from_string(spec.root.architecture) - return True + # Does not look pretty at all!!! +# if spec.root.architecture and \ +# not isinstance(spec.root.architecture, basestring): +# bool_flag = any(( +# self._concretize_platform(spec.root.architecture, platform), +# self._concretize_operating_system(spec.root.architecture, +# platform), +# self._concretize_target(spec.root.target, platform))) +# spec.architecture =spec.root.architecture +# return bool_flag +# else: +# spec.add_architecture_from_string(spec.root.architecture) + + return ret # if there is no target specified used the defaults -- cgit v1.2.3-60-g2f50 From 615ea969f8a998417dfbbd07935848eb37f71c47 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 10:14:50 -0800 Subject: made arch instantiate as a tuple --- lib/spack/spack/spec.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 0a55ec5d76..57dd0c2181 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -461,7 +461,13 @@ class Spec(object): """Called by the parser to set the architecture.""" if self.architecture: raise DuplicateArchitectureError( "Spec for '%s' cannot have two architectures." % self.name) - self.architecture = architecture # a string can be set + platform = spack.architecture.sys_type() + if '-' in architecture: + os, target = architecture.split('-') + else: + os = architecture + target = None + self.architecture = spack.architecture.Arch(platform, os, target) def _add_dependency(self, spec): -- cgit v1.2.3-60-g2f50 From 527bb7abfeeb982d0bf22dadf4b8400502b606dd Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 10:26:09 -0800 Subject: Changed _cmp_key --- lib/spack/spack/architecture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 14a8c56793..dac3a45b79 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -207,7 +207,7 @@ class Arch(namedtuple("Arch", "platform platform_os target")): self.platform_os.name + "-" + self.target.name) def _cmp_key(self): - return (self.platform.name, self.platform_os.name, self.target.name) + return (self.platform, self.platform_os, self.target) def _helper_to_dict(arch_field_dict, arch_field_name, *args): -- cgit v1.2.3-60-g2f50 From 4d7478420968182ab9535de9deffbdd671937972 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 10:27:00 -0800 Subject: Changed add_architecture_from_string and split the work done into two methods add_target_from_string and add_operating_system_from_string --- lib/spack/spack/spec.py | 138 ++++++++++++++++++++++++++---------------------- 1 file changed, 75 insertions(+), 63 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 57dd0c2181..06accc514b 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1256,70 +1256,82 @@ class Spec(object): if os_string in platform.operating_sys: return True return False + + def add_target_from_string(self, arch): + if arch.target is None: + return arch.platform.target('default_target') + else: + return arch.platform.target(arch.target) - def add_architecture_from_string(self, arch): - """ The user is able to provide a architecture string of the form - platform-os-target. This string will be parsed by this function - and turn the architecture string into an architecture tuple of - platform, operating system and target processor classes. - The platform-os-target triplet can be delimited by a '-'. If any - portion of the architecture triplet is empty, spack will supply - the default. If the entire architecture field is blank then - defaults will be provided based off of the platform. - This happens in the concretize_architecture method in concretize.py - - e.g - =linux-ubuntu10-x84_64 -> (linux, ubuntu10, x86_64) - - =cray_xc-SuSE11-haswell -> (cray_xc, SuSE11, haswell) - - =bgq -> (bgq, - default_os, - default_target) - - =elcapitan -> (darwin, elcapitan, x86_64) - - =x86_64 -> (autodetected platform, - default_os, - x86_64) - """ - if arch is None: return - - platform_list = spack.architecture.all_platforms() - platform_names = {plat.__name__.lower():plat for plat in platform_list} - Arch = spack.architecture.Arch - arch_list = arch.split("-") - - # Create instance of current platform, gets overwritten if user - # provided a platform spec. - platform = spack.architecture.sys_type() - target = None - platform_os = None - - for entry in arch_list: - if self._is_valid_platform(entry, platform_names): - # If entry is different from platform name then create it. - # else just keep the already instantiated platform class - if entry != platform.name: - platform = platform_dict[entry]() # Create instance of platform - elif self._is_valid_target(entry, platform): - target = platform.target(entry) - # check if os is valid by checking platform operating sys dict - elif self._is_valid_os(entry, platform): - platform_os = platform.operating_system(entry) - else: - # throw error since entry is unknown - raise UnknownArchitectureSpecError(entry) - - if target is None: - target = platform.target('default') - if platform_os is None: - platform_os = platform.operating_system('default_os') - - self.architecture = Arch(platform=platform, - platform_os=platform_os, - target=target) - + def add_operating_system_from_string(self, arch): + if arch.platform_os is None: + return arch.platform.operating_system('default_os') + else: + return arch.platform.operating_system(arch.platform_os) + + #def add_architecture_from_string(self, arch): + # """ The user is able to provide a architecture string of the form + # platform-os-target. This string will be parsed by this function + # and turn the architecture string into an architecture tuple of + # platform, operating system and target processor classes. + # The platform-os-target triplet can be delimited by a '-'. If any + # portion of the architecture triplet is empty, spack will supply + # the default. If the entire architecture field is blank then + # defaults will be provided based off of the platform. + # This happens in the concretize_architecture method in concretize.py + + # e.g + # =linux-ubuntu10-x84_64 -> (linux, ubuntu10, x86_64) + + # =cray_xc-SuSE11-haswell -> (cray_xc, SuSE11, haswell) + + # =bgq -> (bgq, + # default_os, + # default_target) + + # =elcapitan -> (darwin, elcapitan, x86_64) + + # =x86_64 -> (autodetected platform, + # default_os, + # x86_64) + # """ + # if arch is None: return + # + # platform_list = spack.architecture.all_platforms() + # platform_names = {plat.__name__.lower():plat for plat in platform_list} + # Arch = spack.architecture.Arch + # arch_list = arch.split("-") + # + # # Create instance of current platform, gets overwritten if user + # # provided a platform spec. + # platform = spack.architecture.sys_type() + # target = None + # platform_os = None + + # for entry in arch_list: + # if self._is_valid_platform(entry, platform_names): + # # If entry is different from platform name then create it. + # # else just keep the already instantiated platform class + # if entry != platform.name: + # platform = platform_dict[entry]() # Create instance of platform + # elif self._is_valid_target(entry, platform): + # target = platform.target(entry) + # # check if os is valid by checking platform operating sys dict + # elif self._is_valid_os(entry, platform): + # platform_os = platform.operating_system(entry) + # else: + # # throw error since entry is unknown + # raise UnknownArchitectureSpecError(entry) + + # if target is None: + # target = platform.target('default') + # if platform_os is None: + # platform_os = platform.operating_system('default_os') + + # self.architecture = Arch(platform=platform, + # platform_os=platform_os, + # target=target) + # def satisfies(self, other, deps=True, strict=False): """determine if this spec satisfies all constraints of another. -- cgit v1.2.3-60-g2f50 From f27f2f8e491c7e3a433e070497d074f8376233a0 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 10:27:12 -0800 Subject: Fixed some indentation errors --- lib/spack/spack/concretize.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 04f932aa5d..60efba59c2 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -225,7 +225,7 @@ class DefaultConcretizer(object): spec.add_operating_system_from_string(spec.architecture.platform_os) return True #changed if spec.root.architecture.platform_os: - if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): + if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): spec.architecture.platform_os = spec.root.architecture.platform_os else: spec.add_operating_system_from_string(spec.root.architecture.platform_os) @@ -281,13 +281,13 @@ class DefaultConcretizer(object): """ platform = spack.architecture.sys_type() + if spec.architecture is None: # Create an empty tuple Arch = spack.architecture.Arch # Set the architecture to all defaults - spec.architecture = Arch(platform=platform, - None, - None) + spec.architecture = Arch(platform=platform, platform_os=None, + target=None) return True #If there is a target and it is a tuple and has both filled return #False -- cgit v1.2.3-60-g2f50 From 45887dec8e703232a4d38bd241a20e36606a00cf Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 10:56:46 -0800 Subject: partial work on bug hunting --- lib/spack/spack/architecture.py | 22 +++++++++++----------- lib/spack/spack/concretize.py | 32 +++++++++++++++----------------- lib/spack/spack/spec.py | 21 +++++++++++---------- 3 files changed, 37 insertions(+), 38 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index dac3a45b79..45a9dac8ba 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -203,8 +203,8 @@ class Arch(namedtuple("Arch", "platform platform_os target")): __slots__ = () def __str__(self): - return (self.platform.name +"-"+ - self.platform_os.name + "-" + self.target.name) + return (str(self.platform) +"-"+ + str(self.platform_os) + "-" + str(self.target) ) def _cmp_key(self): return (self.platform, self.platform_os, self.target) @@ -246,14 +246,14 @@ def to_dict(arch): return d -def _platform_from_dict(platform): - """Creates all the platform class module names into a dictionary of - name : key-value pairs. From there we can construct the - platform subclass - """ - platform_list = all_platforms() - platform_names = {plat.__name__.lower():plat for plat in platform_list} - return platform_names[platform['name']]() +#def _platform_from_dict(platform): +# """Creates all the platform class module names into a dictionary of +# name : key-value pairs. From there we can construct the +# platform subclass +# """ +# platform_list = all_platforms() +# platform_names = {plat.__name__.lower():plat for plat in platform_list} +# return platform_names[platform['name']]() def _target_from_dict(target_dict): @@ -288,7 +288,7 @@ def arch_from_dict(d): platform_os_dict = d['platform_os'] target_dict = d['target'] - platform = _platform_from_dict(platform_dict) + platform = d['platform'] platform_os = _operating_system_from_dict(platform_os_dict, platform) target = _target_from_dict(target_dict) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 60efba59c2..afb12838e8 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -217,14 +217,14 @@ class DefaultConcretizer(object): arch.platform = platform return True - def _concretize_operating_system(self, arch, platform): + def _concretize_operating_system(self, spec, platform): if spec.architecture.platform_os is not None: if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): return False else: spec.add_operating_system_from_string(spec.architecture.platform_os) return True #changed - if spec.root.architecture.platform_os: + if spec.root.architecture and spec.root.architecture.platform_os: if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): spec.architecture.platform_os = spec.root.architecture.platform_os else: @@ -243,22 +243,22 @@ class DefaultConcretizer(object): # return True - def _concretize_target(self, arch, platform): - if spec.target is not None: - if isinstance(spec.target,spack.architecture.Target): + def _concretize_target(self, spec, platform): + if spec.architecture.target is not None: + if isinstance(spec.architecture.target,spack.architecture.Target): return False else: - spec.add_target_from_string(spec.target) + spec.add_target_from_string(spec.architecture.target) return True #changed - if spec.root.target: - if isinstance(spec.root.target,spack.architecture.Target): - spec.target = spec.root.target + if spec.root.architecture and spec.root.architecture.target: + if isinstance(spec.root.architecture.target,spack.architecture.Target): + spec.architecture.target = spec.root.architecture.target else: - spec.add_target_from_string(spec.root.target) + spec.add_target_from_string(spec.root.architecture.target) else: platform = spack.architecture.sys_type() - spec.target = platform.target('default') + spec.architecture.target = platform.target('default') return True #changed @@ -283,22 +283,20 @@ class DefaultConcretizer(object): platform = spack.architecture.sys_type() if spec.architecture is None: - # Create an empty tuple - Arch = spack.architecture.Arch # Set the architecture to all defaults - spec.architecture = Arch(platform=platform, platform_os=None, + spec.architecture = spack.architecture.Arch(platform=platform, platform_os=None, target=None) return True #If there is a target and it is a tuple and has both filled return #False # if isinstance(spec.architecture, basestring): # spec.split_architecture_string(spec.architecture) - + print spec.architecture ret = any(( self._concretize_platform(spec.architecture, platform), - self._concretize_operating_system(spec.architecture, platform), - self._concretize_target(spec.architecture, platform))) + self._concretize_operating_system(spec, platform), + self._concretize_target(spec, platform))) # Does not look pretty at all!!! diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 06accc514b..5cb94907d3 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -462,11 +462,12 @@ class Spec(object): if self.architecture: raise DuplicateArchitectureError( "Spec for '%s' cannot have two architectures." % self.name) platform = spack.architecture.sys_type() + print architecture if '-' in architecture: os, target = architecture.split('-') else: - os = architecture - target = None + os = None + target = architecture self.architecture = spack.architecture.Arch(platform, os, target) @@ -1257,17 +1258,17 @@ class Spec(object): return True return False - def add_target_from_string(self, arch): - if arch.target is None: - return arch.platform.target('default_target') + def add_target_from_string(self, target): + if target is None: + self.architecture.target = self.architecture.platform.target('default_target') else: - return arch.platform.target(arch.target) + self.architecture.target = self.architecture.platform.target(target) - def add_operating_system_from_string(self, arch): - if arch.platform_os is None: - return arch.platform.operating_system('default_os') + def add_operating_system_from_string(self, os): + if os is None: + self.architecture.platform_os = self.architecture.platform.operating_system('default_os') else: - return arch.platform.operating_system(arch.platform_os) + self.architecture.platform_os = self.architecture.platform.operating_system(os) #def add_architecture_from_string(self, arch): # """ The user is able to provide a architecture string of the form -- cgit v1.2.3-60-g2f50 From 81e236b2de157405a2bc65e71a9a7fc7d1a13d7d Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 11:04:55 -0800 Subject: Changed Arch to class instead of namedtuple. Now platform is automatically set upon creation of class --- lib/spack/spack/architecture.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 45a9dac8ba..94bd59a9d5 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -196,11 +196,13 @@ class OperatingSystem(object): #NOTE: Key error caused because Architecture has no comparison method @key_ordering -class Arch(namedtuple("Arch", "platform platform_os target")): - """ namedtuple for Architecture. Will have it's own __str__ method - to make printing out the tuple easier and also won't make directory - paths look odd """ - __slots__ = () +class Arch(object): + "Architecture is now a class to help with setting attributes" + + def __init__(self, platform_os=None, target=None): + self.platform = sys_type() + self.platform_os = platform_os + self.target = target def __str__(self): return (str(self.platform) +"-"+ -- cgit v1.2.3-60-g2f50 From 86e90bba8741eec38eae3898576da272ed41d9c2 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 11:05:05 -0800 Subject: another partial, narrowing in on proper arch --- lib/spack/spack/concretize.py | 14 +------- lib/spack/spack/spec.py | 76 ++++--------------------------------------- 2 files changed, 7 insertions(+), 83 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index afb12838e8..add3b8435d 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -210,13 +210,6 @@ class DefaultConcretizer(object): return True # Things changed - def _concretize_platform(self, arch, platform): - if issubclass(arch.platform.__class__, spack.architecture.Platform): - return False - else: - arch.platform = platform - return True - def _concretize_operating_system(self, spec, platform): if spec.architecture.platform_os is not None: if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): @@ -279,13 +272,9 @@ class DefaultConcretizer(object): DAG has an architecture, then use the root otherwise use the defaults on the platform. """ - - platform = spack.architecture.sys_type() - if spec.architecture is None: # Set the architecture to all defaults - spec.architecture = spack.architecture.Arch(platform=platform, platform_os=None, - target=None) + spec.architecture = spack.architecture.Arch() return True #If there is a target and it is a tuple and has both filled return #False @@ -294,7 +283,6 @@ class DefaultConcretizer(object): print spec.architecture ret = any(( - self._concretize_platform(spec.architecture, platform), self._concretize_operating_system(spec, platform), self._concretize_target(spec, platform))) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 5cb94907d3..169c06242e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -468,7 +468,7 @@ class Spec(object): else: os = None target = architecture - self.architecture = spack.architecture.Arch(platform, os, target) + self.architecture = spack.architecture.Arch(os, target) def _add_dependency(self, spec): @@ -1250,13 +1250,10 @@ class Spec(object): return False def _is_valid_target(self, target, platform): - if target in platform.targets: - return True - return False + return target in platform.targets + def _is_valid_os(self, os_string, platform): - if os_string in platform.operating_sys: - return True - return False + return os_string in platform.operating_sys def add_target_from_string(self, target): if target is None: @@ -1270,69 +1267,7 @@ class Spec(object): else: self.architecture.platform_os = self.architecture.platform.operating_system(os) - #def add_architecture_from_string(self, arch): - # """ The user is able to provide a architecture string of the form - # platform-os-target. This string will be parsed by this function - # and turn the architecture string into an architecture tuple of - # platform, operating system and target processor classes. - # The platform-os-target triplet can be delimited by a '-'. If any - # portion of the architecture triplet is empty, spack will supply - # the default. If the entire architecture field is blank then - # defaults will be provided based off of the platform. - # This happens in the concretize_architecture method in concretize.py - - # e.g - # =linux-ubuntu10-x84_64 -> (linux, ubuntu10, x86_64) - - # =cray_xc-SuSE11-haswell -> (cray_xc, SuSE11, haswell) - - # =bgq -> (bgq, - # default_os, - # default_target) - - # =elcapitan -> (darwin, elcapitan, x86_64) - - # =x86_64 -> (autodetected platform, - # default_os, - # x86_64) - # """ - # if arch is None: return - # - # platform_list = spack.architecture.all_platforms() - # platform_names = {plat.__name__.lower():plat for plat in platform_list} - # Arch = spack.architecture.Arch - # arch_list = arch.split("-") - # - # # Create instance of current platform, gets overwritten if user - # # provided a platform spec. - # platform = spack.architecture.sys_type() - # target = None - # platform_os = None - - # for entry in arch_list: - # if self._is_valid_platform(entry, platform_names): - # # If entry is different from platform name then create it. - # # else just keep the already instantiated platform class - # if entry != platform.name: - # platform = platform_dict[entry]() # Create instance of platform - # elif self._is_valid_target(entry, platform): - # target = platform.target(entry) - # # check if os is valid by checking platform operating sys dict - # elif self._is_valid_os(entry, platform): - # platform_os = platform.operating_system(entry) - # else: - # # throw error since entry is unknown - # raise UnknownArchitectureSpecError(entry) - - # if target is None: - # target = platform.target('default') - # if platform_os is None: - # platform_os = platform.operating_system('default_os') - - # self.architecture = Arch(platform=platform, - # platform_os=platform_os, - # target=target) - # + def satisfies(self, other, deps=True, strict=False): """determine if this spec satisfies all constraints of another. @@ -1378,6 +1313,7 @@ class Spec(object): if not self.variants.satisfies(other.variants, strict=strict): return False + # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. if isinstance(self.architecture, basestring): -- cgit v1.2.3-60-g2f50 From 25f20b19673d71fb11ada83567c9e89de5555839 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 11:40:21 -0800 Subject: bug hunting --- lib/spack/spack/architecture.py | 5 ++++- lib/spack/spack/concretize.py | 15 ++++++--------- lib/spack/spack/spec.py | 1 - var/spack/mock_packages/multimethod/package.py | 7 ++++--- 4 files changed, 14 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 94bd59a9d5..c9e4eb132d 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -209,7 +209,10 @@ class Arch(object): str(self.platform_os) + "-" + str(self.target) ) def _cmp_key(self): - return (self.platform, self.platform_os, self.target) + platform = self.platform.name + os = self.platform_os.name if isinstance(self.platform_os, OperatingSystem) else self.platform_os + target = self.target.name if isinstance(self.target, Target) else self.target + return (platform, os, target) def _helper_to_dict(arch_field_dict, arch_field_name, *args): diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index add3b8435d..37204e32a4 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -210,7 +210,7 @@ class DefaultConcretizer(object): return True # Things changed - def _concretize_operating_system(self, spec, platform): + def _concretize_operating_system(self, spec): if spec.architecture.platform_os is not None: if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): return False @@ -223,8 +223,7 @@ class DefaultConcretizer(object): else: spec.add_operating_system_from_string(spec.root.architecture.platform_os) else: - platform = spack.architecture.sys_type() - spec.architecture.platform_os = platform.operating_system('default_os') + spec.architecture.platform_os = spec.architecture.platform.operating_system('default_os') return True #changed @@ -236,7 +235,7 @@ class DefaultConcretizer(object): # return True - def _concretize_target(self, spec, platform): + def _concretize_target(self, spec): if spec.architecture.target is not None: if isinstance(spec.architecture.target,spack.architecture.Target): return False @@ -250,8 +249,7 @@ class DefaultConcretizer(object): else: spec.add_target_from_string(spec.root.architecture.target) else: - platform = spack.architecture.sys_type() - spec.architecture.target = platform.target('default') + spec.architecture.target = spec.architecture.platform.target('default') return True #changed @@ -280,11 +278,10 @@ class DefaultConcretizer(object): #False # if isinstance(spec.architecture, basestring): # spec.split_architecture_string(spec.architecture) - print spec.architecture ret = any(( - self._concretize_operating_system(spec, platform), - self._concretize_target(spec, platform))) + self._concretize_operating_system(spec), + self._concretize_target(spec))) # Does not look pretty at all!!! diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 169c06242e..fdc5e3bc96 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -462,7 +462,6 @@ class Spec(object): if self.architecture: raise DuplicateArchitectureError( "Spec for '%s' cannot have two architectures." % self.name) platform = spack.architecture.sys_type() - print architecture if '-' in architecture: os, target = architecture.split('-') else: diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index b926d42d7b..03da9ef69d 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -120,9 +120,10 @@ class Multimethod(Package): for target in targets: @when('='+target.name) def different_by_target(self): - return self.spec.architecture.target.name - - + if isinstance(self.spec.architecture.target,basestring): + return self.spec.architecture.target + else: + return self.spec.architecture.target.name # # Make sure we can switch methods on different dependencies # -- cgit v1.2.3-60-g2f50 From 47c8e1366f9659850704b03fd39dbfa3dd54fbbb Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 12:08:17 -0800 Subject: Changed to_dictionary and from_dictionary methods --- lib/spack/spack/architecture.py | 114 ++++++++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 46 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index c9e4eb132d..e432374781 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -58,14 +58,11 @@ class Target(object): Targets will have compiler finding strategies """ - def __init__(self, name, compiler_strategy, module_name=None): + def __init__(self, name, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 - self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge # Sets only the platform name to avoid recursiveness - def set_platform(self, platform): - self.platform_name = platform.name def _cmp_key(self): return (self.name, self.module_name) @@ -74,10 +71,14 @@ class Target(object): return self.__str__() def __str__(self): - if self.platform_name: - return self.platform_name + '-' + self.name return self.name + def to_dict(self): + d = {} + d['name'] = self.name + d['module_name'] = self.module_name + + return d @key_ordering class Platform(object): @@ -109,7 +110,6 @@ class Platform(object): raise ValueError( "%s is a spack reserved alias " "and cannot be the name of a target" % name) - target.set_platform(self) self.targets[name] = target def target(self, name): @@ -193,6 +193,14 @@ class OperatingSystem(object): def _cmp_key(self): return (self.name, self.version, self.compiler_strategy) + + def to_dict(self): + d = {} + d['name'] = self.name + d['version'] = self.version + d['compiler_strategy'] = self.compiler_strategy + + return d #NOTE: Key error caused because Architecture has no comparison method @key_ordering @@ -213,43 +221,55 @@ class Arch(object): os = self.platform_os.name if isinstance(self.platform_os, OperatingSystem) else self.platform_os target = self.target.name if isinstance(self.target, Target) else self.target return (platform, os, target) - - -def _helper_to_dict(arch_field_dict, arch_field_name, *args): - """ General method to turn each class in architecture into a - dictionary. Takes as argument the class dictionary, the field name - (platform, platform_os, target) and then any attribute args - """ - d = {} - d[arch_field_name] = {} - for items in args: - d[arch_field_name][items] = arch_field_dict[items] - return d - -def to_dict(arch): - """ Convert the Arch tuple into a dictionary for yaml dumping. This - uses the _helper_to_dict method to create the dictionary from the - provided architecture field. Can assign the architecture - field name (either platform, platform_os or target) and any - attributes that make up that architecture field, - """ - d = {} - platform = arch.platform.__dict__ - platform_os = arch.platform_os.__dict__ - target = arch.target.__dict__ - - platform_dict = _helper_to_dict(platform,'platform','name') - os_dict = _helper_to_dict(platform_os, 'platform_os', 'name','version', - 'compiler_strategy') - target_dict = _helper_to_dict(target,'target', 'name', - 'module_name','platform_name') + def to_dict(self): + d = {} + platform = self.platform + platform_os = self.platform_os + target = self.target + + d['platform'] = self.platform.name + d['platform_os'] = self.platform_os.to_dict() + d['target'] = self.target.to_dict() + + return d - d.update(platform_dict) - d.update(os_dict) - d.update(target_dict) +#def _helper_to_dict(arch_field_dict, arch_field_name, *args): +# """ General method to turn each class in architecture into a +# dictionary. Takes as argument the class dictionary, the field name +# (platform, platform_os, target) and then any attribute args +# """ +# d = {} +# d[arch_field_name] = {} +# for items in args: +# d[arch_field_name][items] = arch_field_dict[items] +# return d +# - return d +#def to_dict(arch): +# """ Convert the Arch tuple into a dictionary for yaml dumping. This +# uses the _helper_to_dict method to create the dictionary from the +# provided architecture field. Can assign the architecture +# field name (either platform, platform_os or target) and any +# attributes that make up that architecture field, +# """ +# d = {} +# +# platform = arch.platform.__dict__ +# platform_os = arch.platform_os.__dict__ +# target = arch.target.__dict__ +# +# platform_dict = _helper_to_dict(platform,'platform','name') +# os_dict = _helper_to_dict(platform_os, 'platform_os', 'name','version', +# 'compiler_strategy') +# target_dict = _helper_to_dict(target,'target', 'name', +# 'module_name','platform_name') +# +# d.update(platform_dict) +# d.update(os_dict) +# d.update(target_dict) +# +# return d #def _platform_from_dict(platform): # """Creates all the platform class module names into a dictionary of @@ -287,17 +307,19 @@ def arch_from_dict(d): helper methods to recreate the arch tuple from the dictionary read from a yaml file """ + arch = Arch() + if d is None: return None - platform_dict = d['platform'] - platform_os_dict = d['platform_os'] + os_dict = d['platform_os'] target_dict = d['target'] - platform = d['platform'] - platform_os = _operating_system_from_dict(platform_os_dict, platform) target = _target_from_dict(target_dict) + platform_os = _operating_system_from_dict(os_dict, arch.platform) + arch.target =target + arch.platform_os = platform_os - return Arch(platform, platform_os, target) + return arch @memoized def all_platforms(): -- cgit v1.2.3-60-g2f50 From 90e90f61c161facfb87dd2dc6389f80d56eba2d7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 12:08:43 -0800 Subject: Got rid of the compiler strategy from target --- lib/spack/spack/platforms/bgq.py | 4 ++-- lib/spack/spack/platforms/cray_xc.py | 6 +++--- lib/spack/spack/platforms/darwin.py | 2 +- lib/spack/spack/platforms/linux.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/platforms/bgq.py b/lib/spack/spack/platforms/bgq.py index b550f1d5cb..e0eb76f336 100644 --- a/lib/spack/spack/platforms/bgq.py +++ b/lib/spack/spack/platforms/bgq.py @@ -9,8 +9,8 @@ class Bgq(Platform): def __init__(self): super(Bgq, self).__init__('bgq') - self.add_target(self.front_end, Target(self.front_end, 'PATH')) - self.add_target(self.back_end, Target(self.back_end, 'PATH')) + self.add_target(self.front_end, Target(self.front_end)) + self.add_target(self.back_end, Target(self.back_end,)) @classmethod def detect(self): diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index b71481e46e..4843a47c62 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -31,11 +31,11 @@ class CrayXc(Platform): # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target('sandybridge', Target('sandybridge', 'PATH')) + self.add_target('sandybridge', Target('sandybridge')) self.add_target('ivybridge', - Target('ivybridge', 'MODULES', 'craype-ivybridge')) + Target('ivybridge', 'craype-ivybridge')) self.add_target('haswell', - Target('haswell', 'MODULES', 'craype-haswell')) + Target('haswell','craype-haswell')) self.add_operating_system('SuSE', LinuxDistro()) self.add_operating_system('CNL', Cnl()) diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index c48f7d17d2..4c3d38851f 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -10,7 +10,7 @@ class Darwin(Platform): def __init__(self): super(Darwin, self).__init__('darwin') - self.add_target(self.default, Target(self.default, 'PATH')) + self.add_target(self.default, Target(self.default)) mac_os = MacOsx() self.default_os = mac_os.name self.add_operating_system(mac_os.name, mac_os) diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index d9e33511a1..3243a1dcdf 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -10,7 +10,7 @@ class Linux(Platform): def __init__(self): super(Linux, self).__init__('linux') - self.add_target(self.default, Target(self.default, 'PATH')) + self.add_target(self.default, Target(self.default)) linux_dist = LinuxDistro() self.default_os = linux_dist.name self.add_operating_system(linux_dist.name, linux_dist) -- cgit v1.2.3-60-g2f50 From e46bac19fa9baafe1e4b32ff940a883218f8b019 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 12:09:02 -0800 Subject: Changed architecture to dict --- lib/spack/spack/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index fdc5e3bc96..6cca61ca4e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -671,7 +671,7 @@ class Spec(object): if self.architecture: # TODO: Fix the target.to_dict to account for the tuple # Want it to be a dict of dicts - d['architecture'] = spack.architecture.to_dict(self.architecture) + d['architecture'] = self.architecture.to_dict() else: d['architecture'] = None -- cgit v1.2.3-60-g2f50 From 975cba295bc3141a9888292623702f1ee45f3d3a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 14:16:57 -0800 Subject: Added 'better' tests --- lib/spack/spack/test/architecture.py | 60 +++++++++++++++--------------------- 1 file changed, 24 insertions(+), 36 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 5bc78d1b65..a390a3fbaa 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -14,43 +14,31 @@ from spack.platforms.darwin import Darwin class ArchitectureTest(unittest.TestCase): - def setUp(self): - zlib = spack.spec.Spec("zlib") - zlib.concretize() - self.architecture = zlib.architecture - self.platform = sys_type() - self.platform_os = self.platform.operating_system('default_os') - self.target = self.platform.target('default') - - #def test_to_dict_function_with_target(self): - # d = spack.architecture.to_dict(self.architecture) - # print d['target'] - # self.assertEquals(d['target'], {'name': self.target.name, - # 'module_name' : self.target.module_name, - # 'platform_name' : self.target.platform_name, - # 'compiler_strategy': 'MODULES' - # }) - def test_to_dict_function_with_architecture(self): - d = spack.architecture.to_dict(self.architecture) - self.assertEquals(d, {'architecture': - {'platform' : {'name': 'crayxc'}, - 'platform_os': { - 'compiler_strategy': 'MODULES', - 'name':'CNL', - 'version':'10'}, - 'target' : {'platform_name' :'crayxc', - 'module_name': 'craype-haswell', - 'name':'haswell'}}}) - - #def test_to_dict_function_with_operating_system(self): - # d = spack.architecture.to_dict(self.architecture) - # self.assertEquals(d['platform_os'], {'name': self.platform_os.name, - # 'version': self.platform_os.version, - # 'compiler_strategy': self.platform_os.compiler_strategy}) - - def test_architecture_from_dict(self): - pass + arch = Arch() + arch.platform_os = arch.platform.operating_system('default_os') + arch.target = arch.platform.target('default') + + d = arch.to_dict() + self.assertEqual(d, {'platform' : 'crayxc', + 'platform_os' : {'name': 'CNL', + 'compiler_strategy' : 'MODULES', + 'version':'10'}, + 'target' : {'name': 'haswell', + 'module_name': 'craype-haswell'}}) + + def test_from_dict_function_with_architecture(self): + d = {'platform':'crayxc', + 'platform_os' : {'name' : 'CNL', 'compiler_strategy': 'MODULES', + 'version': '10'}, + 'target' : {'name':'haswell', 'module_name': 'craype-haswell'}} + + arch = spack.architecture.arch_from_dict(d) + self.assertIsInstance(arch, Arch) + self.assertIsInstance(arch.platform, Platform) + self.assertIsInstance(arch.platform_os, OperatingSystem) + self.assertIsInstance(arch.target, Target) + def test_platform_class_and_compiler_strategies(self): a = CrayXc() -- cgit v1.2.3-60-g2f50 From 676591ffc0d62a4d3b20653a2faa788c06d96d77 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 14:17:46 -0800 Subject: fixed a bug and a test bug --- lib/spack/spack/spec.py | 2 +- .../mock_configs/site_spackconfig/compilers.yaml | 23 +++++++++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 6cca61ca4e..6d2138d4c8 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1307,7 +1307,7 @@ class Spec(object): if not self.compiler.satisfies(other.compiler, strict=strict): return False elif strict and (other.compiler and not self.compiler): - return True + return False if not self.variants.satisfies(other.variants, strict=strict): return False diff --git a/var/spack/mock_configs/site_spackconfig/compilers.yaml b/var/spack/mock_configs/site_spackconfig/compilers.yaml index fcbf7a53f1..72fa252881 100644 --- a/var/spack/mock_configs/site_spackconfig/compilers.yaml +++ b/var/spack/mock_configs/site_spackconfig/compilers.yaml @@ -13,10 +13,23 @@ compilers: fc: /path/to/gfortran modules: None gcc@5.2.0: - cc: cc - cxx: CC - f77: ftn - fc: ftn - modules: + cc: cc + cxx: CC + f77: ftn + fc: ftn + modules: - PrgEnv-gnu - gcc/5.2.0 + intel@15.0.1: + cc: cc + ccx: CC + f77: ftn + fc: ftn + modules: + - PrgEnv-intel + - intel/15.0.1 + intel@15.1.2: + cc: /path/to/icc + cxx: /path/to/ic++ + f77: /path/to/ifort + fc: /path/to/ifort -- cgit v1.2.3-60-g2f50 From 6e7b00a0f6229a2c1df7a5b2f7244bdf5a793d65 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 15:54:23 -0800 Subject: Made module handling much saner and safer --- lib/spack/spack/build_environment.py | 2 +- lib/spack/spack/compiler.py | 26 ++++++++++++---------- lib/spack/spack/compilers/__init__.py | 21 +++++++++++------ lib/spack/spack/concretize.py | 9 ++++---- lib/spack/spack/test/architecture.py | 8 +++---- .../mock_configs/site_spackconfig/compilers.yaml | 5 +++++ 6 files changed, 43 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 6e03394777..0cb4d77499 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -171,7 +171,7 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) - if compiler.modules: + if compiler.strategy == 'MODULES': for mod in compiler.modules: load_module(mod) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 46f2dfaec7..5d851f5a3d 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -106,26 +106,28 @@ class Compiler(object): PrgEnv_compiler = None - def __init__(self, cspec, paths, modules=None): + def __init__(self, cspec, strategy, paths, modules=None): def check(exe): if exe is None: return None _verify_executables(exe) return exe + self.strategy = strategy + self.cc = check(paths[0]) self.cxx = check(paths[1]) - self.f77 = check(paths[2]) - if len(paths) == 3: - self.fc = self.f77 - else: - self.fc = check(paths[3]) + if len(paths) > 2: + self.f77 = check(paths[2]) + if len(paths) == 3: + self.fc = self.f77 + else: + self.fc = check(paths[3]) self.spec = cspec self.modules = modules - @property def version(self): return self.spec.version @@ -267,7 +269,7 @@ class Compiler(object): if newcount <= prevcount: continue - compilers[ver] = cls(spec, paths) + compilers[ver] = cls(spec, 'PATH', paths) return list(compilers.values()) @@ -287,7 +289,7 @@ class Compiler(object): for name, version in matches: v = version - comp = cls(spack.spec.CompilerSpec(name + '@' + v), + comp = cls(spack.spec.CompilerSpec(name + '@' + v), 'MODULES', ['cc', 'CC', 'ftn'], [cls.PrgEnv, name +'/' + v]) compilers.append(comp) @@ -302,12 +304,12 @@ class Compiler(object): def __str__(self): """Return a string represntation of the compiler toolchain.""" - if self.modules: + if self.strategy is 'MODULES': return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc, self.modules)))) + self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc, self.modules)))) else: return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) + self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc)))) class CompilerAccessError(spack.error.SpackError): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index e38ef949d7..32885062fd 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -129,6 +129,11 @@ def add_compilers_to_config(scope, *compilers): for compiler in compilers: compiler_entry = {} + val = getattr(compiler, 'strategy') + if not val: + val = 'None' + compiler_entry[c] = val + for c in _required_instance_vars: val = getattr(compiler, c) if not val: @@ -190,6 +195,11 @@ def compilers_for_spec(compiler_spec): raise InvalidCompilerConfigurationError(cspec) cls = class_for_compiler_name(cspec.name) + + strategy = items['strategy'] + if not strategy: + raise InvalidCompilerConfigurationError(cspec) + compiler_paths = [] for c in _required_instance_vars: compiler_path = items[c] @@ -203,22 +213,19 @@ def compilers_for_spec(compiler_spec): items[m] = None mods = items[m] - return cls(cspec, compiler_paths, mods) + return cls(cspec, strategy, compiler_paths, mods) matches = find(compiler_spec) return [get_compiler(cspec) for cspec in matches] @_auto_compiler_spec -def compiler_for_spec(compiler_spec, target): +def compiler_for_spec(compiler_spec, operating_system): """Get the compiler that satisfies compiler_spec. compiler_spec must be concrete.""" assert(compiler_spec.concrete) - compilers = compilers_for_spec(compiler_spec) - if target.compiler_strategy == "PATH": - compilers = [c for c in compilers if c.modules is None] - elif target.compiler_strategy == "MODULES": - compilers = [c for c in compilers if c.modules is not None] + compilers = [c for c in compilers_for_spec(compiler_spec) + if c.strategy == operating_system.strategy] if len(compilers) < 1: raise NoCompilerForSpecError(compiler_spec, target) if len(compilers) > 1: diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 37204e32a4..7a87a824d5 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -354,10 +354,11 @@ class DefaultConcretizer(object): # Should think whether this can be more efficient def _proper_compiler_style(cspec, architecture): compilers = spack.compilers.compilers_for_spec(cspec) - if architecture.platform_os.compiler_strategy == 'PATH': - filter(lambda c: not c.modules, compilers) - if architecture.platform_os.compiler_strategy == 'MODULES': - filter(lambda c: c.modules, compilers) + filter(lambda c: c.strategy == architecture.platform_os.compiler_strategy, compilers) +#if architecture.platform_os.compiler_strategy == 'PATH': + # filter(lambda c: not c.modules, compilers) + #if architecture.platform_os.compiler_strategy == 'MODULES': + # filter(lambda c: c.modules, compilers) return compilers diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index a390a3fbaa..75a67bf02f 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -34,10 +34,10 @@ class ArchitectureTest(unittest.TestCase): 'target' : {'name':'haswell', 'module_name': 'craype-haswell'}} arch = spack.architecture.arch_from_dict(d) - self.assertIsInstance(arch, Arch) - self.assertIsInstance(arch.platform, Platform) - self.assertIsInstance(arch.platform_os, OperatingSystem) - self.assertIsInstance(arch.target, Target) + self.assertTrue( isinstance(arch, Arch) ) + self.assertTrue( isinstance(arch.platform, Platform) ) + self.assertTrue( isinstance(arch.platform_os, OperatingSystem) ) + self.assertTrue( isinstance(arch.target, Target) ) def test_platform_class_and_compiler_strategies(self): diff --git a/var/spack/mock_configs/site_spackconfig/compilers.yaml b/var/spack/mock_configs/site_spackconfig/compilers.yaml index 72fa252881..5f8b38007b 100644 --- a/var/spack/mock_configs/site_spackconfig/compilers.yaml +++ b/var/spack/mock_configs/site_spackconfig/compilers.yaml @@ -6,12 +6,14 @@ compilers: f77: None fc: None modules: None + strategy: PATH gcc@4.5.0: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran modules: None + strategy: PATH gcc@5.2.0: cc: cc cxx: CC @@ -20,6 +22,7 @@ compilers: modules: - PrgEnv-gnu - gcc/5.2.0 + strategy: MODULES intel@15.0.1: cc: cc ccx: CC @@ -28,8 +31,10 @@ compilers: modules: - PrgEnv-intel - intel/15.0.1 + strategy: MODULES intel@15.1.2: cc: /path/to/icc cxx: /path/to/ic++ f77: /path/to/ifort fc: /path/to/ifort + strategy: PATH \ No newline at end of file -- cgit v1.2.3-60-g2f50 From ffb9574312e8b6368c7696c2eadd17e46ec9a400 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 15:59:23 -0800 Subject: fixed type/bug --- lib/spack/spack/compilers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 32885062fd..b50ad02b07 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -225,7 +225,7 @@ def compiler_for_spec(compiler_spec, operating_system): be concrete.""" assert(compiler_spec.concrete) compilers = [c for c in compilers_for_spec(compiler_spec) - if c.strategy == operating_system.strategy] + if c.strategy == operating_system.compiler_strategy] if len(compilers) < 1: raise NoCompilerForSpecError(compiler_spec, target) if len(compilers) > 1: -- cgit v1.2.3-60-g2f50 From 65d4169f003ce73c59f12e426e1e7fa4bf2062f3 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 16:03:11 -0800 Subject: fixed type/bug --- lib/spack/spack/compilers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index b50ad02b07..518eea55c4 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -132,7 +132,7 @@ def add_compilers_to_config(scope, *compilers): val = getattr(compiler, 'strategy') if not val: val = 'None' - compiler_entry[c] = val + compiler_entry['strategy'] = val for c in _required_instance_vars: val = getattr(compiler, c) -- cgit v1.2.3-60-g2f50 From 16d8c25b231e795f8c4f3d11f937a4658596b238 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 16:12:32 -0800 Subject: fixed floating 'target' --- lib/spack/spack/compilers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 518eea55c4..d271b60261 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -227,7 +227,7 @@ def compiler_for_spec(compiler_spec, operating_system): compilers = [c for c in compilers_for_spec(compiler_spec) if c.strategy == operating_system.compiler_strategy] if len(compilers) < 1: - raise NoCompilerForSpecError(compiler_spec, target) + raise NoCompilerForSpecError(compiler_spec, operating_system) if len(compilers) > 1: raise CompilerSpecInsufficientlySpecificError(compiler_spec) return compilers[0] -- cgit v1.2.3-60-g2f50 From 58efe1550d913e0bf31f2d5b6e279066cf5363d5 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 16:14:42 -0800 Subject: Default PATH arg --- lib/spack/spack/architecture.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e432374781..85ced6e942 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -170,7 +170,7 @@ class OperatingSystem(object): find compilers we call find_compilers method for each operating system """ - def __init__(self, name, version, compiler_strategy): + def __init__(self, name, version, compiler_strategy="PATH"): self.name = name self.version = version self.compiler_strategy = compiler_strategy @@ -180,19 +180,9 @@ class OperatingSystem(object): def __repr__(self): return self.__str__() - - - def compiler_strategy(self): - """ Operating Systems will now be in charge of the compiler finding - strategy. They will each have their own find_compilers method - which depending on their compiler strategy will find the compilers - using a specific method (i.e PATH vs MODULES). - """ - raise NotImplementedError() def _cmp_key(self): return (self.name, self.version, self.compiler_strategy) - def to_dict(self): d = {} -- cgit v1.2.3-60-g2f50 From 2b487287419528142b16d9e4201a44a034c11d20 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 16:15:34 -0800 Subject: Changed os subclasses CNL to change attribute to MODULES --- lib/spack/spack/operating_systems/cnl.py | 6 ------ lib/spack/spack/operating_systems/linux_distro.py | 11 +---------- lib/spack/spack/operating_systems/mac_osx.py | 13 +------------ 3 files changed, 2 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py index 20cb4c215b..9a0bf6c194 100644 --- a/lib/spack/spack/operating_systems/cnl.py +++ b/lib/spack/spack/operating_systems/cnl.py @@ -11,9 +11,3 @@ class Cnl(OperatingSystem): name = 'CNL' version = '10' super(Cnl, self).__init__(name, version, "MODULES") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py index c5547749d8..81c7a86430 100644 --- a/lib/spack/spack/operating_systems/linux_distro.py +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -12,13 +12,4 @@ class LinuxDistro(OperatingSystem): name = py_platform.dist()[0] version = py_platform.dist()[1] - super(LinuxDistro, self).__init__(name, version, "PATH") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass - - - + super(LinuxDistro, self).__init__(name, version) diff --git a/lib/spack/spack/operating_systems/mac_osx.py b/lib/spack/spack/operating_systems/mac_osx.py index 555c5cfb14..a9de03d2cc 100644 --- a/lib/spack/spack/operating_systems/mac_osx.py +++ b/lib/spack/spack/operating_systems/mac_osx.py @@ -26,15 +26,4 @@ class MacOsx(OperatingSystem): except KeyError: name = "mac_os" - super(MacOsx, self).__init__(name, mac_ver, "PATH") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass - - - - - + super(MacOsx, self).__init__(name, mac_ver) -- cgit v1.2.3-60-g2f50 From 487b2495a1dbda730a6525f17f175f3eff1674c0 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 16:33:10 -0800 Subject: made spack more conservative in compiler strategies attempted --- lib/spack/spack/compiler.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 5d851f5a3d..999984740f 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -34,6 +34,7 @@ from llnl.util.filesystem import join_path import spack.error import spack.spec +import spack.architecture from spack.util.multiproc import parmap from spack.util.executable import * from spack.util.environment import get_path @@ -221,7 +222,15 @@ class Compiler(object): @classmethod def find(cls, *path): - return cls.find_in_path(*path) + cls.find_in_modules() + compilers = [] + platform = spack.architecture.sys_type() + strategies = [o.compiler_strategy for o in platform.operating_systems.values()] + if 'PATH' in strategies: + compilers.extend(cls.find_in_path(*path)) + if 'MODULES' in strategies: + compilers.extend(cls.find_in_modules()) + return compilers + @classmethod def find_in_path(cls, *path): -- cgit v1.2.3-60-g2f50 From 4f9a309de850f339ed43cd788be45d208bebf6c6 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 16:35:02 -0800 Subject: made spack more conservative in compiler strategies attempted --- lib/spack/spack/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 999984740f..c8a025f83c 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -224,7 +224,7 @@ class Compiler(object): def find(cls, *path): compilers = [] platform = spack.architecture.sys_type() - strategies = [o.compiler_strategy for o in platform.operating_systems.values()] + strategies = [o.compiler_strategy for o in platform.operating_sys.values()] if 'PATH' in strategies: compilers.extend(cls.find_in_path(*path)) if 'MODULES' in strategies: -- cgit v1.2.3-60-g2f50 From 688eca23ff4488a4cd90125b69d46cb69271a5cd Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 23 Mar 2016 09:54:57 -0700 Subject: cleanup --- lib/spack/spack/spec.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 4d2c61c0c4..d800a1667d 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -110,7 +110,7 @@ import spack import spack.architecture import spack.parse import spack.error -import spack.compilers as compilers +#import spack.compilers as compilers from spack.version import * from spack.util.naming import mod_to_class @@ -1174,9 +1174,9 @@ class Spec(object): spack.repo.get(spec.fullname) # validate compiler in addition to the package name. - if spec.compiler: - if not compilers.supported(spec.compiler): - raise UnsupportedCompilerError(spec.compiler.name) +# if spec.compiler: +# if not compilers.supported(spec.compiler): +# raise UnsupportedCompilerError(spec.compiler.name) # Ensure that variants all exist. for vname, variant in spec.variants.items(): -- cgit v1.2.3-60-g2f50 From 315623d3612d64fd058324d4dbbe5264def759c0 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 24 Mar 2016 16:55:46 -0700 Subject: Fixed things from merge. --- lib/spack/spack/architecture.py | 7 +++++++ lib/spack/spack/build_environment.py | 10 +++++----- lib/spack/spack/cmd/compiler.py | 2 +- lib/spack/spack/compilers/__init__.py | 34 ++++++++++++++++++---------------- lib/spack/spack/concretize.py | 6 +++--- lib/spack/spack/config.py | 8 ++++++++ lib/spack/spack/spec.py | 1 + lib/spack/spack/test/multimethod.py | 4 ++-- 8 files changed, 45 insertions(+), 27 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index aef38b8a55..8d9686cf99 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -202,6 +202,13 @@ class Arch(object): self.platform_os = platform_os self.target = target + @property + def concrete(self): + return all( (self.platform is not None, isinstance(self.platform, Platform), + self.platform_os is not None, isinstance(self.platform_os, OperatingSystem), + self.target is not None, isinstance(self.target, Target) ) ) + + def __str__(self): return (str(self.platform) +"-"+ str(self.platform_os) + "-" + str(self.target) ) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index a64ce8aeb4..8ed3c4e1a1 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -101,12 +101,12 @@ def load_module(mod): # We do this without checking that they are already installed # for ease of programming because unloading a module that is not # loaded does nothing. - text = modulecmd('show', mod, return_oe=True).split() + text = modulecmd('show', mod, output=str, error=str).split() for i, word in enumerate(text): if word == 'conflict': - exec(compile(modulecmd('unload', text[i+1], return_oe=True), '', 'exec')) + exec(compile(modulecmd('unload', text[i+1], output=str, error=str), '', 'exec')) # Load the module now that there are no conflicts - load = modulecmd('load', mod, return_oe=True) + load = modulecmd('load', mod, output=str, error=str) exec(compile(load, '', 'exec')) @@ -119,7 +119,7 @@ def get_path_from_module(mod): modulecmd.add_default_arg('python') # Read the module - text = modulecmd('show', mod, return_oe=True).split('\n') + text = modulecmd('show', mod, output=str, error=str).split('\n') # If it lists its package directory, return that for line in text: @@ -348,8 +348,8 @@ def parent_class_modules(cls): def setup_package(pkg): """Execute all environment setup routines.""" - set_compiler_environment_variables(pkg) set_build_environment_variables(pkg) + set_compiler_environment_variables(pkg) # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index bed5db3d47..a7de61e1e5 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -import argparse +from external import argparse import llnl.util.tty as tty from llnl.util.tty.color import colorize diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 93c5172dde..55e2db700f 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -46,7 +46,7 @@ from spack.util.environment import get_path _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] -_optional_instance_vars = ['modules'] +_optional_instance_vars = ['modules', 'strategy'] _default_order = [] # TODO: customize order in config file @@ -69,7 +69,7 @@ def _to_dict(compiler): return { str(compiler.spec) : dict( (attr, getattr(compiler, attr, None)) - for attr in _required_instance_vars) + for attr in _required_instance_vars + _optional_instance_vars) } @@ -77,20 +77,24 @@ def get_compiler_config(arch=None, scope=None): """Return the compiler configuration for the specified architecture. """ # Check whether we're on a front-end (native) architecture. - my_arch = spack.architecture.sys_type() + + my_arch = spack.architecture.Arch() + if isinstance(arch, basestring): + if arch == 'all': + my_arch.platform.name = 'all' + arch = my_arch if arch is None: arch = my_arch def init_compiler_config(): """Compiler search used when Spack has no compilers.""" - config[arch] = {} + config[arch.platform.name] = {} compilers = find_compilers(*get_path('PATH')) for compiler in compilers: - config[arch].update(_to_dict(compiler)) + config[arch.platform.name].update(_to_dict(compiler)) spack.config.update_config('compilers', config, scope=scope) config = spack.config.get_config('compilers', scope=scope) - # Update the configuration if there are currently no compilers # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. @@ -105,7 +109,7 @@ def get_compiler_config(arch=None, scope=None): if not site_config: init_compiler_config() - return config[arch] if arch in config else {} + return config[arch.platform.name] if arch.platform.name in config else {} def add_compilers_to_config(compilers, arch=None, scope=None): @@ -117,15 +121,15 @@ def add_compilers_to_config(compilers, arch=None, scope=None): - scope: configuration scope to modify. """ if arch is None: - arch = spack.architecture.sys_type() + arch = spack.architecture.Arch() compiler_config = get_compiler_config(arch, scope) for compiler in compilers: compiler_config[str(compiler.spec)] = dict( (c, getattr(compiler, c, "None")) - for c in _required_instance_vars + ['strategy'] + _optional_instance_vars) + for c in _required_instance_vars + _optional_instance_vars) - update = { arch : compiler_config } + update = { arch.platform.name : compiler_config } spack.config.update_config('compilers', update, scope) @@ -139,7 +143,7 @@ def remove_compiler_from_config(compiler_spec, arch=None, scope=None): - scope: configuration scope to modify. """ if arch is None: - arch = spack.architecture.sys_type() + arch = spack.architecture.Arch() compiler_config = get_compiler_config(arch, scope) del compiler_config[str(compiler_spec)] @@ -154,7 +158,6 @@ def all_compilers_config(arch=None, scope=None): """ # Get compilers for this architecture. arch_config = get_compiler_config(arch, scope) - # Merge 'all' compilers with arch-specific ones. # Arch-specific compilers have higher precedence. merged_config = get_compiler_config('all', scope=scope) @@ -262,10 +265,9 @@ def compilers_for_spec(compiler_spec, arch=None, scope=None): else: compiler_paths.append(None) - for m in _optional_instance_vars: - if m not in items: - items[m] = None - mods = items[m] + if 'modules' not in items: + items['modules'] = None + mods = items['modules'] return cls(cspec, strategy, compiler_paths, mods) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 330ecdf509..c5dad93aef 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -79,7 +79,7 @@ class DefaultConcretizer(object): externals = spec_externals(pkg) buildable = not is_spec_nobuild(pkg) if buildable: - result.append((pkg, None)) + result.append((pkg, None, None)) for ext in externals: if ext[0].satisfies(spec): result.append(ext) @@ -354,7 +354,7 @@ class DefaultConcretizer(object): link to this one, to maximize compatibility. """ # Pass on concretizing the compiler if the target is not yet determined - if not spec.architecture.target: + if not spec.architecture.platform_os: #Although this usually means changed, this means awaiting other changes return True @@ -371,7 +371,7 @@ class DefaultConcretizer(object): return compilers - all_compilers = spack.compilers.all_compilers() + all_compilers = spack.compilers.all_compilers(spec.architecture) if (spec.compiler and spec.compiler.concrete and diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 182c174baa..c9a770a53f 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -168,6 +168,10 @@ section_schemas = { {'type' : 'null' }]}, 'fc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, + 'strategy': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'modules': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]} },},},},},},},}, 'mirrors': { @@ -224,6 +228,10 @@ section_schemas = { 'type': 'boolean', 'default': False, }, + 'module': { + 'anyOf' : [{'type': 'string'}, + {'type': 'null'}] + }, 'providers': { 'type': 'object', 'default': {}, diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index d800a1667d..fa2e9fd4c8 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -542,6 +542,7 @@ class Spec(object): and self.versions.concrete and self.variants.concrete and self.architecture + and self.architecture.concrete and self.compiler and self.compiler.concrete and self.dependencies.concrete) diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 020d25015d..6a723d123e 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -92,10 +92,10 @@ class MultiMethodTest(MockPackagesTest): platform = spack.architecture.sys_type() targets = platform.targets.values() for target in targets[:-1]: - pkg = spack.db.get('multimethod='+target.name) + pkg = spack.repo.get('multimethod='+target.name) self.assertEqual(pkg.different_by_target(), target.name) - pkg = spack.db.get('multimethod='+targets[-1].name) + pkg = spack.repo.get('multimethod='+targets[-1].name) if len(targets) == 1: self.assertEqual(pkg.different_by_target(), targets[-1].name) else: -- cgit v1.2.3-60-g2f50 From 618a9b2d7b1fbe297a2165181016fc45153cecf1 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 25 Mar 2016 15:41:22 -0700 Subject: Fixed bug in platform names that I introduced yesterday --- lib/spack/spack/compilers/__init__.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 55e2db700f..9601ae3e3a 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -28,6 +28,7 @@ system and configuring Spack to use multiple compilers. import imp import os import platform +import copy from llnl.util.lang import memoized, list_modules from llnl.util.filesystem import join_path @@ -77,28 +78,27 @@ def get_compiler_config(arch=None, scope=None): """Return the compiler configuration for the specified architecture. """ # Check whether we're on a front-end (native) architecture. - my_arch = spack.architecture.Arch() - if isinstance(arch, basestring): - if arch == 'all': - my_arch.platform.name = 'all' - arch = my_arch if arch is None: arch = my_arch + if isinstance(arch, basestring) and arch == 'all': + name = 'all' + else: + name = arch.platform.name def init_compiler_config(): """Compiler search used when Spack has no compilers.""" - config[arch.platform.name] = {} + config[name] = {} compilers = find_compilers(*get_path('PATH')) for compiler in compilers: - config[arch.platform.name].update(_to_dict(compiler)) + config[name].update(_to_dict(compiler)) spack.config.update_config('compilers', config, scope=scope) config = spack.config.get_config('compilers', scope=scope) # Update the configuration if there are currently no compilers # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. - if arch == my_arch and arch not in config: + if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: if scope is None: # We know no compilers were configured in any scope. init_compiler_config() @@ -109,7 +109,7 @@ def get_compiler_config(arch=None, scope=None): if not site_config: init_compiler_config() - return config[arch.platform.name] if arch.platform.name in config else {} + return config[name] if name in config else {} def add_compilers_to_config(compilers, arch=None, scope=None): -- cgit v1.2.3-60-g2f50 From 57ced372291bfa537e37d325d8e4333623eb306d Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 25 Mar 2016 17:03:49 -0700 Subject: Fixed a bug in the testing --- lib/spack/spack/test/mock_packages_test.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 079cbcc136..8f04076f19 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -42,11 +42,15 @@ compilers: cxx: /path/to/clang++ f77: None fc: None + strategy: PATH + modules: None gcc@4.5.0: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran + strategy: PATH + modules: None """ mock_packages_config = """\ -- cgit v1.2.3-60-g2f50 From 7e9baf9e2533eb040aa142bd3bf6ffbd142516ac Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 29 Mar 2016 11:23:36 -0700 Subject: Made architecture test robust for more architectures --- lib/spack/spack/test/architecture.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 75a67bf02f..2274a9901d 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -13,32 +13,26 @@ from spack.platforms.bgq import Bgq from spack.platforms.darwin import Darwin class ArchitectureTest(unittest.TestCase): - - def test_to_dict_function_with_architecture(self): + + def test_dict_functions_for_architecture(self): arch = Arch() arch.platform_os = arch.platform.operating_system('default_os') arch.target = arch.platform.target('default') d = arch.to_dict() - self.assertEqual(d, {'platform' : 'crayxc', - 'platform_os' : {'name': 'CNL', - 'compiler_strategy' : 'MODULES', - 'version':'10'}, - 'target' : {'name': 'haswell', - 'module_name': 'craype-haswell'}}) - - def test_from_dict_function_with_architecture(self): - d = {'platform':'crayxc', - 'platform_os' : {'name' : 'CNL', 'compiler_strategy': 'MODULES', - 'version': '10'}, - 'target' : {'name':'haswell', 'module_name': 'craype-haswell'}} - - arch = spack.architecture.arch_from_dict(d) + + new_arch = spack.architecture.arch_from_dict(d) + self.assertEqual(arch, new_arch) + self.assertTrue( isinstance(arch, Arch) ) self.assertTrue( isinstance(arch.platform, Platform) ) self.assertTrue( isinstance(arch.platform_os, OperatingSystem) ) self.assertTrue( isinstance(arch.target, Target) ) - + self.assertTrue( isinstance(new_arch, Arch) ) + self.assertTrue( isinstance(new_arch.platform, Platform) ) + self.assertTrue( isinstance(new_arch.platform_os, OperatingSystem) ) + self.assertTrue( isinstance(new_arch.target, Target) ) + def test_platform_class_and_compiler_strategies(self): a = CrayXc() -- cgit v1.2.3-60-g2f50 From 9c071ea40d6bacef2056184f7b256c2f1fb7cbd0 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 4 Apr 2016 13:44:24 -0700 Subject: Changed modulecmd parameters. return_oe is not one of the parameters in the mainline spack repo. Changed the args to the appropriate ones in the new spack --- lib/spack/spack/compiler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index e3ea918aca..c285c8a29d 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -285,16 +285,15 @@ class Compiler(object): @classmethod def find_in_modules(cls): compilers = [] - if cls.PrgEnv: if not cls.PrgEnv_compiler: tty.die('Must supply PrgEnv_compiler with PrgEnv') modulecmd = which('modulecmd') modulecmd.add_default_arg('python') - output = modulecmd('avail', cls.PrgEnv_compiler, return_oe=True) + + output = modulecmd('avail', cls.PrgEnv_compiler, output=str, error=str) matches = re.findall(r'(%s)/([\d\.]+[\d])' % cls.PrgEnv_compiler, output) - for name, version in matches: v = version comp = cls(spack.spec.CompilerSpec(name + '@' + v), 'MODULES', -- cgit v1.2.3-60-g2f50 From 6d73c862096ecd6426d42dfe666c0908c2069a21 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 4 Apr 2016 13:45:15 -0700 Subject: Gives error on cori that [PrgEnv-xx, gcc/x.x.x] is not part of the schema. Changed it to oneOf : type : array to get rid of the error. --- lib/spack/spack/config.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index c9a770a53f..f978ed12b0 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -159,7 +159,9 @@ section_schemas = { 'type': 'object', 'additionalProperties': False, 'required': ['cc', 'cxx', 'f77', 'fc'], - 'properties': { + 'properties': { + 'strategy': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, 'cc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, 'cxx': { 'anyOf': [ {'type' : 'string' }, @@ -168,10 +170,10 @@ section_schemas = { {'type' : 'null' }]}, 'fc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, - 'strategy': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]}, - 'modules': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]} + 'modules': { 'anyOf': [ {'type' : 'string'}, + {'type' : 'null' }, + {'type': 'array'}, + ]} },},},},},},},}, 'mirrors': { @@ -221,7 +223,7 @@ section_schemas = { 'items' : { 'anyOf' : [ { 'type' : 'string' }, { 'type' : 'number'}]}}, #version strings 'compiler': { - 'type' : 'array', + 'typee' : 'array', 'default' : [], 'items' : { 'type' : 'string' } }, #compiler specs 'nobuild': { -- cgit v1.2.3-60-g2f50 From 9bfc83ac5c3ae09cbcb281296d210d17984cd1db Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 4 Apr 2016 15:36:10 -0700 Subject: Changed so that py_platform.dist() grabs the major version number. --- lib/spack/spack/operating_systems/linux_distro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py index 81c7a86430..d0f24d842f 100644 --- a/lib/spack/spack/operating_systems/linux_distro.py +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -10,6 +10,6 @@ class LinuxDistro(OperatingSystem): """ def __init__(self): name = py_platform.dist()[0] - version = py_platform.dist()[1] + version = py_platform.dist()[1].split(".")[0] # Grabs major version from tuple super(LinuxDistro, self).__init__(name, version) -- cgit v1.2.3-60-g2f50 From 3bf75fab3f37bc2edb70bd49060c3831777c21dc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 6 Apr 2016 12:03:26 -0700 Subject: Cleaned up architecture. Changed operating_system from dictionary method --- lib/spack/spack/architecture.py | 68 +++++++++-------------------------------- 1 file changed, 14 insertions(+), 54 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 8d9686cf99..555fb14408 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -210,8 +210,13 @@ class Arch(object): def __str__(self): + if self.platform.name == 'darwin': + os_name = self.platform_os.name + else: + os_name = str(self.platform_os) + return (str(self.platform) +"-"+ - str(self.platform_os) + "-" + str(self.target) ) + os_name + "-" + str(self.target)) def _cmp_key(self): platform = self.platform.name @@ -231,52 +236,6 @@ class Arch(object): return d -#def _helper_to_dict(arch_field_dict, arch_field_name, *args): -# """ General method to turn each class in architecture into a -# dictionary. Takes as argument the class dictionary, the field name -# (platform, platform_os, target) and then any attribute args -# """ -# d = {} -# d[arch_field_name] = {} -# for items in args: -# d[arch_field_name][items] = arch_field_dict[items] -# return d -# - -#def to_dict(arch): -# """ Convert the Arch tuple into a dictionary for yaml dumping. This -# uses the _helper_to_dict method to create the dictionary from the -# provided architecture field. Can assign the architecture -# field name (either platform, platform_os or target) and any -# attributes that make up that architecture field, -# """ -# d = {} -# -# platform = arch.platform.__dict__ -# platform_os = arch.platform_os.__dict__ -# target = arch.target.__dict__ -# -# platform_dict = _helper_to_dict(platform,'platform','name') -# os_dict = _helper_to_dict(platform_os, 'platform_os', 'name','version', -# 'compiler_strategy') -# target_dict = _helper_to_dict(target,'target', 'name', -# 'module_name','platform_name') -# -# d.update(platform_dict) -# d.update(os_dict) -# d.update(target_dict) -# -# return d - -#def _platform_from_dict(platform): -# """Creates all the platform class module names into a dictionary of -# name : key-value pairs. From there we can construct the -# platform subclass -# """ -# platform_list = all_platforms() -# platform_names = {plat.__name__.lower():plat for plat in platform_list} -# return platform_names[platform['name']]() - def _target_from_dict(target_dict): """ Creates new instance of target and assigns all the attributes of @@ -284,20 +243,21 @@ def _target_from_dict(target_dict): """ target = Target.__new__(Target) target.name = target_dict['name'] - #target.compiler_strategy = target_dict['compiler_strategy'] target.module_name = target_dict['module_name'] if 'platform_name' in target_dict: target.platform_name = target_dict['platform_name'] return target -def _operating_system_from_dict(os_dict, platform_class): +def _operating_system_from_dict(os_dict): """ uses platform's operating system method to grab the constructed operating systems that are valid on the platform. """ # NOTE: Might need a better way to create operating system objects - name = os_dict['name'] - return platform_class.operating_system(name) - + operating_system = OperatingSystem.__new__(OperatingSystem) + operating_system.name = os_dict['name'] + operating_system.version = os_dict['version'] + operating_system.compiler_strategy = os_dict['compiler_strategy'] + return operating_system def arch_from_dict(d): """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict @@ -312,8 +272,8 @@ def arch_from_dict(d): target_dict = d['target'] target = _target_from_dict(target_dict) - platform_os = _operating_system_from_dict(os_dict, arch.platform) - arch.target =target + platform_os = _operating_system_from_dict(os_dict) + arch.target = target arch.platform_os = platform_os return arch -- cgit v1.2.3-60-g2f50 From 749508af010ee90009a61bb42a8f8214a6fabd21 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 6 Apr 2016 12:05:29 -0700 Subject: Found loops calling all_compilers that were always reading in the config file which lead to a slowdown, the fix is to cache the config file so we don't waste time re-reading the config file in again. --- lib/spack/spack/compilers/__init__.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 9601ae3e3a..caf438e232 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -151,20 +151,25 @@ def remove_compiler_from_config(compiler_spec, arch=None, scope=None): spack.config.update_config('compilers', update, scope) +_cache_config_file = {} def all_compilers_config(arch=None, scope=None): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ # Get compilers for this architecture. - arch_config = get_compiler_config(arch, scope) - # Merge 'all' compilers with arch-specific ones. - # Arch-specific compilers have higher precedence. - merged_config = get_compiler_config('all', scope=scope) - merged_config = spack.config._merge_yaml(merged_config, arch_config) + global _cache_config_file #Create a cache of the config file so we don't load all the time. - return merged_config + if not _cache_config_file: + arch_config = get_compiler_config(arch, scope) + # Merge 'all' compilers with arch-specific ones. + # Arch-specific compilers have higher precedence. + _cache_config_file = get_compiler_config('all', scope=scope) + _cache_config_file = spack.config._merge_yaml(_cache_config_file, arch_config) + return _cache_config_file + else: + return _cache_config_file def all_compilers(arch=None, scope=None): # Return compiler specs from the merged config. -- cgit v1.2.3-60-g2f50 From 918ad28a936abf6112ba91ca7eb2e121f84e60cf Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 6 Apr 2016 13:58:29 -0700 Subject: Changed all target values to architecture. Also added a sort_key method so that older architecture specs can be sorted with the newer ones --- lib/spack/spack/build_environment.py | 90 ++---------------------------------- lib/spack/spack/cmd/find.py | 18 ++++++-- 2 files changed, 16 insertions(+), 92 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 8ed3c4e1a1..87fc310b5a 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -32,8 +32,6 @@ import sys import shutil import multiprocessing import platform -import re - from llnl.util.filesystem import * import spack @@ -59,9 +57,6 @@ SPACK_DEBUG = 'SPACK_DEBUG' SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC' SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR' -SPACK_CRAYPE = 'SPACK_CRAYPE' -SPACK_COMP_MODULE = 'SPACK_COMP_MODULE' - class MakeExecutable(Executable): """Special callable executable object for make so the user can @@ -88,68 +83,6 @@ class MakeExecutable(Executable): return super(MakeExecutable, self).__call__(*args, **kwargs) -def load_module(mod): - """Takes a module name and removes modules until it is possible to - load that module. It then loads the provided module. Depends on the - modulecmd implementation of modules used in cray and lmod. - """ - #Create an executable of the module command that will output python code - modulecmd = which('modulecmd') - modulecmd.add_default_arg('python') - - # Read the module and remove any conflicting modules - # We do this without checking that they are already installed - # for ease of programming because unloading a module that is not - # loaded does nothing. - text = modulecmd('show', mod, output=str, error=str).split() - for i, word in enumerate(text): - if word == 'conflict': - exec(compile(modulecmd('unload', text[i+1], output=str, error=str), '', 'exec')) - # Load the module now that there are no conflicts - load = modulecmd('load', mod, output=str, error=str) - exec(compile(load, '', 'exec')) - - -def get_path_from_module(mod): - """Inspects a TCL module for entries that indicate the absolute path - at which the library supported by said module can be found. - """ - # Create a modulecmd executable - modulecmd = which('modulecmd') - modulecmd.add_default_arg('python') - - # Read the module - text = modulecmd('show', mod, output=str, error=str).split('\n') - - # If it lists its package directory, return that - for line in text: - if line.find(mod.upper()+'_DIR') >= 0: - words = line.split() - return words[2] - - # If it lists a -rpath instruction, use that - for line in text: - rpath = line.find('-rpath/') - if rpath >= 0: - return line[rpath+6:line.find('/lib')] - - # If it lists a -L instruction, use that - for line in text: - L = line.find('-L/') - if L >= 0: - return line[L+2:line.find('/lib')] - - # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that - for line in text: - if line.find('LD_LIBRARY_PATH') >= 0: - words = line.split() - path = words[2] - return path[:path.find('/lib')] - - # Unable to find module path - return None - - def set_compiler_environment_variables(pkg): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -176,10 +109,6 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) - if compiler.strategy == 'MODULES': - for mod in compiler.modules: - load_module(mod) - def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. @@ -243,8 +172,6 @@ def set_build_environment_variables(pkg): pkg_config_dirs.append(pcdir) path_set("PKG_CONFIG_PATH", pkg_config_dirs) - if pkg.spec.architecture.target.module_name: - load_module(pkg.spec.architecture.target.module_name) def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. @@ -314,21 +241,10 @@ def set_module_variables_for_package(pkg, m): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" - - # First load all modules for external packages and update the external - # packages' paths to reflect what is found in the modules so that we can - # rpath through the modules when possible, but if not possible they are - # already loaded. - for spec in pkg.spec.traverse(root=False): - if spec.external_module: - load_module(spec.external_module) - spec.external = get_path_from_module(spec.external_module) - - # Construct rpaths from the paths of each dep rpaths = [pkg.prefix.lib, pkg.prefix.lib64] - rpaths.extend(d.prefix.lib for d in pkg.spec.traverse(root=False) + rpaths.extend(d.prefix.lib for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib)) - rpaths.extend(d.prefix.lib64 for d in pkg.spec.traverse(root=False) + rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) return rpaths @@ -348,8 +264,8 @@ def parent_class_modules(cls): def setup_package(pkg): """Execute all environment setup routines.""" - set_build_environment_variables(pkg) set_compiler_environment_variables(pkg) + set_build_environment_variables(pkg) # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 4223c53b06..95f035c7bf 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -90,18 +90,26 @@ def display_specs(specs, **kwargs): hlen = None # Make a dict with specs keyed by target and compiler. - index = index_by(specs, ('target', 'compiler')) + index = index_by(specs, ('architecture', 'compiler')) - # Traverse the index and print out each package - for i, (target, compiler) in enumerate(sorted(index)): + # Create sort key based off of whether read architecture is string + def sort_key(index_dict): + if isinstance(index_dict[0],basestring): + return(str, index_dict[1]._cmp_key()) + elif isinstance(index_dict[0], spack.architecture.Arch): + return (index_dict[0]._cmp_key(), index_dict[1]._cmp_key()) + + sorted_index = sorted(index, key=sort_key) + + for i, (architecture, compiler) in enumerate(sorted_index): if i > 0: print header = "%s{%s} / %s{%s}" % ( - spack.spec.target_color, target, + spack.spec.architecture_color, architecture, spack.spec.compiler_color, compiler) tty.hline(colorize(header), char='-') - specs = index[(target,compiler)] + specs = index[(architecture, compiler)] specs.sort() nfmt = '.' if namespace else '_' -- cgit v1.2.3-60-g2f50 From 99117cbc9067df814c99f8427f658ad9a811f69b Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 6 Apr 2016 14:07:22 -0700 Subject: pulled from current repo the build_environment.py. Added module loading methods from our cray port --- lib/spack/spack/build_environment.py | 71 ++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 87fc310b5a..12a529d070 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -83,6 +83,66 @@ class MakeExecutable(Executable): return super(MakeExecutable, self).__call__(*args, **kwargs) +def load_module(mod): + """Takes a module name and removes modules until it is possible to + load that module. It then loads the provided module. Depends on the + modulecmd implementation of modules used in cray and lmod. + """ + #Create an executable of the module command that will output python code + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module and remove any conflicting modules + # We do this without checking that they are already installed + # for ease of programming because unloading a module that is not + # loaded does nothing. + text = modulecmd('show', mod, output=str, error=str).split() + for i, word in enumerate(text): + if word == 'conflict': + exec(compile(modulecmd('unload', text[i+1], output=str, error=str), '', 'exec')) + # Load the module now that there are no conflicts + load = modulecmd('load', mod, output=str, error=str) + exec(compile(load, '', 'exec')) + +def get_path_from_module(mod): + """Inspects a TCL module for entries that indicate the absolute path + at which the library supported by said module can be found. + """ + # Create a modulecmd executable + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module + text = modulecmd('show', mod, output=str, error=str).split('\n') + + # If it lists its package directory, return that + for line in text: + if line.find(mod.upper()+'_DIR') >= 0: + words = line.split() + return words[2] + + # If it lists a -rpath instruction, use that + for line in text: + rpath = line.find('-rpath/') + if rpath >= 0: + return line[rpath+6:line.find('/lib')] + + # If it lists a -L instruction, use that + for line in text: + L = line.find('-L/') + if L >= 0: + return line[L+2:line.find('/lib')] + + # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that + for line in text: + if line.find('LD_LIBRARY_PATH') >= 0: + words = line.split() + path = words[2] + return path[:path.find('/lib')] + + # Unable to find module path + return None + def set_compiler_environment_variables(pkg): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -109,6 +169,9 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) + if compiler.strategy == 'MODULES': + for mod in compiler.modules: + load_module(mod) def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. @@ -172,6 +235,9 @@ def set_build_environment_variables(pkg): pkg_config_dirs.append(pcdir) path_set("PKG_CONFIG_PATH", pkg_config_dirs) + if pkg.spec.architecture.target.module_name: + load_module(pkg.spec.architecture.target.module_name) + def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. @@ -241,6 +307,11 @@ def set_module_variables_for_package(pkg, m): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" + for spec in pkg.spec.traverse(root=False): + if spec.external_module: + load_module(spec.external_module) + spec.external = get_path_from_module(spec.external_module) + rpaths = [pkg.prefix.lib, pkg.prefix.lib64] rpaths.extend(d.prefix.lib for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib)) -- cgit v1.2.3-60-g2f50 From 143a4d7f7688b506811a564768ed49a2a12d1992 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 7 Apr 2016 14:36:26 -0700 Subject: Changed default to default_target, also added frontend and backend to operating system --- lib/spack/spack/architecture.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 555fb14408..ca306b0484 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -118,14 +118,14 @@ class Platform(object): front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ - if name == 'default': + if name == 'default_target': name = self.default - elif name == 'front_end' or name == 'fe': + elif name == 'frontend' or name == 'fe': name = self.front_end - elif name == 'back_end' or name == 'be': + elif name == 'backend' or name == 'be': name = self.back_end - return self.targets[name] + return self.targets.get(name, None) def add_operating_system(self, name, os_class): """ Add the operating_system class object into the @@ -136,12 +136,12 @@ class Platform(object): def operating_system(self, name): if name == 'default_os': name = self.default_os - if name == 'front_os': + if name == 'frontend' or name == "fe": name = self.front_os - if name == 'back_os': + if name == 'backend' or name == 'be': name = self.back_os - return self.operating_sys[name] + return self.operating_sys.get(name, None) @classmethod def detect(self): -- cgit v1.2.3-60-g2f50 From 07df40320331c118306c40c36deae8d57ca6073c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 7 Apr 2016 14:37:00 -0700 Subject: Cleaned up the concretize_architecture method by removing commented out code --- lib/spack/spack/concretize.py | 62 ++++--------------------------------------- 1 file changed, 5 insertions(+), 57 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index c5dad93aef..14edcf1f3d 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -236,14 +236,6 @@ class DefaultConcretizer(object): return True #changed -# """ Future method for concretizing operating system """ -# if isinstance(arch.platform_os, spack.architecture.OperatingSystem): -# return False -# else: -# arch.arch_os = platform.operating_system('default_os') -# return True - - def _concretize_target(self, spec): if spec.architecture.target is not None: if isinstance(spec.architecture.target,spack.architecture.Target): @@ -258,16 +250,10 @@ class DefaultConcretizer(object): else: spec.add_target_from_string(spec.root.architecture.target) else: - spec.architecture.target = spec.architecture.platform.target('default') + spec.architecture.target = spec.architecture.platform.target('default_target') return True #changed -# if isinstance(arch.target, spack.architecture.Target): -# return False -# else: -# arch.target = platform.target('default') -# return True - def concretize_architecture(self, spec): """If the spec is empty provide the defaults of the platform. If the architecture is not a basestring, then check if either the platform, @@ -283,50 +269,12 @@ class DefaultConcretizer(object): # Set the architecture to all defaults spec.architecture = spack.architecture.Arch() return True - #If there is a target and it is a tuple and has both filled return - #False -# if isinstance(spec.architecture, basestring): -# spec.split_architecture_string(spec.architecture) - - ret = any(( - self._concretize_operating_system(spec), - self._concretize_target(spec))) - - - # Does not look pretty at all!!! -# if spec.root.architecture and \ -# not isinstance(spec.root.architecture, basestring): -# bool_flag = any(( -# self._concretize_platform(spec.root.architecture, platform), -# self._concretize_operating_system(spec.root.architecture, -# platform), -# self._concretize_target(spec.root.target, platform))) -# spec.architecture =spec.root.architecture -# return bool_flag -# else: -# spec.add_architecture_from_string(spec.root.architecture) - + + # Concretize the operating_system and target based of the spec + ret = any((self._concretize_operating_system(spec), + self._concretize_target(spec))) return ret - # if there is no target specified used the defaults - - #if spec.target is not None: - # if isinstance(spec.target,spack.architecture.Target): - # return False - # else: - # spec.add_target_from_string(spec.target) - # return True #changed - - #if spec.root.target: - # if isinstance(spec.root.target,spack.architecture.Target): - # spec.target = spec.root.target - # else: - # spec.add_target_from_string(spec.root.target) - #else: - # platform = spack.architecture.sys_type() - # spec.target = platform.target('default') - - #return True #changed def concretize_variants(self, spec): -- cgit v1.2.3-60-g2f50 From 0ad317213c0f32c125c7043b5e57aa678ba63f8f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 7 Apr 2016 14:38:04 -0700 Subject: Changed architecture parser so that if user just enters in frontend or backend, then both the os and target will take those names. In the concretize method the frontend target/os and backend target/os will be picked to match each other --- lib/spack/spack/spec.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index fa2e9fd4c8..94d881da28 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -462,11 +462,10 @@ class Spec(object): """Called by the parser to set the architecture.""" if self.architecture: raise DuplicateArchitectureError( "Spec for '%s' cannot have two architectures." % self.name) - platform = spack.architecture.sys_type() if '-' in architecture: os, target = architecture.split('-') - else: - os = None + elif architecture == 'frontend' or architecture == 'backend': + os = architecture target = architecture self.architecture = spack.architecture.Arch(os, target) -- cgit v1.2.3-60-g2f50 From 7b777568ed27f2dc0f15283d0209541b65b47b24 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 7 Apr 2016 14:38:43 -0700 Subject: Added new concretization of architecture tests, test each combination of user input to make sure the correct os and target are chosen --- lib/spack/spack/test/architecture.py | 45 ++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 2274a9901d..1df1e3ffa2 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -6,7 +6,7 @@ import os import platform import spack from spack.architecture import * -import spack.spec +from spack.spec import * from spack.platforms.cray_xc import CrayXc from spack.platforms.linux import Linux from spack.platforms.bgq import Bgq @@ -17,7 +17,7 @@ class ArchitectureTest(unittest.TestCase): def test_dict_functions_for_architecture(self): arch = Arch() arch.platform_os = arch.platform.operating_system('default_os') - arch.target = arch.platform.target('default') + arch.target = arch.platform.target('default_target') d = arch.to_dict() @@ -55,3 +55,44 @@ class ArchitectureTest(unittest.TestCase): my_platform_class = Darwin() self.assertEqual(str(output_platform_class), str(my_platform_class)) + + def setUp(self): + self.platform = sys_type() + + def test_user_front_end_input(self): + """Test when user inputs just frontend that both the frontend target + and frontend operating system match + """ + frontend_os = self.platform.operating_system("frontend") + frontend_target = self.platform.target("frontend") + print frontend_os + print frontend_target + frontend_spec = Spec("zlib=frontend") + frontend_spec.concretize() + self.assertEqual(frontend_os, frontend_spec.architecture.platform_os) + self.assertEqual(frontend_target, frontend_spec.architecture.target) + + def test_user_back_end_input(self): + """Test when user inputs backend that both the backend target and + backend operating system match + """ + backend_os = self.platform.operating_system("backend") + backend_target = self.platform.target("backend") + print backend_os + print backend_target + backend_spec = Spec("zlib=backend") + backend_spec.concretize() + self.assertEqual(backend_os, backend_spec.architecture.platform_os) + self.assertEqual(backend_target, backend_spec.architecture.target) + + def test_user_defaults(self): + default_os = self.platform.operating_system("default_os") + default_target = self.platform.target("default_target") + + default_spec = Spec("zlib") # default is no args + default_spec.concretize() + self.assertEqual(default_os, default_spec.architecture.platform_os) + self.assertEqual(default_target, default_spec.architecture.target) + + def test_user_combination(self): + -- cgit v1.2.3-60-g2f50 From 6ff6c805af2a4626e0611f3798eedd3a6f1b3e06 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 10:28:30 -0700 Subject: Assigned self.front_os, and back_os to self.default. Helps with testing. If we're on a darwin or linux machine, chances are there is no back end or front end operating system, but rather than have those as None, I just assign them the default which is detected via python platform. --- lib/spack/spack/platforms/darwin.py | 4 ++++ lib/spack/spack/platforms/linux.py | 2 ++ 2 files changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 4c3d38851f..7901f82d06 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -12,7 +12,11 @@ class Darwin(Platform): super(Darwin, self).__init__('darwin') self.add_target(self.default, Target(self.default)) mac_os = MacOsx() + self.default_os = mac_os.name + self.front_os = mac_os.name + self.back_os = mac_os.name + self.add_operating_system(mac_os.name, mac_os) @classmethod diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 3243a1dcdf..18050ac79e 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -13,6 +13,8 @@ class Linux(Platform): self.add_target(self.default, Target(self.default)) linux_dist = LinuxDistro() self.default_os = linux_dist.name + self.front_os = linux_dist.name + self.back_os = linux_dist.name self.add_operating_system(linux_dist.name, linux_dist) @classmethod -- cgit v1.2.3-60-g2f50 From 0d1a1b7526ce1c129bed352b175f33f2875b8b41 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 10:29:37 -0700 Subject: Changed _set_architecture so if user inputs only the target, then os is None. This prevents the os being referenced before assignment error --- lib/spack/spack/spec.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 94d881da28..28cd6a0274 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -461,12 +461,17 @@ class Spec(object): def _set_architecture(self, architecture): """Called by the parser to set the architecture.""" if self.architecture: raise DuplicateArchitectureError( - "Spec for '%s' cannot have two architectures." % self.name) + "Spec for '%s' cannot have two architectures." % self.name) if '-' in architecture: os, target = architecture.split('-') + elif architecture == 'frontend' or architecture == 'backend': os = architecture target = architecture + else: + os = None + target = architecture + self.architecture = spack.architecture.Arch(os, target) -- cgit v1.2.3-60-g2f50 From 277efc1dfbb00117db65918ff22928d42e1194e2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 11:17:56 -0700 Subject: Added test that works on every type of combination the user could possibly enter for arch spec --- lib/spack/spack/test/architecture.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 1df1e3ffa2..1ca3e58831 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -65,8 +65,6 @@ class ArchitectureTest(unittest.TestCase): """ frontend_os = self.platform.operating_system("frontend") frontend_target = self.platform.target("frontend") - print frontend_os - print frontend_target frontend_spec = Spec("zlib=frontend") frontend_spec.concretize() self.assertEqual(frontend_os, frontend_spec.architecture.platform_os) @@ -78,8 +76,6 @@ class ArchitectureTest(unittest.TestCase): """ backend_os = self.platform.operating_system("backend") backend_target = self.platform.target("backend") - print backend_os - print backend_target backend_spec = Spec("zlib=backend") backend_spec.concretize() self.assertEqual(backend_os, backend_spec.architecture.platform_os) @@ -94,5 +90,24 @@ class ArchitectureTest(unittest.TestCase): self.assertEqual(default_os, default_spec.architecture.platform_os) self.assertEqual(default_target, default_spec.architecture.target) - def test_user_combination(self): + def test_user_input_combination(self): + os_list = self.platform.operating_sys.keys() + target_list = self.platform.targets.keys() + additional = ["fe", "be", "frontend", "backend"] + + os_list.extend(additional) + target_list.extend(additional) + combinations = itertools.product(os_list, target_list) + results = [] + for arch in combinations: + o,t = arch + arch_spec = "-".join(arch) + spec = Spec("zlib=%s" % arch_spec) + spec.concretize() + results.append(spec.architecture.platform_os == self.platform.operating_system(o)) + results.append(spec.architecture.target == self.platform.target(t)) + res = all(results) + print res + self.assertTrue(res) + -- cgit v1.2.3-60-g2f50 From 618b3f5f2d149fe3c20f90c0890748c8f3589afb Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 11:18:30 -0700 Subject: Changed default to default_target so the tests can pass --- lib/spack/spack/test/spec_semantics.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 0b310488a9..84d8c0edd8 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -308,7 +308,7 @@ class SpecSematicsTest(MockPackagesTest): def test_constrain_target(self): platform = spack.architecture.sys_type() - target = platform.target('default').name + target = platform.target('default_target').name self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) @@ -339,7 +339,7 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_changed('libelf', '+debug') self.check_constrain_changed('libelf', '~debug') platform = spack.architecture.sys_type() - self.check_constrain_changed('libelf', '='+platform.target('default').name) + self.check_constrain_changed('libelf', '='+platform.target('default_target').name) def test_constrain_not_changed(self): @@ -351,8 +351,8 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_not_changed('libelf+debug', '+debug') self.check_constrain_not_changed('libelf~debug', '~debug') platform = spack.architecture.sys_type() - default = platform.target('default').name - self.check_constrain_not_changed('libelf='+default, '='+default) + default_target = platform.target('default_target').name + self.check_constrain_not_changed('libelf='+default_target, '='+default_target) self.check_constrain_not_changed('libelf^foo', 'libelf^foo') self.check_constrain_not_changed('libelf^foo^bar', 'libelf^foo^bar') @@ -365,8 +365,8 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_changed('libelf^foo', 'libelf^foo+debug') self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') platform = spack.architecture.sys_type() - default = platform.target('default').name - self.check_constrain_changed('libelf^foo', 'libelf^foo='+default) + default_target = platform.target('default_target').name + self.check_constrain_changed('libelf^foo', 'libelf^foo='+default_target) def test_constrain_dependency_not_changed(self): @@ -377,6 +377,6 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') platform = spack.architecture.sys_type() - default = platform.target('default').name - self.check_constrain_not_changed('libelf^foo='+default, 'libelf^foo='+default) + default_target = platform.target('default_target').name + self.check_constrain_not_changed('libelf^foo='+default_target, 'libelf^foo='+default_target) -- cgit v1.2.3-60-g2f50 From 5bcd1e7ccd9ff6568c06a13998f2d552d1b37b60 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 12:43:08 -0700 Subject: If we already have the name and the platform, then we should probably go ahead and concretize it in the constructor. Else leave it as None and concretize it later. --- lib/spack/spack/architecture.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index ca306b0484..e2c189980a 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -199,8 +199,13 @@ class Arch(object): def __init__(self, platform_os=None, target=None): self.platform = sys_type() + if platform_os: + platform_os = self.platform.operating_system(platform_os) self.platform_os = platform_os - self.target = target + if target: + target = self.platform.target(target) + self.target = target + @property def concrete(self): -- cgit v1.2.3-60-g2f50 From 01a36ab33341e2af91c6e6f5ca58efbeecfb6ec4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 12:44:27 -0700 Subject: Changed the logic of concretize. We don't have to worry about whether the part of the spec is a string since any string parsed is made into an object via the Arch constructor. Any dependencies will take the root arch spec as well --- lib/spack/spack/concretize.py | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 14edcf1f3d..3422e86ae8 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -220,39 +220,29 @@ class DefaultConcretizer(object): return True # Things changed def _concretize_operating_system(self, spec): - if spec.architecture.platform_os is not None: - if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): - return False - else: - spec.add_operating_system_from_string(spec.architecture.platform_os) - return True #changed + platform = spec.architecture.platform + if spec.architecture.platform_os is not None and isinstance( + spec.architecture.platform_os,spack.architecture.OperatingSystem): + return False + if spec.root.architecture and spec.root.architecture.platform_os: if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): spec.architecture.platform_os = spec.root.architecture.platform_os - else: - spec.add_operating_system_from_string(spec.root.architecture.platform_os) else: spec.architecture.platform_os = spec.architecture.platform.operating_system('default_os') - - return True #changed + return True #changed def _concretize_target(self, spec): - if spec.architecture.target is not None: - if isinstance(spec.architecture.target,spack.architecture.Target): - return False - else: - spec.add_target_from_string(spec.architecture.target) - return True #changed - + platform = spec.architecture.platform + if spec.architecture.target is not None and isinstance( + spec.architecture.target, spack.architecture.Target): + return False if spec.root.architecture and spec.root.architecture.target: if isinstance(spec.root.architecture.target,spack.architecture.Target): spec.architecture.target = spec.root.architecture.target - else: - spec.add_target_from_string(spec.root.architecture.target) else: spec.architecture.target = spec.architecture.platform.target('default_target') - - return True #changed + return True #changed def concretize_architecture(self, spec): """If the spec is empty provide the defaults of the platform. If the -- cgit v1.2.3-60-g2f50 From 3a281b239998133c7ed3fe03a54e44d643d0d0bd Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 12:46:44 -0700 Subject: Commented out the add_x_from_string methods since we don't need to worry about it. Also, changed the way architecture is set. Todd wanted to have users be able to input just frontend and/or backend in the architecture spec. This will be able to parse those inputs. --- lib/spack/spack/spec.py | 35 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 28cd6a0274..eee9318550 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -465,7 +465,7 @@ class Spec(object): if '-' in architecture: os, target = architecture.split('-') - elif architecture == 'frontend' or architecture == 'backend': + elif architecture in ['frontend','backend','fe','be']: os = architecture target = architecture else: @@ -1302,28 +1302,17 @@ class Spec(object): except SpecError: return parse_anonymous_spec(spec_like, self.name) - def _is_valid_platform(self, platform, platform_list): - if platform in platform_list: - return True - return False - - def _is_valid_target(self, target, platform): - return target in platform.targets - - def _is_valid_os(self, os_string, platform): - return os_string in platform.operating_sys - - def add_target_from_string(self, target): - if target is None: - self.architecture.target = self.architecture.platform.target('default_target') - else: - self.architecture.target = self.architecture.platform.target(target) - - def add_operating_system_from_string(self, os): - if os is None: - self.architecture.platform_os = self.architecture.platform.operating_system('default_os') - else: - self.architecture.platform_os = self.architecture.platform.operating_system(os) + #def add_target_from_string(self, target): + # if target is None: + # self.architecture.target = self.architecture.platform.target('default_target') + # else: + # self.architecture.target = self.architecture.platform.target(target) + + #def add_operating_system_from_string(self, os): + # if os is None: + # self.architecture.platform_os = self.architecture.platform.operating_system('default_os') + # else: + # self.architecture.platform_os = self.architecture.platform.operating_system(os) def satisfies(self, other, deps=True, strict=False): -- cgit v1.2.3-60-g2f50 From 8cd13d4b3552e6737ed9fc768bbe544024e05b10 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 28 Apr 2016 12:49:21 -0700 Subject: Make arch command print out the platform. --- lib/spack/spack/cmd/arch.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/arch.py b/lib/spack/spack/cmd/arch.py index db27544ffd..cd0ce2bb2e 100644 --- a/lib/spack/spack/cmd/arch.py +++ b/lib/spack/spack/cmd/arch.py @@ -28,8 +28,4 @@ import spack.architecture as architecture description = "Print the architecture for this machine" def arch(parser, args): - configured_sys_type = architecture.get_sys_type_from_spack_globals() - if not configured_sys_type: - configured_sys_type = "autodetect" - print "Configured sys_type: %s" % configured_sys_type - print "Autodetected default sys_type: %s" % architecture.sys_type() + print architecture.sys_type() -- cgit v1.2.3-60-g2f50 From 90b7b7ba5c483054a3b6687f84306601d0e62d15 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 4 May 2016 21:42:59 -0700 Subject: Reworked compiler finding/storing/format to allow for multiple compilers with the same spec for different operating systems. TODO: fix config tests. All others up to date --- lib/spack/spack/architecture.py | 96 +++++++++++- lib/spack/spack/build_environment.py | 5 +- lib/spack/spack/cmd/compiler.py | 4 +- lib/spack/spack/compiler.py | 166 +++++--------------- lib/spack/spack/compilers/__init__.py | 234 ++++++++++++++--------------- lib/spack/spack/concretize.py | 8 +- lib/spack/spack/config.py | 35 +++-- lib/spack/spack/operating_systems/cnl.py | 51 ++++++- lib/spack/spack/test/architecture.py | 39 +++-- lib/spack/spack/test/config.py | 95 +++++++++--- lib/spack/spack/test/mock_packages_test.py | 43 +++++- lib/spack/spack/test/spec_dag.py | 2 +- 12 files changed, 456 insertions(+), 322 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e2c189980a..301495f728 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,6 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import os from collections import namedtuple import imp import platform as py_platform @@ -32,7 +33,10 @@ from llnl.util.filesystem import join_path import llnl.util.tty as tty import spack +import spack.compilers from spack.util.naming import mod_to_class +from spack.util.environment import get_path +from spack.util.multiproc import parmap import spack.error as serr @@ -143,6 +147,7 @@ class Platform(object): return self.operating_sys.get(name, None) + @classmethod def detect(self): """ Subclass is responsible for implementing this method. @@ -170,10 +175,9 @@ class OperatingSystem(object): find compilers we call find_compilers method for each operating system """ - def __init__(self, name, version, compiler_strategy="PATH"): + def __init__(self, name, version): self.name = name self.version = version - self.compiler_strategy = compiler_strategy def __str__(self): return self.name + self.version @@ -182,13 +186,96 @@ class OperatingSystem(object): return self.__str__() def _cmp_key(self): - return (self.name, self.version, self.compiler_strategy) + return (self.name, self.version) + + + def find_compilers(self, *paths): + """ + Return a list of compilers found in the suppied paths. + This invokes the find() method for each Compiler class, + and appends the compilers detected to a list. + """ + if not paths: + paths = get_path('PATH') + # Make sure path elements exist, and include /bin directories + # under prefixes. + filtered_path = [] + for p in paths: + # Eliminate symlinks and just take the real directories. + p = os.path.realpath(p) + if not os.path.isdir(p): + continue + filtered_path.append(p) + + # Check for a bin directory, add it if it exists + bin = join_path(p, 'bin') + if os.path.isdir(bin): + filtered_path.append(os.path.realpath(bin)) + + # Once the paths are cleaned up, do a search for each type of + # compiler. We can spawn a bunch of parallel searches to reduce + # the overhead of spelunking all these directories. + types = spack.compilers.all_compiler_types() + compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *filtered_path), types) + + # ensure all the version calls we made are cached in the parent + # process, as well. This speeds up Spack a lot. + clist = reduce(lambda x,y: x+y, compiler_lists) + return clist + + def find_compiler(self, cmp_cls, *path): + """Try to find the given type of compiler in the user's + environment. For each set of compilers found, this returns + compiler objects with the cc, cxx, f77, fc paths and the + version filled in. + + This will search for compilers with the names in cc_names, + cxx_names, etc. and it will group them if they have common + prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would + be grouped with g++-mp-4.7 and gfortran-mp-4.7. + """ + dicts = parmap( + lambda t: cmp_cls._find_matches_in_path(*t), + [(cmp_cls.cc_names, cmp_cls.cc_version) + tuple(path), + (cmp_cls.cxx_names, cmp_cls.cxx_version) + tuple(path), + (cmp_cls.f77_names, cmp_cls.f77_version) + tuple(path), + (cmp_cls.fc_names, cmp_cls.fc_version) + tuple(path)]) + + all_keys = set() + for d in dicts: + all_keys.update(d) + + compilers = {} + for k in all_keys: + ver, pre, suf = k + + # Skip compilers with unknown version. + if ver == 'unknown': + continue + + paths = tuple(pn[k] if k in pn else None for pn in dicts) + spec = spack.spec.CompilerSpec(cmp_cls.name, ver) + + if ver in compilers: + prev = compilers[ver] + + # prefer the one with more compilers. + prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc] + newcount = len([p for p in paths if p is not None]) + prevcount = len([p for p in prev_paths if p is not None]) + + # Don't add if it's not an improvement over prev compiler. + if newcount <= prevcount: + continue + + compilers[ver] = cmp_cls(spec, self, paths) + + return list(compilers.values()) def to_dict(self): d = {} d['name'] = self.name d['version'] = self.version - d['compiler_strategy'] = self.compiler_strategy return d @@ -261,7 +348,6 @@ def _operating_system_from_dict(os_dict): operating_system = OperatingSystem.__new__(OperatingSystem) operating_system.name = os_dict['name'] operating_system.version = os_dict['version'] - operating_system.compiler_strategy = os_dict['compiler_strategy'] return operating_system def arch_from_dict(d): diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 12a529d070..82d0792e26 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -169,9 +169,8 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) - if compiler.strategy == 'MODULES': - for mod in compiler.modules: - load_module(mod) + for mod in compiler.modules: + load_module(mod) def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index a7de61e1e5..e3d311b6b8 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -80,7 +80,7 @@ def compiler_add(args): if not paths: paths = get_path('PATH') - compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) + compilers = [c for c in spack.compilers.find_compilers(*paths) if c.spec not in spack.compilers.all_compilers(scope=args.scope)] if compilers: @@ -125,6 +125,8 @@ def compiler_info(args): print "\tcxx = %s" % c.cxx print "\tf77 = %s" % c.f77 print "\tfc = %s" % c.fc + print "\tmodules = %s" % c.modules + print "\toperating system = %s" % c.operating_system def compiler_list(args): diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index c285c8a29d..8b973d8190 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -106,14 +106,14 @@ class Compiler(object): PrgEnv_compiler = None - def __init__(self, cspec, strategy, paths, modules=None): + def __init__(self, cspec, operating_system, paths, modules=[], alias=None): def check(exe): if exe is None: return None _verify_executables(exe) return exe - self.strategy = strategy + self.operating_system = operating_system self.cc = check(paths[0]) self.cxx = check(paths[1]) @@ -126,62 +126,33 @@ class Compiler(object): self.spec = cspec self.modules = modules - + self.alias = alias @property def version(self): return self.spec.version - # - # Compiler classes have methods for querying the version of - # specific compiler executables. This is used when discovering compilers. - # - # Compiler *instances* are just data objects, and can only be - # constructed from an actual set of executables. - # - - @classmethod - def default_version(cls, cc): - """Override just this to override all compiler version functions.""" - return dumpversion(cc) - - @classmethod - def cc_version(cls, cc): - return cls.default_version(cc) - - @classmethod - def cxx_version(cls, cxx): - return cls.default_version(cxx) - - @classmethod - def f77_version(cls, f77): - return cls.default_version(f77) - - @classmethod - def fc_version(cls, fc): - return cls.default_version(fc) - @classmethod def _find_matches_in_path(cls, compiler_names, detect_version, *path): """Finds compilers in the paths supplied. - - Looks for all combinations of ``compiler_names`` with the - ``prefixes`` and ``suffixes`` defined for this compiler - class. If any compilers match the compiler_names, - prefixes, or suffixes, uses ``detect_version`` to figure - out what version the compiler is. - - This returns a dict with compilers grouped by (prefix, - suffix, version) tuples. This can be further organized by - find(). - """ + + Looks for all combinations of ``compiler_names`` with the + ``prefixes`` and ``suffixes`` defined for this compiler + class. If any compilers match the compiler_names, + prefixes, or suffixes, uses ``detect_version`` to figure + out what version the compiler is. + + This returns a dict with compilers grouped by (prefix, + suffix, version) tuples. This can be further organized by + find(). + """ if not path: path = get_path('PATH') - + prefixes = [''] + cls.prefixes suffixes = [''] + cls.suffixes - + checks = [] for directory in path: if not (os.path.isdir(directory) and @@ -219,89 +190,34 @@ class Compiler(object): successful = [key for key in parmap(check, checks) if key is not None] return dict(((v, p, s), path) for v, p, s, path in successful) - @classmethod - def find(cls, *path): - compilers = [] - platform = spack.architecture.sys_type() - strategies = [o.compiler_strategy for o in platform.operating_sys.values()] - if 'PATH' in strategies: - compilers.extend(cls.find_in_path(*path)) - if 'MODULES' in strategies: - compilers.extend(cls.find_in_modules()) - return compilers - + # + # Compiler classes have methods for querying the version of + # specific compiler executables. This is used when discovering compilers. + # + # Compiler *instances* are just data objects, and can only be + # constructed from an actual set of executables. + # @classmethod - def find_in_path(cls, *path): - """Try to find this type of compiler in the user's - environment. For each set of compilers found, this returns - compiler objects with the cc, cxx, f77, fc paths and the - version filled in. - - This will search for compilers with the names in cc_names, - cxx_names, etc. and it will group them if they have common - prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would - be grouped with g++-mp-4.7 and gfortran-mp-4.7. - """ - dicts = parmap( - lambda t: cls._find_matches_in_path(*t), - [(cls.cc_names, cls.cc_version) + tuple(path), - (cls.cxx_names, cls.cxx_version) + tuple(path), - (cls.f77_names, cls.f77_version) + tuple(path), - (cls.fc_names, cls.fc_version) + tuple(path)]) - - all_keys = set() - for d in dicts: - all_keys.update(d) - - compilers = {} - for k in all_keys: - ver, pre, suf = k - - # Skip compilers with unknown version. - if ver == 'unknown': - continue - - paths = tuple(pn[k] if k in pn else None for pn in dicts) - spec = spack.spec.CompilerSpec(cls.name, ver) - - if ver in compilers: - prev = compilers[ver] - - # prefer the one with more compilers. - prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc] - newcount = len([p for p in paths if p is not None]) - prevcount = len([p for p in prev_paths if p is not None]) - - # Don't add if it's not an improvement over prev compiler. - if newcount <= prevcount: - continue - - compilers[ver] = cls(spec, 'PATH', paths) - - return list(compilers.values()) + def default_version(cls, cc): + """Override just this to override all compiler version functions.""" + return dumpversion(cc) + @classmethod + def cc_version(cls, cc): + return cls.default_version(cc) @classmethod - def find_in_modules(cls): - compilers = [] - if cls.PrgEnv: - if not cls.PrgEnv_compiler: - tty.die('Must supply PrgEnv_compiler with PrgEnv') - - modulecmd = which('modulecmd') - modulecmd.add_default_arg('python') - - output = modulecmd('avail', cls.PrgEnv_compiler, output=str, error=str) - matches = re.findall(r'(%s)/([\d\.]+[\d])' % cls.PrgEnv_compiler, output) - for name, version in matches: - v = version - comp = cls(spack.spec.CompilerSpec(name + '@' + v), 'MODULES', - ['cc', 'CC', 'ftn'], [cls.PrgEnv, name +'/' + v]) + def cxx_version(cls, cxx): + return cls.default_version(cxx) - compilers.append(comp) + @classmethod + def f77_version(cls, f77): + return cls.default_version(f77) - return compilers + @classmethod + def fc_version(cls, fc): + return cls.default_version(fc) def __repr__(self): @@ -311,12 +227,8 @@ class Compiler(object): def __str__(self): """Return a string represntation of the compiler toolchain.""" - if self.strategy is 'MODULES': - return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc, self.modules)))) - else: - return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc)))) + return "%s(%s)" % ( + self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc, self.modules, str(self.operating_system))))) class CompilerAccessError(spack.error.SpackError): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index caf438e232..8ef6e976ba 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -29,6 +29,10 @@ import imp import os import platform import copy +import hashlib +import base64 +import yaml +import sys from llnl.util.lang import memoized, list_modules from llnl.util.filesystem import join_path @@ -46,8 +50,8 @@ from spack.util.naming import mod_to_class from spack.util.environment import get_path _imported_compilers_module = 'spack.compilers' -_required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] -_optional_instance_vars = ['modules', 'strategy'] +_path_instance_vars = ['cc', 'cxx', 'f77', 'fc'] +_other_instance_vars = ['modules', 'operating_system'] _default_order = [] # TODO: customize order in config file @@ -67,93 +71,85 @@ def _auto_compiler_spec(function): def _to_dict(compiler): """Return a dict version of compiler suitable to insert in YAML.""" + d = {} + d['spec'] = str(compiler.spec) + d['paths'] = dict( (attr, getattr(compiler, attr, None)) for attr in _path_instance_vars ) + d['operating_system'] = compiler.operating_system.to_dict() + d['modules'] = compiler.modules + + if not compiler.alias: + yaml_text = yaml.dump( + d, default_flow_style=True, width=sys.maxint) + sha = hashlib.sha1(yaml_text) + compiler.alias = base64.b32encode(sha.digest()).lower()[:8] return { - str(compiler.spec) : dict( - (attr, getattr(compiler, attr, None)) - for attr in _required_instance_vars + _optional_instance_vars) + compiler.alias: d } -def get_compiler_config(arch=None, scope=None): +def get_compiler_config(scope=None): """Return the compiler configuration for the specified architecture. """ - # Check whether we're on a front-end (native) architecture. - my_arch = spack.architecture.Arch() - if arch is None: - arch = my_arch - if isinstance(arch, basestring) and arch == 'all': - name = 'all' - else: - name = arch.platform.name - def init_compiler_config(): """Compiler search used when Spack has no compilers.""" - config[name] = {} - compilers = find_compilers(*get_path('PATH')) + config = {} + compilers = find_compilers() for compiler in compilers: - config[name].update(_to_dict(compiler)) + config.update(_to_dict(compiler)) spack.config.update_config('compilers', config, scope=scope) config = spack.config.get_config('compilers', scope=scope) # Update the configuration if there are currently no compilers # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. - if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: - if scope is None: - # We know no compilers were configured in any scope. +# if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: + if scope is None: + # We know no compilers were configured in any scope. + init_compiler_config() + elif scope == 'user': + # Check the site config and update the user config if + # nothing is configured at the site level. + site_config = spack.config.get_config('compilers', scope='site') + if not site_config: init_compiler_config() - elif scope == 'user': - # Check the site config and update the user config if - # nothing is configured at the site level. - site_config = spack.config.get_config('compilers', scope='site') - if not site_config: - init_compiler_config() - return config[name] if name in config else {} + return config -def add_compilers_to_config(compilers, arch=None, scope=None): +def add_compilers_to_config(compilers, scope=None): """Add compilers to the config for the specified architecture. Arguments: - compilers: a list of Compiler objects. - - arch: arch to add compilers for. - scope: configuration scope to modify. """ - if arch is None: - arch = spack.architecture.Arch() - - compiler_config = get_compiler_config(arch, scope) + compiler_config = get_compiler_config(scope) for compiler in compilers: - compiler_config[str(compiler.spec)] = dict( - (c, getattr(compiler, c, "None")) - for c in _required_instance_vars + _optional_instance_vars) + compiler_config = _to_dict(compiler) - update = { arch.platform.name : compiler_config } - spack.config.update_config('compilers', update, scope) + spack.config.update_config('compilers', compiler_config, scope) @_auto_compiler_spec -def remove_compiler_from_config(compiler_spec, arch=None, scope=None): +def remove_compiler_from_config(compiler_spec, scope=None): """Remove compilers from the config, by spec. Arguments: - compiler_specs: a list of CompilerSpec objects. - - arch: arch to add compilers for. - scope: configuration scope to modify. """ - if arch is None: - arch = spack.architecture.Arch() - - compiler_config = get_compiler_config(arch, scope) - del compiler_config[str(compiler_spec)] - update = { arch : compiler_config } + compiler_config = get_compiler_config(scope) + matches = [(a,c) for (a,c) in compiler_config.items() if c['spec'] == compiler_spec] + if len(matches) == 1: + del compiler_config[matches[0][0]] + else: + CompilerSpecInsufficientlySpecificError(compiler_spec) - spack.config.update_config('compilers', update, scope) + spack.config.update_config('compilers', compiler_config, scope) _cache_config_file = {} -def all_compilers_config(arch=None, scope=None): +def all_compilers_config(scope=None): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ @@ -161,20 +157,16 @@ def all_compilers_config(arch=None, scope=None): global _cache_config_file #Create a cache of the config file so we don't load all the time. if not _cache_config_file: - arch_config = get_compiler_config(arch, scope) - # Merge 'all' compilers with arch-specific ones. - # Arch-specific compilers have higher precedence. - _cache_config_file = get_compiler_config('all', scope=scope) - _cache_config_file = spack.config._merge_yaml(_cache_config_file, arch_config) + _cache_config_file = get_compiler_config(scope) return _cache_config_file else: return _cache_config_file -def all_compilers(arch=None, scope=None): +def all_compilers(scope=None): # Return compiler specs from the merged config. - return [spack.spec.CompilerSpec(s) - for s in all_compilers_config(arch, scope)] + return [spack.spec.CompilerSpec(s['spec']) + for s in all_compilers_config(scope).values()] def default_compiler(): @@ -189,37 +181,19 @@ def default_compiler(): return sorted(versions)[-1] -def find_compilers(*path): +def find_compilers(): """Return a list of compilers found in the suppied paths. - This invokes the find() method for each Compiler class, - and appends the compilers detected to a list. + This invokes the find_compilers() method for each operating + system associated with the host platform, and appends + the compilers detected to a list. """ - # Make sure path elements exist, and include /bin directories - # under prefixes. - filtered_path = [] - for p in path: - # Eliminate symlinks and just take the real directories. - p = os.path.realpath(p) - if not os.path.isdir(p): - continue - filtered_path.append(p) - - # Check for a bin directory, add it if it exists - bin = join_path(p, 'bin') - if os.path.isdir(bin): - filtered_path.append(os.path.realpath(bin)) - - # Once the paths are cleaned up, do a search for each type of - # compiler. We can spawn a bunch of parallel searches to reduce - # the overhead of spelunking all these directories. - types = all_compiler_types() - compiler_lists = parmap(lambda cls: cls.find(*filtered_path), types) - - # ensure all the version calls we made are cached in the parent - # process, as well. This speeds up Spack a lot. - clist = reduce(lambda x,y: x+y, compiler_lists) - return clist + # Find compilers for each operating system class + oss = all_os_classes() + compiler_lists = [] + for os in oss: + compiler_lists.extend(os.find_compilers()) + return compiler_lists def supported_compilers(): """Return a set of names of compilers supported by Spack. @@ -237,47 +211,60 @@ def supported(compiler_spec): @_auto_compiler_spec -def find(compiler_spec, arch=None, scope=None): +def find(compiler_spec, scope=None): """Return specs of available compilers that match the supplied compiler spec. Return an list if nothing found.""" - return [c for c in all_compilers(arch, scope) if c.satisfies(compiler_spec)] + return [c for c in all_compilers(scope) if c.satisfies(compiler_spec)] @_auto_compiler_spec -def compilers_for_spec(compiler_spec, arch=None, scope=None): +def compilers_for_spec(compiler_spec, scope=None): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ - config = all_compilers_config(arch, scope) - - def get_compiler(cspec): - items = config[str(cspec)] - - if not all(n in items for n in _required_instance_vars): - raise InvalidCompilerConfigurationError(cspec) - - cls = class_for_compiler_name(cspec.name) - - strategy = items['strategy'] - if not strategy: - raise InvalidCompilerConfigurationError(cspec) - - compiler_paths = [] - for c in _required_instance_vars: - compiler_path = items[c] - if compiler_path != "None": - compiler_paths.append(compiler_path) + config = all_compilers_config(scope) + + def get_compilers(cspec): + compilers = [] + + for aka, cmp in config.items(): + if cmp['spec'] != str(cspec): + continue + items = cmp + alias = aka + + if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)): + raise InvalidCompilerConfigurationError(cspec) + + cls = class_for_compiler_name(cspec.name) + + compiler_paths = [] + for c in _path_instance_vars: + compiler_path = items['paths'][c] + if compiler_path != "None": + compiler_paths.append(compiler_path) + else: + compiler_paths.append(None) + + mods = items.get('modules') + if mods == 'None': + mods = [] + + if 'operating_system' in items: + operating_system = spack.architecture._operating_system_from_dict( items['operating_system'] ) else: - compiler_paths.append(None) + operating_system = None - if 'modules' not in items: - items['modules'] = None - mods = items['modules'] + compilers.append(cls(cspec, operating_system, compiler_paths, mods, alias)) - return cls(cspec, strategy, compiler_paths, mods) + return compilers - matches = find(compiler_spec, arch, scope) - return [get_compiler(cspec) for cspec in matches] + matches = set(find(compiler_spec, scope)) + compilers = [] + for cspec in matches: + compilers.extend(get_compilers(cspec)) + return compilers +# return [get_compilers(cspec) for cspec in matches] @_auto_compiler_spec @@ -285,8 +272,9 @@ def compiler_for_spec(compiler_spec, operating_system): """Get the compiler that satisfies compiler_spec. compiler_spec must be concrete.""" assert(compiler_spec.concrete) + compilers = [c for c in compilers_for_spec(compiler_spec) - if c.strategy == operating_system.compiler_strategy] + if c.operating_system == operating_system] if len(compilers) < 1: raise NoCompilerForSpecError(compiler_spec, operating_system) if len(compilers) > 1: @@ -308,8 +296,20 @@ def class_for_compiler_name(compiler_name): return cls +def all_os_classes(): + """ + Return the list of classes for all operating systems available on + this platform + """ + classes = [] + + platform = spack.architecture.sys_type() + for os_class in platform.operating_sys.values(): + classes.append(os_class) + + return classes + def all_compiler_types(): -# return [class_for_compiler_name(c) for c in ['gcc']] return [class_for_compiler_name(c) for c in supported_compilers()] @@ -318,7 +318,7 @@ class InvalidCompilerConfigurationError(spack.error.SpackError): super(InvalidCompilerConfigurationError, self).__init__( "Invalid configuration for [compiler \"%s\"]: " % compiler_spec, "Compiler configuration must contain entries for all compilers: %s" - % _required_instance_vars) + % _path_instance_vars) class NoCompilersError(spack.error.SpackError): diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 3422e86ae8..58ea83ac4a 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -301,15 +301,11 @@ class DefaultConcretizer(object): # Should think whether this can be more efficient def _proper_compiler_style(cspec, architecture): compilers = spack.compilers.compilers_for_spec(cspec) - filter(lambda c: c.strategy == architecture.platform_os.compiler_strategy, compilers) -#if architecture.platform_os.compiler_strategy == 'PATH': - # filter(lambda c: not c.modules, compilers) - #if architecture.platform_os.compiler_strategy == 'MODULES': - # filter(lambda c: c.modules, compilers) + filter(lambda c: c.operating_system == architecture.platform_os, compilers) return compilers - all_compilers = spack.compilers.all_compilers(spec.architecture) + all_compilers = spack.compilers.all_compilers() if (spec.compiler and spec.compiler.concrete and diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index f978ed12b0..49f2b90bfc 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -151,17 +151,16 @@ section_schemas = { 'default': {}, 'additionalProperties': False, 'patternProperties': { - r'\w[\w-]*': { # architecture + r'\w[\w-]*': { # alias 'type': 'object', 'additionalProperties': False, - 'patternProperties': { - r'\w[\w-]*@\w[\w-]*': { # compiler spec + 'required': ['paths', 'spec', 'modules', 'operating_system'], + 'properties': { + 'paths': { 'type': 'object', - 'additionalProperties': False, 'required': ['cc', 'cxx', 'f77', 'fc'], - 'properties': { - 'strategy': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]}, + 'additionalProperties': False, + 'properties': { 'cc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, 'cxx': { 'anyOf': [ {'type' : 'string' }, @@ -169,13 +168,21 @@ section_schemas = { 'f77': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, 'fc': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]}, - 'modules': { 'anyOf': [ {'type' : 'string'}, - {'type' : 'null' }, - {'type': 'array'}, - ]} - },},},},},},},}, - + {'type' : 'null' }]}}}, + 'spec': { 'type': 'string'},#r'\w[\w-]*@\w[\w-]*' + 'operating_system': { + 'type': 'object', + 'required': ['name', 'version'], + 'additionalProperties': False, + 'properties': { + 'name': {'type': 'string'}, + 'version': {'type': 'string'} + }}, + 'modules': { 'anyOf': [ {'type' : 'string'}, + {'type' : 'null' }, + {'type': 'array'}, + ]} + },},},},},}, 'mirrors': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack mirror configuration file schema', diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py index 9a0bf6c194..c160a60be8 100644 --- a/lib/spack/spack/operating_systems/cnl.py +++ b/lib/spack/spack/operating_systems/cnl.py @@ -1,4 +1,11 @@ +import re +import os + from spack.architecture import OperatingSystem +from spack.util.executable import * +import spack.spec +from spack.util.multiproc import parmap +import spack.compilers class Cnl(OperatingSystem): """ Compute Node Linux (CNL) is the operating system used for the Cray XC @@ -10,4 +17,46 @@ class Cnl(OperatingSystem): def __init__(self): name = 'CNL' version = '10' - super(Cnl, self).__init__(name, version, "MODULES") + super(Cnl, self).__init__(name, version) + + + def find_compilers(self, *paths): + types = spack.compilers.all_compiler_types() + compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types) + + # ensure all the version calls we made are cached in the parent + # process, as well. This speeds up Spack a lot. + clist = reduce(lambda x,y: x+y, compiler_lists) + return clist + + + def find_compiler(self, cmp_cls, *paths): + compilers = [] + if cmp_cls.PrgEnv: + if not cmp_cls.PrgEnv_compiler: + tty.die('Must supply PrgEnv_compiler with PrgEnv') + + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Save the environment variable to restore later + old_modulepath = os.environ['MODULEPATH'] + # if given any explicit paths, search them for module files too + if paths: + module_paths = ':' + ':'.join(p for p in paths) + os.environ['MODULEPATH'] = module_paths + + output = modulecmd('avail', cmp_cls.PrgEnv_compiler, output=str, error=str) + matches = re.findall(r'(%s)/([\d\.]+[\d])' % cmp_cls.PrgEnv_compiler, output) + for name, version in matches: + v = version + comp = cmp_cls(spack.spec.CompilerSpec(name + '@' + v), self, + ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name +'/' + v]) + + compilers.append(comp) + + # Restore modulepath environment variable + if paths: + os.environ['MODULEPATH'] = old_modulepath + + return compilers diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 1ca3e58831..4cba2da9a5 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -12,8 +12,18 @@ from spack.platforms.linux import Linux from spack.platforms.bgq import Bgq from spack.platforms.darwin import Darwin -class ArchitectureTest(unittest.TestCase): +from spack.test.mock_packages_test import * +#class ArchitectureTest(unittest.TestCase): +class ArchitectureTest(MockPackagesTest): + + def setUp(self): + super(ArchitectureTest, self).setUp() + self.platform = sys_type() + + def tearDown(self): + super(ArchitectureTest, self).tearDown() + def test_dict_functions_for_architecture(self): arch = Arch() arch.platform_os = arch.platform.operating_system('default_os') @@ -34,13 +44,13 @@ class ArchitectureTest(unittest.TestCase): self.assertTrue( isinstance(new_arch.target, Target) ) - def test_platform_class_and_compiler_strategies(self): - a = CrayXc() - t = a.operating_system('default_os') - self.assertEquals(t.compiler_strategy, 'MODULES') - b = Linux() - s = b.operating_system('default_os') - self.assertEquals(s.compiler_strategy, 'PATH') +# def test_platform_class_and_compiler_strategies(self): +# a = CrayXc() +# t = a.operating_system('default_os') +# self.assertEquals(t.compiler_strategy, 'MODULES') +# b = Linux() +# s = b.operating_system('default_os') +# self.assertEquals(s.compiler_strategy, 'PATH') def test_sys_type(self): output_platform_class = sys_type() @@ -56,16 +66,13 @@ class ArchitectureTest(unittest.TestCase): self.assertEqual(str(output_platform_class), str(my_platform_class)) - def setUp(self): - self.platform = sys_type() - def test_user_front_end_input(self): """Test when user inputs just frontend that both the frontend target and frontend operating system match """ frontend_os = self.platform.operating_system("frontend") frontend_target = self.platform.target("frontend") - frontend_spec = Spec("zlib=frontend") + frontend_spec = Spec("libelf=frontend") frontend_spec.concretize() self.assertEqual(frontend_os, frontend_spec.architecture.platform_os) self.assertEqual(frontend_target, frontend_spec.architecture.target) @@ -76,7 +83,7 @@ class ArchitectureTest(unittest.TestCase): """ backend_os = self.platform.operating_system("backend") backend_target = self.platform.target("backend") - backend_spec = Spec("zlib=backend") + backend_spec = Spec("libelf=backend") backend_spec.concretize() self.assertEqual(backend_os, backend_spec.architecture.platform_os) self.assertEqual(backend_target, backend_spec.architecture.target) @@ -85,7 +92,7 @@ class ArchitectureTest(unittest.TestCase): default_os = self.platform.operating_system("default_os") default_target = self.platform.target("default_target") - default_spec = Spec("zlib") # default is no args + default_spec = Spec("libelf") # default is no args default_spec.concretize() self.assertEqual(default_os, default_spec.architecture.platform_os) self.assertEqual(default_target, default_spec.architecture.target) @@ -103,11 +110,11 @@ class ArchitectureTest(unittest.TestCase): for arch in combinations: o,t = arch arch_spec = "-".join(arch) - spec = Spec("zlib=%s" % arch_spec) + spec = Spec("libelf=%s" % arch_spec) spec.concretize() results.append(spec.architecture.platform_os == self.platform.operating_system(o)) results.append(spec.architecture.target == self.platform.target(t)) res = all(results) - print res + self.assertTrue(res) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 0562d2d620..2a8d642584 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -33,43 +33,91 @@ from spack.test.mock_packages_test import * # Some sample compiler config data a_comps = { - "all": { - "gcc@4.7.3" : { + 'gcc473': { + 'paths': { "cc" : "/gcc473", "cxx": "/g++473", "f77": None, - "fc" : None }, - "gcc@4.5.0" : { + "fc" : None + }, + 'modules': None, + 'spec': 'gcc@4.7.3', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + }, + 'gcc450': { + 'paths': { "cc" : "/gcc450", "cxx": "/g++450", - "f77": "/gfortran", - "fc" : "/gfortran" }, - "clang@3.3" : { + "f77": 'gfortran', + "fc" : 'gfortran' + }, + 'modules': None, + 'spec': 'gcc@4.5.0', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + }, + 'clang33': { + 'paths': { "cc" : "", "cxx": "", - "f77": "", - "fc" : "" } - } + "f77": '', + "fc" : '' }, + 'modules': None, + 'spec': 'clang@3.3', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + } } b_comps = { - "all": { - "icc@10.0" : { + 'icc100': { + 'paths': { "cc" : "/icc100", - "cxx": "/icc100", + "cxx": "/icp100", "f77": None, - "fc" : None }, - "icc@11.1" : { + "fc" : None + }, + 'modules': None, + 'spec': 'icc@10.0', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + }, + 'icc111': { + 'paths': { "cc" : "/icc111", "cxx": "/icp111", - "f77": "/ifort", - "fc" : "/ifort" }, - "clang@3.3" : { - "cc" : "/clang", - "cxx": "/clang++", - "f77": None, - "fc" : None} - } + "f77": 'ifort', + "fc" : 'ifort' + }, + 'modules': None, + 'spec': 'icc@11.1', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + }, + 'clang33': { + 'paths': { + "cc" : "", + "cxx": "", + "f77": '', + "fc" : '' }, + 'modules': None, + 'spec': 'clang@3.3', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + } } class ConfigTest(MockPackagesTest): @@ -96,7 +144,6 @@ class ConfigTest(MockPackagesTest): actual = config['all'][key][c] self.assertEqual(expected, actual) - def test_write_key_in_memory(self): # Write b_comps "on top of" a_comps. spack.config.update_config('compilers', a_comps, 'test_low_priority') diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 8f04076f19..612008ac58 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -36,21 +36,50 @@ from spack.spec import Spec mock_compiler_config = """\ compilers: - all: - clang@3.3: + clang3.3CNL: + spec: clang@3.3 + operating_system: + name: CNL + version: '10' + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' + clang3.3RHL: + spec: clang@3.3 + operating_system: + name: redhat + version: '6.7' + paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None - strategy: PATH - modules: None - gcc@4.5.0: + modules: 'None' + gcc4.5.0CNL: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: + name: CNL + version: '10' + spec: gcc@4.5.0 + modules: 'None' + gcc4.5.0RHL: + paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - strategy: PATH - modules: None + operating_system: + name: RHL + version: '6.7' + spec: gcc@4.5.0 + modules: 'None' """ mock_packages_config = """\ diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index eee65c5c09..4ed7de9c7d 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -243,7 +243,7 @@ class SpecDagTest(MockPackagesTest): if len(platform.targets) > 1: first = platform.targets.values()[0].name second = platform.targets.values()[1].name - set_pkg_dep('mpileaks', 'mpich='+first) + self.set_pkg_dep('mpileaks', 'mpich='+first) spec = Spec('mpileaks ^mpich='+ second +' ^callpath ^dyninst ^libelf ^libdwarf') self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) -- cgit v1.2.3-60-g2f50 From 4c9dd028d2305360329767831e65bafe403c6e8a Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 12 May 2016 17:20:48 -0700 Subject: Fixed error causing hang in env/cc --- lib/spack/env/cc | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 4a3e6eddc9..91916c4532 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -153,6 +153,29 @@ fi # Save original command for debug logging input_command="$@" +# +# Filter '.' and Spack environment directories out of PATH so that +# this script doesn't just call itself +# +IFS=':' read -ra env_path <<< "$PATH" +IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" +spack_env_dirs+=(".") +PATH="" +for dir in "${env_path[@]}"; do + remove="" + for rm_dir in "${spack_env_dirs[@]}"; do + if [ "$dir" = "$rm_dir" ]; then remove=True; fi + done + if [ -z "$remove" ]; then + if [ -z "$PATH" ]; then + PATH="$dir" + else + PATH="$PATH:$dir" + fi + fi +done +export PATH + if [ "$mode" == vcheck ] ; then exec ${command} "$@" fi @@ -317,29 +340,6 @@ unset LD_LIBRARY_PATH unset LD_RUN_PATH unset DYLD_LIBRARY_PATH -# -# Filter '.' and Spack environment directories out of PATH so that -# this script doesn't just call itself -# -IFS=':' read -ra env_path <<< "$PATH" -IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" -spack_env_dirs+=(".") -PATH="" -for dir in "${env_path[@]}"; do - remove="" - for rm_dir in "${spack_env_dirs[@]}"; do - if [ "$dir" = "$rm_dir" ]; then remove=True; fi - done - if [ -z "$remove" ]; then - if [ -z "$PATH" ]; then - PATH="$dir" - else - PATH="$PATH:$dir" - fi - fi -done -export PATH - full_command=("$command") full_command+=("${args[@]}") -- cgit v1.2.3-60-g2f50 From f43cab0951e5023736d3e332d747e0db212acb72 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 13 May 2016 14:39:20 -0700 Subject: fixed pkgconfig issue --- lib/spack/spack/build_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 82d0792e26..8016bc4151 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -232,7 +232,7 @@ def set_build_environment_variables(pkg): pcdir = join_path(p, libdir, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) - path_set("PKG_CONFIG_PATH", pkg_config_dirs) + path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) if pkg.spec.architecture.target.module_name: load_module(pkg.spec.architecture.target.module_name) -- cgit v1.2.3-60-g2f50 From fce6ecb05c2aa8f025c27f4245994efef2987b38 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 May 2016 15:04:06 -0700 Subject: Changed _proper_compiler_spec to return filtered list. Apparently filter returns a list rather than filtering the original list. Lead to compiler finding problems if user specified frontend compilers --- lib/spack/spack/concretize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 58ea83ac4a..9ad5271542 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -301,8 +301,9 @@ class DefaultConcretizer(object): # Should think whether this can be more efficient def _proper_compiler_style(cspec, architecture): compilers = spack.compilers.compilers_for_spec(cspec) - filter(lambda c: c.operating_system == architecture.platform_os, compilers) - return compilers + return filter(lambda c: c.operating_system == + architecture.platform_os, compilers) + #return compilers all_compilers = spack.compilers.all_compilers() -- cgit v1.2.3-60-g2f50 From 7705603f7302f7e7c1076d3aff53cb1829fb8e74 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 16 May 2016 09:43:46 -0700 Subject: Added more compilers to mock packages tests --- lib/spack/spack/test/mock_packages_test.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 612008ac58..81b56daacd 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -47,6 +47,17 @@ compilers: f77: None fc: None modules: 'None' + clang3.3SUSE: + spec: clang@3.3 + operating_system: + name: SuSE + version: '11' + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' clang3.3RHL: spec: clang@3.3 operating_system: @@ -69,6 +80,17 @@ compilers: version: '10' spec: gcc@4.5.0 modules: 'None' + gcc4.5.0SUSE: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: + name: SuSE + version: '11' + spec: gcc@4.5.0 + modules: 'None' gcc4.5.0RHL: paths: cc: /path/to/gcc -- cgit v1.2.3-60-g2f50 From 0fd9cdb861b9b794cc6aa20cd5ecca1989a43820 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 16 May 2016 09:56:07 -0700 Subject: Fixed config tests for new compiler config format --- lib/spack/spack/test/config.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 2a8d642584..daa4c572bd 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -138,11 +138,17 @@ class ConfigTest(MockPackagesTest): """Check that named compilers in comps match Spack's config.""" config = spack.config.get_config('compilers') compiler_list = ['cc', 'cxx', 'f77', 'fc'] - for key in compiler_names: - for c in compiler_list: - expected = comps['all'][key][c] - actual = config['all'][key][c] - self.assertEqual(expected, actual) + param_list = ['modules', 'paths', 'spec', 'operating_system'] + for alias, compiler in config.items(): + if compiler['spec'] in compiler_names: + for p in param_list: + expected = comps[alias][p] + actual = config[alias][p] + self.assertEqual(expected, actual) + for c in compiler_list: + expected = comps[alias]['paths'][c] + actual = config[alias]['paths'][c] + self.assertEqual(expected, actual) def test_write_key_in_memory(self): # Write b_comps "on top of" a_comps. -- cgit v1.2.3-60-g2f50 From 80310a3b7c09c8b59b3414b0c20aed6c36620675 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 16:50:50 -0700 Subject: Finished merge on these files --- lib/spack/env/cc | 44 +++++++++---------- lib/spack/spack/compilers/pgi.py | 13 +++--- lib/spack/spack/config.py | 46 +++++++++++-------- lib/spack/spack/spec.py | 95 ++++++++++++++++++++-------------------- 4 files changed, 104 insertions(+), 94 deletions(-) (limited to 'lib') diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 9758b74f37..1e405ae6e9 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -174,6 +174,28 @@ if [[ -z $command ]]; then die "ERROR: Compiler '$SPACK_COMPILER_SPEC' does not support compiling $language programs." fi +# +# Filter '.' and Spack environment directories out of PATH so that +# this script doesn't just call itself +# +IFS=':' read -ra env_path <<< "$PATH" +IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" +spack_env_dirs+=("" ".") +PATH="" +for dir in "${env_path[@]}"; do + addpath=true + for env_dir in "${spack_env_dirs[@]}"; do + if [[ $dir == $env_dir ]]; then + addpath=false + break + fi + done + if $addpath; then + PATH="${PATH:+$PATH:}$dir" + fi +done +export PATH + if [[ $mode == vcheck ]]; then exec ${command} "$@" fi @@ -286,28 +308,6 @@ unset LD_LIBRARY_PATH unset LD_RUN_PATH unset DYLD_LIBRARY_PATH -# -# Filter '.' and Spack environment directories out of PATH so that -# this script doesn't just call itself -# -IFS=':' read -ra env_path <<< "$PATH" -IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" -spack_env_dirs+=("" ".") -PATH="" -for dir in "${env_path[@]}"; do - addpath=true - for env_dir in "${spack_env_dirs[@]}"; do - if [[ $dir == $env_dir ]]; then - addpath=false - break - fi - done - if $addpath; then - PATH="${PATH:+$PATH:}$dir" - fi -done -export PATH - full_command=("$command" "${args[@]}") # In test command mode, write out full command for Spack tests. diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index d3fad15b0c..6d36d8bfa6 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -44,7 +44,12 @@ class Pgi(Compiler): 'f77' : 'pgi/pgfortran', 'fc' : 'pgi/pgfortran' } -#ifdef NEW + + + PrgEnv = 'PrgEnv-pgi' + PrgEnv_compiler = 'pgi' + + @property def openmp_flag(self): return "-mp" @@ -53,12 +58,6 @@ class Pgi(Compiler): def cxx11_flag(self): return "-std=c++11" - -#else /* not NEW */ - PrgEnv = 'PrgEnv-pgi' - PrgEnv_compiler = 'pgi' - -#endif /* not NEW */ @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3d6c175c7c..ec04c81787 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -167,6 +167,18 @@ section_schemas = { 'f77': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, 'fc': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cxxflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'fflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cppflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'ldflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'ldlibs': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}}}, 'spec': { 'type': 'string'},#r'\w[\w-]*@\w[\w-]*' 'operating_system': { @@ -234,15 +246,11 @@ section_schemas = { 'items' : { 'type' : 'string' } }, #compiler specs 'buildable': { 'type': 'boolean', -#ifdef NEW 'default': True, -#else /* not NEW */ - 'default': False, }, - 'module': { - 'anyOf' : [{'type': 'string'}, - {'type': 'null'}] -#endif /* not NEW */ + 'modules': { + 'type' : 'object', + 'default' : {}, }, 'providers': { 'type': 'object', @@ -687,7 +695,8 @@ def spec_externals(spec): external_specs = [] pkg_paths = allpkgs.get(name, {}).get('paths', None) - if not pkg_paths: + pkg_modules = allpkgs.get(name, {}).get('modules', None) + if (not pkg_paths) and (not pkg_modules): return [] for external_spec, path in pkg_paths.iteritems(): @@ -695,20 +704,21 @@ def spec_externals(spec): # skip entries without paths (avoid creating extra Specs) continue -#ifdef NEW external_spec = spack.spec.Spec(external_spec, external=path) if external_spec.satisfies(spec): external_specs.append(external_spec) + + for external_spec, module in pkg_modules.iteritems(): + if not module: + continue + + path = get_path_from_module(module) + + external_spec = spack.spec.Spec(external_spec, external=path, external_module=module) + if external_spec.satisfies(spec): + external_specs.append(external_spec) + return external_specs -#else /* not NEW */ - module = allpkgs.get(pkg, {}).get('module', None) - if not path: - if not module: - continue - path = get_path_from_module(module) - spec_locations.append( (spack.spec.Spec(pkg), path, module) ) - return spec_locations -#endif /* not NEW */ def is_spec_buildable(spec): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index f71d56435e..e61ffc0912 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -497,14 +497,10 @@ class Spec(object): # package.py files for. self._normal = kwargs.get('normal', False) self._concrete = kwargs.get('concrete', False) -#ifdef NEW # Allow a spec to be constructed with an external path. self.external = kwargs.get('external', None) -#else /* not NEW */ - self.external = None - self.external_module = None -#endif /* not NEW */ + self.external_module = kwargs.get('external_module', None) # This allows users to construct a spec DAG with literals. # Note that given two specs a and b, Spec(a) copies a, but @@ -538,8 +534,10 @@ class Spec(object): Known flags currently include "arch" """ valid_flags = FlagMap.valid_compiler_flags() - if name == 'arch': - self._set_architecture(value) + if name == 'os' or name == 'operating_system': + self._set_os(value) + elif name == 'target': + self._set_target(value) elif name in valid_flags: assert(self.compiler_flags is not None) self.compiler_flags[name] = value.split() @@ -553,12 +551,13 @@ class Spec(object): self.compiler = compiler - def _set_architecture(self, architecture): - """Called by the parser to set the architecture.""" - if self.architecture: raise DuplicateArchitectureError( - "Spec for '%s' cannot have two architectures." % self.name) - self.architecture = architecture + def _set_os(self, value): + """Called by the parser to set the architecture operating system""" + self.architecture.platform_os = self.architecture.platform.operating_system(value) + def _set_target(self, value): + """Called by the parser to set the architecture target""" + self.architecture.target = self.architecture.platform.target(value) def _add_dependency(self, spec): """Called by the parser to add another spec as a dependency.""" @@ -793,9 +792,9 @@ class Spec(object): if self.architecture: # TODO: Fix the target.to_dict to account for the tuple # Want it to be a dict of dicts - d['architecture'] = self.architecture.to_dict() + d['arch'] = self.architecture.to_dict() else: - d['architecture'] = None + d['arch'] = None if self.compiler: d.update(self.compiler.to_dict()) @@ -824,17 +823,12 @@ class Spec(object): spec = Spec(name) spec.namespace = node.get('namespace', None) spec.versions = VersionList.from_dict(node) -#ifdef NEW - spec.architecture = node['arch'] if 'hash' in node: spec._hash = node['hash'] -#else /* not NEW */ - # TODO: Need to fix the architecture.Target.from_dict - spec.architecture = spack.architecture.arch_from_dict(node['architecture']) + spec.architecture = spack.architecture.arch_from_dict(node['arch']) -#endif /* not NEW */ if node['compiler'] is None: spec.compiler = None else: @@ -1423,9 +1417,19 @@ class Spec(object): # TODO: Check out the logic here if self.architecture is not None and other.architecture is not None: - if self.architecture != other.architecture: - raise UnsatisfiableArchitectureSpecError(self.architecture, - other.architecture) + if self.architecture.platform is not None and other.architecture.platform is not None: + if self.architecture.platform != other.architecture.platform: + raise UnsatisfiableArchitectureSpecError(self.architecture, + other.architecture) + if self.architecture.platform_os is not None and other.architecture.platform_os is not None: + if self.architecture.platform_os != other.architecture.platform_os: + raise UnsatisfiableArchitectureSpecError(self.architecture, + other.architecture) + if self.architecture.target is not None and other.architecture.target is not None: + if self.architecture.target != other.architecture.target: + raise UnsatisfiableArchitectureSpecError(self.architecture, + other.architecture) + changed = False if self.compiler is not None and other.compiler is not None: @@ -1440,7 +1444,14 @@ class Spec(object): changed |= self.compiler_flags.constrain(other.compiler_flags) old = self.architecture - self.architecture = self.architecture or other.architecture + if self.architecture is None or other.architecture is None: + self.architecture = self.architecture or other.architecture + elif self.architecture.platform is None or other.architecture.platform is None: + self.architecture.platform = self.architecture.platform or other.architecture.platform + elif self.architecture.platform_os is None of other.architecture.platform_os is None: + self.architecture.platform_os = self.architecture.platform_os or other.architecture.platform_os + elif self.architecture.target is None or other.architecture.target is None: + self.architecture.target = self.architecture.target or other.architecture.target changed |= (self.architecture != old) if deps: @@ -1572,16 +1583,18 @@ class Spec(object): # Architecture satisfaction is currently just string equality. # If not strict, None means unconstrained. - if isinstance(self.architecture, basestring): - self.add_architecture_from_string(self.architecture) - if isinstance(other.architecture, basestring): - other.add_architecture_from_string(other.architecture) + # TODO: Need to make sure that comparisons can be made via classes if self.architecture and other.architecture: - if self.architecture != other.architecture: + if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or + (self.architecture.platform_os and other.architecture.platform_os and self.architecture.platform_os != other.architecture.platform_os) or + (self.architecture.target and other.architecture.target and self.architecture.target != other.architecture.target)): return False - elif strict and (other.architecture and not self.architecture): + elif strict and ((other.architecture and not self.architecture) or + (other.architecture.platform and not self.architecture.platform) or + (other.architecture.platform_os and not self.architecture.platform_os) or + (other.architecture.target and not self.architecture.target)): return False if not self.compiler_flags.satisfies(other.compiler_flags, strict=strict): @@ -1663,7 +1676,7 @@ class Spec(object): self.architecture != other.architecture and self.compiler != other.compiler and \ self.variants != other.variants and self._normal != other._normal and \ self.concrete != other.concrete and self.external != other.external and \ - self.external_module != other.external_module) + self.external_module != other.external_module and self.compiler_flags != other.compiler_flags) # Local node attributes get copied first. self.name = other.name @@ -1677,12 +1690,9 @@ class Spec(object): self.variants = other.variants.copy() self.variants.spec = self self.external = other.external + self.external_module = other.external_module self.namespace = other.namespace -#ifdef NEW self._hash = other._hash -#else /* not NEW */ - self.external_module = other.external_module -#endif /* not NEW */ # If we copy dependencies, preserve DAG structure in the new spec if kwargs.get('deps', True): @@ -1983,7 +1993,7 @@ class Spec(object): write(fmt % str(self.variants), '+') elif named_str == 'ARCHITECTURE': if self.architecture: - write(fmt % str(self.architecture), '=') + write(fmt % str(self.architecture), ' arch=') elif named_str == 'SHA1': if self.dependencies: out.write(fmt % str(self.dag_hash(7))) @@ -2202,14 +2212,11 @@ class SpecParser(spack.parse.Parser): spec.name = spec_name spec.versions = VersionList() spec.variants = VariantMap(spec) - spec.architecture = None + spec.architecture = spack.architecture.Arch() spec.compiler = None spec.external = None -#ifdef NEW - spec.compiler_flags = FlagMap(spec) -#else /* not NEW */ spec.external_module = None -#endif /* not NEW */ + spec.compiler_flags = FlagMap(spec) spec.dependents = DependencyMap() spec.dependencies = DependencyMap() spec.namespace = spec_namespace @@ -2284,12 +2291,6 @@ class SpecParser(spack.parse.Parser): self.check_identifier() return self.token.value - def architecture(self): - #TODO: Make this work properly as a subcase of variant (includes adding names to grammar) - self.expect(ID) - return self.token.value - - def version(self): start = None end = None -- cgit v1.2.3-60-g2f50 From faa0f2a13c01a9f279d59912cff5d00c499cb990 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 18 May 2016 17:01:59 -0700 Subject: got rid of ifdef --- lib/spack/spack/cmd/find.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index c768faea91..8b91f45cfd 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -239,8 +239,5 @@ def find(parser, args): mode=args.mode, long=args.long, very_long=args.very_long, -#ifdef NEW show_flags=args.show_flags) -#else /* not NEW */ namespace=args.namespace) -#endif /* not NEW */ -- cgit v1.2.3-60-g2f50 From 9f8ff32bcce99c8a188f285e235e9c32ee245f97 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 18 May 2016 17:05:11 -0700 Subject: Got rid of ifdefs, changed parameters for Compiler: added kwargs --- lib/spack/spack/compiler.py | 65 +++++++++++---------------------------------- 1 file changed, 15 insertions(+), 50 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index dfae72010a..ce4555bc56 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -101,7 +101,6 @@ class Compiler(object): def cxx_rpath_arg(self): return '-Wl,-rpath,' -#ifdef NEW @property def f77_rpath_arg(self): return '-Wl,-rpath,' @@ -109,32 +108,32 @@ class Compiler(object): @property def fc_rpath_arg(self): return '-Wl,-rpath,' -#else /* not NEW */ # Cray PrgEnv name that can be used to load this compiler PrgEnv = None - # Name of module used to switch versions of this compiler PrgEnv_compiler = None -#endif /* not NEW */ - -#ifdef NEW - - def __init__(self, cspec, cc, cxx, f77, fc, **kwargs): -#else /* not NEW */ - def __init__(self, cspec, operating_system, paths, modules=[], alias=None): -#endif /* not NEW */ + def __init__(self, cspec, operating_system, + paths, modules=[], alias=None, **kwargs): def check(exe): if exe is None: return None _verify_executables(exe) return exe -#ifdef NEW - self.cc = check(cc) - self.cxx = check(cxx) - self.f77 = check(f77) - self.fc = check(fc) + self.cc = check(paths[0]) + self.cxx = check(paths[1]) + if len(paths) > 2: + self.f77 = check(paths[2]) + if len(paths) == 3: + self.fc = self.f77 + else: + self.fc = check(paths[3]) + + #self.cc = check(cc) + #self.cxx = check(cxx) + #self.f77 = check(f77) + #self.fc = check(fc) # Unfortunately have to make sure these params are accepted # in the same order they are returned by sorted(flags) @@ -145,19 +144,7 @@ class Compiler(object): if value is not None: self.flags[flag] = value.split() -#else /* not NEW */ self.operating_system = operating_system - - self.cc = check(paths[0]) - self.cxx = check(paths[1]) - if len(paths) > 2: - self.f77 = check(paths[2]) - if len(paths) == 3: - self.fc = self.f77 - else: - self.fc = check(paths[3]) - -#endif /* not NEW */ self.spec = cspec self.modules = modules self.alias = alias @@ -287,28 +274,6 @@ class Compiler(object): successful.reverse() return dict(((v, p, s), path) for v, p, s, path in successful) - @classmethod - def default_version(cls, cc): - """Override just this to override all compiler version functions.""" - return dumpversion(cc) - - @classmethod - def cc_version(cls, cc): - return cls.default_version(cc) - - @classmethod - def cxx_version(cls, cxx): - return cls.default_version(cxx) - - @classmethod - def f77_version(cls, f77): - return cls.default_version(f77) - - @classmethod - def fc_version(cls, fc): - return cls.default_version(fc) - - def __repr__(self): """Return a string representation of the compiler toolchain.""" return self.__str__() -- cgit v1.2.3-60-g2f50 From b4b794223065b3301da9f89eab536bf8d162c150 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 18 May 2016 17:07:23 -0700 Subject: Moved searching of flags into get compilers in compilers_for_spec --- lib/spack/spack/compilers/__init__.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 000ef2058e..3432c142bf 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -254,17 +254,18 @@ def compilers_for_spec(compiler_spec, scope=None): else: operating_system = None - compilers.append(cls(cspec, operating_system, compiler_paths, mods, alias)) - -#ifdef NEW - flags = {} - for f in spack.spec.FlagMap.valid_compiler_flags(): - if f in items: - flags[f] = items[f] - return cls(cspec, *compiler_paths, **flags) -#else /* not NEW */ - return compilers -#endif /* not NEW */ + flags = {} + for f in spack.spec.FlagMap.valid_compiler_flags(): + if f in items: + flags[f] = items[f] + + compilers.append(cls(cspec, operating_system, compiler_paths, mods, alias, **flags)) + +##ifdef NEW +# return cls(cspec, *compiler_paths, **flags) +##else /* not NEW */ +# return compilers +##endif /* not NEW */ matches = set(find(compiler_spec, scope)) compilers = [] -- cgit v1.2.3-60-g2f50 From deb5011d08467c0a19b52e5ee9fefa3fb0854b89 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 17:14:16 -0700 Subject: more merge work --- lib/spack/spack/build_environment.py | 23 ++++++++--------------- lib/spack/spack/spec.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 4974cc1e76..7aeea5c672 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -213,13 +213,12 @@ def set_compiler_environment_variables(pkg): if flags[flag] != []: env.set('SPACK_' + flag.upper(), ' '.join(f for f in flags[flag])) -#ifdef NEW env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler)) - return env -#else /* not NEW */ + for mod in compiler.modules: load_module(mod) -#endif /* not NEW */ + + return env def set_build_environment_variables(pkg, env): @@ -283,16 +282,13 @@ def set_build_environment_variables(pkg, env): pcdir = join_path(p, maybe, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) -#ifdef NEW - env.set_path('PKG_CONFIG_PATH', pkg_config_dirs) - return env -#else /* not NEW */ - path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) + env.prepend_path('PKG_CONFIG_PATH', pkg_config_dirs) if pkg.spec.architecture.target.module_name: load_module(pkg.spec.architecture.target.module_name) -#endif /* not NEW */ + + return env def set_module_variables_for_package(pkg, module): @@ -368,16 +364,13 @@ def set_module_variables_for_package(pkg, module): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" - for spec in pkg.spec.traverse(root=False): - if spec.external_module: - load_module(spec.external_module) - spec.external = get_path_from_module(spec.external_module) - rpaths = [pkg.prefix.lib, pkg.prefix.lib64] rpaths.extend(d.prefix.lib for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib)) rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) + for mod in pkg.spec.compiler.modules: + rpaths.append(get_path_for_module(mod)) return rpaths diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index e61ffc0912..9d217d17ca 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -120,6 +120,7 @@ from spack.version import * from spack.util.string import * from spack.util.prefix import Prefix from spack.virtual import ProviderIndex +from spack.build_environment import get_path_from_module, load_module # Valid pattern for an identifier in Spack identifier_re = r'\w[\w-]*' @@ -1085,6 +1086,15 @@ class Spec(object): if s.namespace is None: s.namespace = spack.repo.repo_for_pkg(s.name).namespace + + for s in self.traverse(root=False): + if spec.external_module: + compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture.platform_os) + for mod in compiler.modules: + load_module(mod) + + spec.external = get_path_from_module(spec.external_module) + # Mark everything in the spec as concrete, as well. self._mark_concrete() -- cgit v1.2.3-60-g2f50 From b1e5eafb80c8cd87f4d496fe7fdb7a74a77da603 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 17:18:26 -0700 Subject: more merge work --- lib/spack/spack/test/config.py | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 7d71cc1c7f..8fffc09437 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -142,19 +142,6 @@ class ConfigTest(MockPackagesTest): """Check that named compilers in comps match Spack's config.""" config = spack.config.get_config('compilers') compiler_list = ['cc', 'cxx', 'f77', 'fc'] -#ifdef NEW - for key in compiler_names: - for c in compiler_list: - expected = comps[arch][key][c] - actual = config[arch][key][c] - self.assertEqual(expected, actual) - - def test_write_list_in_memory(self): - spack.config.update_config('repos', repos_low, 'test_low_priority') - spack.config.update_config('repos', repos_high, 'test_high_priority') - config = spack.config.get_config('repos') - self.assertEqual(config, repos_high+repos_low) -#else /* not NEW */ param_list = ['modules', 'paths', 'spec', 'operating_system'] for alias, compiler in config.items(): if compiler['spec'] in compiler_names: @@ -166,7 +153,12 @@ class ConfigTest(MockPackagesTest): expected = comps[alias]['paths'][c] actual = config[alias]['paths'][c] self.assertEqual(expected, actual) -#endif /* not NEW */ + + def test_write_list_in_memory(self): + spack.config.update_config('repos', repos_low, 'test_low_priority') + spack.config.update_config('repos', repos_high, 'test_high_priority') + config = spack.config.get_config('repos') + self.assertEqual(config, repos_high+repos_low) def test_write_key_in_memory(self): # Write b_comps "on top of" a_comps. @@ -187,9 +179,8 @@ class ConfigTest(MockPackagesTest): spack.config.clear_config_caches() # Same check again, to ensure consistency. -#ifdef NEW - self.check_config(a_comps, 'x86_64_E5v2_IntelIB', 'gcc@4.7.3', 'gcc@4.5.0') - self.check_config(b_comps, 'x86_64_E5v3', 'icc@10.0', 'icc@11.1', 'clang@3.3') + self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') + self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3') def test_write_to_same_priority_file(self): # Write b_comps in the same file as a_comps. @@ -200,9 +191,5 @@ class ConfigTest(MockPackagesTest): spack.config.clear_config_caches() # Same check again, to ensure consistency. - self.check_config(a_comps, 'x86_64_E5v2_IntelIB', 'gcc@4.7.3', 'gcc@4.5.0') - self.check_config(b_comps, 'x86_64_E5v3', 'icc@10.0', 'icc@11.1', 'clang@3.3') -#else /* not NEW */ self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3') -#endif /* not NEW */ -- cgit v1.2.3-60-g2f50 From 5417f1cdc676ab6c244cce1f3594a9620ba24b16 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 17:21:03 -0700 Subject: more merge work --- lib/spack/spack/test/__init__.py | 40 ++----------------------------------- lib/spack/spack/test/multimethod.py | 22 ++------------------ 2 files changed, 4 insertions(+), 58 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index cac7c2f058..480e6290e7 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -31,8 +31,8 @@ from llnl.util.filesystem import join_path from llnl.util.tty.colify import colify from spack.test.tally_plugin import Tally """Names of tests to be included in Spack's test suite""" -#ifdef NEW -test_names = ['versions', 'url_parse', 'url_substitution', 'packages', 'stage', + +test_names = ['architecture', 'versions', 'url_parse', 'url_substitution', 'packages', 'stage', 'spec_syntax', 'spec_semantics', 'spec_dag', 'concretize', 'multimethod', 'install', 'package_sanity', 'config', 'directory_layout', 'pattern', 'python_version', 'git_fetch', @@ -41,42 +41,6 @@ test_names = ['versions', 'url_parse', 'url_substitution', 'packages', 'stage', 'make_executable', 'configure_guess', 'lock', 'database', 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.uninstall', 'cmd.test_install'] -#else /* not NEW */ -test_names = ['architecture', - 'versions', - 'url_parse', - 'url_substitution', - 'packages', - 'stage', - 'spec_syntax', - 'spec_semantics', - 'spec_dag', - 'concretize', - 'multimethod', - 'install', - 'package_sanity', - 'config', - 'directory_layout', - 'pattern', - 'python_version', - 'git_fetch', - 'svn_fetch', - 'hg_fetch', - 'mirror', - 'url_extrapolate', - 'cc', - 'link_tree', - 'spec_yaml', - 'optional_deps', - 'make_executable', - 'configure_guess', - 'unit_install', - 'lock', - 'database', - 'namespace_trie', - 'yaml', - 'sbang'] -#endif /* not NEW */ def list_tests(): diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 7636bd6c38..034e6b3923 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -92,36 +92,18 @@ class MultiMethodTest(MockPackagesTest): self.assertEqual(pkg.has_a_default(), 'default') -#ifdef NEW - def test_architecture_match(self): - pkg = spack.repo.get('multimethod arch=x86_64') - self.assertEqual(pkg.different_by_architecture(), 'x86_64') - - pkg = spack.repo.get('multimethod arch=ppc64') - self.assertEqual(pkg.different_by_architecture(), 'ppc64') - - pkg = spack.repo.get('multimethod arch=ppc32') - self.assertEqual(pkg.different_by_architecture(), 'ppc32') - - pkg = spack.repo.get('multimethod arch=arm64') - self.assertEqual(pkg.different_by_architecture(), 'arm64') - - pkg = spack.repo.get('multimethod arch=macos') - self.assertRaises(NoSuchMethodError, pkg.different_by_architecture) -#else /* not NEW */ def test_target_match(self): platform = spack.architecture.sys_type() targets = platform.targets.values() for target in targets[:-1]: - pkg = spack.repo.get('multimethod='+target.name) + pkg = spack.repo.get('multimethod target='+target.name) self.assertEqual(pkg.different_by_target(), target.name) - pkg = spack.repo.get('multimethod='+targets[-1].name) + pkg = spack.repo.get('multimethod target='+targets[-1].name) if len(targets) == 1: self.assertEqual(pkg.different_by_target(), targets[-1].name) else: self.assertRaises(NoSuchMethodError, pkg.different_by_target) -#endif /* not NEW */ def test_dependency_match(self): -- cgit v1.2.3-60-g2f50 From 6926f4d0da05ee2839e455af32d52098d9828be7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 18 May 2016 17:23:10 -0700 Subject: Changed arch tests to new spec target= os= --- lib/spack/spack/test/spec_semantics.py | 109 +++++++++++++++------------------ 1 file changed, 50 insertions(+), 59 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index b6c9d61293..8cdb91e206 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -23,6 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest +import spack.architecture from spack.spec import * from spack.test.mock_packages_test import * @@ -138,23 +139,31 @@ class SpecSematicsTest(MockPackagesTest): self.check_unsatisfiable('foo %gcc@4.7', '%gcc@4.7.3') -#ifdef NEW def test_satisfies_architecture(self): - self.check_satisfies('foo arch=chaos_5_x86_64_ib', ' arch=chaos_5_x86_64_ib') - self.check_satisfies('foo arch=bgqos_0', ' arch=bgqos_0') + platform = self.architecture.sys_type() + if platform.name == 'crayxc': + self.check_satisfies('foo target=frontend os=frontend', 'target=frontend os=frontend') + self.check_satisfies('foo target=backend os=backend', 'target=backend', 'os=backend') + self.check_satisfies('foo target=default_target os=default_os','target=default_target os=default_os') - self.check_unsatisfiable('foo arch=bgqos_0', ' arch=chaos_5_x86_64_ib') - self.check_unsatisfiable('foo arch=chaos_5_x86_64_ib', ' arch=bgqos_0') -#else /* not NEW */ - def test_satisfies_target(self): - platform = spack.architecture.sys_type() - targets = platform.targets.values() - for target in targets: - self.check_satisfies('foo='+target.name, '='+target.name) -#endif /* not NEW */ - for i in range(1,len(targets)): - self.check_unsatisfiable('foo='+targets[i-1].name, '='+targets[i].name) +#ifdef NEW + #def test_satisfies_architecture(self): + # self.check_satisfies('foo arch=chaos_5_x86_64_ib', ' arch=chaos_5_x86_64_ib') + # self.check_satisfies('foo arch=bgqos_0', ' arch=bgqos_0') + + # self.check_unsatisfiable('foo arch=bgqos_0', ' arch=chaos_5_x86_64_ib') + # self.check_unsatisfiable('foo arch=chaos_5_x86_64_ib', ' arch=bgqos_0') +#els#e /* not NEW */ + #def test_satisfies_target(self): + # platform = spack.architecture.sys_type() + # targets = platform.targets.values() + # for target in targets: + # self.check_satisfies('foo='+target.name, '='+target.name) +#end#if /* not NEW */ + + # for i in range(1,len(targets)): + # self.check_unsatisfiable('foo='+targets[i-1].name, '='+targets[i].name) def test_satisfies_dependencies(self): self.check_satisfies('mpileaks^mpich', '^mpich') @@ -355,22 +364,29 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain('libelf+debug~foo', 'libelf+debug', 'libelf+debug~foo') -#ifdef NEW def test_constrain_compiler_flags(self): self.check_constrain('libelf cflags="-O3" cppflags="-Wall"', 'libelf cflags="-O3"', 'libelf cppflags="-Wall"') self.check_constrain('libelf cflags="-O3" cppflags="-Wall"', 'libelf cflags="-O3"', 'libelf cflags="-O3" cppflags="-Wall"') - def test_constrain_arch(self): - self.check_constrain('libelf arch=bgqos_0', 'libelf arch=bgqos_0', 'libelf arch=bgqos_0') - self.check_constrain('libelf arch=bgqos_0', 'libelf', 'libelf arch=bgqos_0') -#else /* not NEW */ - def test_constrain_target(self): - platform = spack.architecture.sys_type() - target = platform.target('default_target').name - self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) - self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) -#endif /* not NEW */ + def test_constrain_architecture(self): + self.check_constrain('libelf target=default_target os=default_os', + 'libelf target=default_target os=default_os', + 'libelf target=default_target os=default_os') + self.check_constrain('libelf target=default_target os=default_os', + 'libelf', + 'libelf target=default_target os=default_os') + + #def test_constrain_arch(self): + # self.check_constrain('libelf arch=bgqos_0', 'libelf arch=bgqos_0', 'libelf arch=bgqos_0') + # self.check_constrain('libelf arch=bgqos_0', 'libelf', 'libelf arch=bgqos_0') +#els#e /* not NEW */ + #def test_constrain_target(self): + # platform = spack.architecture.sys_type() + # target = platform.target('default_target').name + # self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) + # self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) +#end#if /* not NEW */ def test_constrain_compiler(self): @@ -384,20 +400,11 @@ class SpecSematicsTest(MockPackagesTest): self.check_invalid_constraint('libelf+debug', 'libelf~debug') self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') -#ifdef NEW self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') - - self.check_invalid_constraint('libelf arch=bgqos_0', 'libelf arch=x86_54') -#else /* not NEW */ - - platform = spack.architecture.sys_type() - targets = platform.targets.values() - if len(targets) > 1: - self.check_invalid_constraint('libelf='+targets[0].name, 'libelf='+targets[1].name) -#endif /* not NEW */ - + self.check_invalid_constraint('libelf target=default_target os=default_os', + 'libelf target=x86_64 os=ubuntu') def test_constrain_changed(self): self.check_constrain_changed('libelf', '@1.0') @@ -406,14 +413,13 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_changed('libelf%gcc', '%gcc@4.5') self.check_constrain_changed('libelf', '+debug') self.check_constrain_changed('libelf', '~debug') -#ifdef NEW self.check_constrain_changed('libelf', 'debug=2') self.check_constrain_changed('libelf', 'cppflags="-O3"') self.check_constrain_changed('libelf', ' arch=bgqos_0') -#else /* not NEW */ + platform = spack.architecture.sys_type() - self.check_constrain_changed('libelf', '='+platform.target('default_target').name) -#endif /* not NEW */ + self.check_constrain_changed('libelf', 'target='+platform.target('default_target').name) + self.check_constrain_changed('libelf', 'os='+platform.operating_system('default_os').name) def test_constrain_not_changed(self): @@ -424,17 +430,12 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_not_changed('libelf%gcc@4.5', '%gcc@4.5') self.check_constrain_not_changed('libelf+debug', '+debug') self.check_constrain_not_changed('libelf~debug', '~debug') -#ifdef NEW self.check_constrain_not_changed('libelf debug=2', 'debug=2') self.check_constrain_not_changed('libelf cppflags="-O3"', 'cppflags="-O3"') - self.check_constrain_not_changed('libelf arch=bgqos_0', ' arch=bgqos_0') -#else /* not NEW */ + platform = spack.architecture.sys_type() default_target = platform.target('default_target').name - self.check_constrain_not_changed('libelf='+default_target, '='+default_target) -#endif /* not NEW */ - self.check_constrain_not_changed('libelf^foo', 'libelf^foo') - self.check_constrain_not_changed('libelf^foo^bar', 'libelf^foo^bar') + self.check_constrain_not_changed('libelf target='+default_target, 'target='+default_target) def test_constrain_dependency_changed(self): @@ -444,14 +445,9 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_changed('libelf^foo%gcc', 'libelf^foo%gcc@4.5') self.check_constrain_changed('libelf^foo', 'libelf^foo+debug') self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') -#ifdef NEW - self.check_constrain_changed('libelf^foo', 'libelf^foo cppflags="-O3"') - self.check_constrain_changed('libelf^foo', 'libelf^foo arch=bgqos_0') -#else /* not NEW */ platform = spack.architecture.sys_type() default_target = platform.target('default_target').name - self.check_constrain_changed('libelf^foo', 'libelf^foo='+default_target) -#endif /* not NEW */ + self.check_constrain_changed('libelf^foo', 'libelf^foo target='+default_target) def test_constrain_dependency_not_changed(self): @@ -461,13 +457,8 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_not_changed('libelf^foo%gcc@4.5', 'libelf^foo%gcc@4.5') self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') -#ifdef NEW self.check_constrain_not_changed('libelf^foo cppflags="-O3"', 'libelf^foo cppflags="-O3"') - self.check_constrain_not_changed('libelf^foo arch=bgqos_0', 'libelf^foo arch=bgqos_0') -#else /* not NEW */ platform = spack.architecture.sys_type() default_target = platform.target('default_target').name - self.check_constrain_not_changed('libelf^foo='+default_target, 'libelf^foo='+default_target) - -#endif /* not NEW */ + self.check_constrain_not_changed('libelf^foo target='+default_target, 'libelf^foo target='+default_target) -- cgit v1.2.3-60-g2f50 From b25da51638c6c3abd97a1f77cc4feffd144a62d1 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 18 May 2016 17:26:06 -0700 Subject: Changed unsatisfiable architecture test --- lib/spack/spack/test/spec_dag.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 3c005c4ead..99a986a528 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -29,6 +29,7 @@ You can find the dummy packages here:: spack/lib/spack/spack/test/mock_packages """ import spack +import spack.architecture import spack.package from llnl.util.lang import list_modules @@ -240,21 +241,12 @@ class SpecDagTest(MockPackagesTest): self.assertRaises(spack.spec.UnsatisfiableCompilerSpecError, spec.normalize) -#ifdef NEW def test_unsatisfiable_architecture(self): - self.set_pkg_dep('mpileaks', 'mpich arch=bgqos_0') - spec = Spec('mpileaks ^mpich arch=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') - self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) -#else /* not NEW */ - def test_unsatisfiable_target(self): platform = spack.architecture.sys_type() - if len(platform.targets) > 1: - first = platform.targets.values()[0].name - second = platform.targets.values()[1].name - self.set_pkg_dep('mpileaks', 'mpich='+first) - spec = Spec('mpileaks ^mpich='+ second +' ^callpath ^dyninst ^libelf ^libdwarf') - self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) -#endif /* not NEW */ + + self.set_pkg_dep('mpileaks', 'mpich target=%s' % platform.target('default_target')) + spec = Spec('mpileaks ^mpich target=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') + self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) def test_invalid_dep(self): -- cgit v1.2.3-60-g2f50 From 01d5ffcd87d4801874dc810fb8123447971b867d Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 17:42:11 -0700 Subject: Fixed the first set of merge bugs --- lib/spack/spack/architecture.py | 8 ++++---- lib/spack/spack/cmd/find.py | 2 +- lib/spack/spack/compilers/__init__.py | 6 +----- lib/spack/spack/concretize.py | 11 ++++++----- lib/spack/spack/spec.py | 2 +- lib/spack/spack/test/architecture.py | 7 +++---- 6 files changed, 16 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index d7f908cfb1..41778795c3 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -323,8 +323,8 @@ class Arch(object): target = self.target d['platform'] = self.platform.name - d['platform_os'] = self.platform_os.to_dict() - d['target'] = self.target.to_dict() + d['platform_os'] = self.platform_os.to_dict() if self.platform_os else None + d['target'] = self.target.to_dict() if self.target else None return d @@ -362,8 +362,8 @@ def arch_from_dict(d): os_dict = d['platform_os'] target_dict = d['target'] - target = _target_from_dict(target_dict) - platform_os = _operating_system_from_dict(os_dict) + target = _target_from_dict(target_dict) if os_dict else None + platform_os = _operating_system_from_dict(os_dict) if os_dict else None arch.target = target arch.platform_os = platform_os diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 8b91f45cfd..c2bba13dc8 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -239,5 +239,5 @@ def find(parser, args): mode=args.mode, long=args.long, very_long=args.very_long, - show_flags=args.show_flags) + show_flags=args.show_flags, namespace=args.namespace) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 3432c142bf..8fe7a17116 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -261,11 +261,7 @@ def compilers_for_spec(compiler_spec, scope=None): compilers.append(cls(cspec, operating_system, compiler_paths, mods, alias, **flags)) -##ifdef NEW -# return cls(cspec, *compiler_paths, **flags) -##else /* not NEW */ -# return compilers -##endif /* not NEW */ + return compilers matches = set(find(compiler_spec, scope)) compilers = [] diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 72b184dd8f..f38afd38dc 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -282,11 +282,8 @@ class DefaultConcretizer(object): return False #Find the another spec that has a compiler, or the root if none do -#ifdef NEW other_spec = spec if spec.compiler else find_spec(spec, lambda(x) : x.compiler) -#else /* not NEW */ - other_spec = find_spec(spec, lambda(x) : x.compiler) -#endif /* not NEW */ + if not other_spec: other_spec = spec.root other_compiler = other_spec.compiler @@ -321,6 +318,10 @@ class DefaultConcretizer(object): compiler is used, defaulting to no compiler flags in the spec. Default specs set at the compiler level will still be added later. """ + if not spec.architecture.platform_os: + #Although this usually means changed, this means awaiting other changes + return True + ret = False for flag in spack.spec.FlagMap.valid_compiler_flags(): try: @@ -352,7 +353,7 @@ class DefaultConcretizer(object): # Include the compiler flag defaults from the config files # This ensures that spack will detect conflicts that stem from a change # in default compiler flags. - compiler = spack.compilers.compiler_for_spec(spec.compiler) + compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture.platform_os) for flag in compiler.flags: if flag not in spec.compiler_flags: spec.compiler_flags[flag] = compiler.flags[flag] diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 9d217d17ca..7c04baaa91 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1458,7 +1458,7 @@ class Spec(object): self.architecture = self.architecture or other.architecture elif self.architecture.platform is None or other.architecture.platform is None: self.architecture.platform = self.architecture.platform or other.architecture.platform - elif self.architecture.platform_os is None of other.architecture.platform_os is None: + elif self.architecture.platform_os is None or other.architecture.platform_os is None: self.architecture.platform_os = self.architecture.platform_os or other.architecture.platform_os elif self.architecture.target is None or other.architecture.target is None: self.architecture.target = self.architecture.target or other.architecture.target diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 4cba2da9a5..2927e468a0 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -72,7 +72,7 @@ class ArchitectureTest(MockPackagesTest): """ frontend_os = self.platform.operating_system("frontend") frontend_target = self.platform.target("frontend") - frontend_spec = Spec("libelf=frontend") + frontend_spec = Spec("libelf os=frontend target=frontend") frontend_spec.concretize() self.assertEqual(frontend_os, frontend_spec.architecture.platform_os) self.assertEqual(frontend_target, frontend_spec.architecture.target) @@ -83,7 +83,7 @@ class ArchitectureTest(MockPackagesTest): """ backend_os = self.platform.operating_system("backend") backend_target = self.platform.target("backend") - backend_spec = Spec("libelf=backend") + backend_spec = Spec("libelf os=backend target=backend") backend_spec.concretize() self.assertEqual(backend_os, backend_spec.architecture.platform_os) self.assertEqual(backend_target, backend_spec.architecture.target) @@ -109,8 +109,7 @@ class ArchitectureTest(MockPackagesTest): results = [] for arch in combinations: o,t = arch - arch_spec = "-".join(arch) - spec = Spec("libelf=%s" % arch_spec) + spec = Spec("libelf os=%s target=%s" % (o, t)) spec.concretize() results.append(spec.architecture.platform_os == self.platform.operating_system(o)) results.append(spec.architecture.target == self.platform.target(t)) -- cgit v1.2.3-60-g2f50 From 63459ab0c78512906c35c5d5f9fbd5a072a3203a Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 18:21:52 -0700 Subject: Fixed some of the bugs --- lib/spack/spack/spec.py | 24 ++++++++++++---------- lib/spack/spack/test/spec_semantics.py | 37 +++++++++++++++------------------- 2 files changed, 29 insertions(+), 32 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 7c04baaa91..f88475d1c8 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1088,12 +1088,12 @@ class Spec(object): for s in self.traverse(root=False): - if spec.external_module: - compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture.platform_os) + if s.external_module: + compiler = spack.compilers.compiler_for_spec(s.compiler, s.architecture.platform_os) for mod in compiler.modules: load_module(mod) - spec.external = get_path_from_module(spec.external_module) + s.external = get_path_from_module(s.external_module) # Mark everything in the spec as concrete, as well. self._mark_concrete() @@ -1426,6 +1426,7 @@ class Spec(object): other.variants[v]) # TODO: Check out the logic here + print self.architecture, other.architecture, "^^^^^^^^^^^^^^^^^^^^^^^" if self.architecture is not None and other.architecture is not None: if self.architecture.platform is not None and other.architecture.platform is not None: if self.architecture.platform != other.architecture.platform: @@ -1453,16 +1454,17 @@ class Spec(object): changed |= self.compiler_flags.constrain(other.compiler_flags) - old = self.architecture + old = str(self.architecture) if self.architecture is None or other.architecture is None: self.architecture = self.architecture or other.architecture - elif self.architecture.platform is None or other.architecture.platform is None: - self.architecture.platform = self.architecture.platform or other.architecture.platform - elif self.architecture.platform_os is None or other.architecture.platform_os is None: - self.architecture.platform_os = self.architecture.platform_os or other.architecture.platform_os - elif self.architecture.target is None or other.architecture.target is None: - self.architecture.target = self.architecture.target or other.architecture.target - changed |= (self.architecture != old) + else: + if self.architecture.platform is None or other.architecture.platform is None: + self.architecture.platform = self.architecture.platform or other.architecture.platform + if self.architecture.platform_os is None or other.architecture.platform_os is None: + self.architecture.platform_os = self.architecture.platform_os or other.architecture.platform_os + if self.architecture.target is None or other.architecture.target is None: + self.architecture.target = self.architecture.target or other.architecture.target + changed |= (str(self.architecture) != old) if deps: changed |= self._constrain_dependencies(other) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 8cdb91e206..9bd32a3d10 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -140,7 +140,7 @@ class SpecSematicsTest(MockPackagesTest): def test_satisfies_architecture(self): - platform = self.architecture.sys_type() + platform = spack.architecture.sys_type() if platform.name == 'crayxc': self.check_satisfies('foo target=frontend os=frontend', 'target=frontend os=frontend') self.check_satisfies('foo target=backend os=backend', 'target=backend', 'os=backend') @@ -377,34 +377,28 @@ class SpecSematicsTest(MockPackagesTest): 'libelf', 'libelf target=default_target os=default_os') - #def test_constrain_arch(self): - # self.check_constrain('libelf arch=bgqos_0', 'libelf arch=bgqos_0', 'libelf arch=bgqos_0') - # self.check_constrain('libelf arch=bgqos_0', 'libelf', 'libelf arch=bgqos_0') -#els#e /* not NEW */ - #def test_constrain_target(self): - # platform = spack.architecture.sys_type() - # target = platform.target('default_target').name - # self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) - # self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) -#end#if /* not NEW */ - - def test_constrain_compiler(self): self.check_constrain('libelf %gcc@4.4.7', 'libelf %gcc@4.4.7', 'libelf %gcc@4.4.7') self.check_constrain('libelf %gcc@4.4.7', 'libelf', 'libelf %gcc@4.4.7') def test_invalid_constraint(self): - self.check_invalid_constraint('libelf@0:2.0', 'libelf@2.1:3') - self.check_invalid_constraint('libelf@0:2.5%gcc@4.8:4.9', 'libelf@2.1:3%gcc@4.5:4.7') +# self.check_invalid_constraint('libelf@0:2.0', 'libelf@2.1:3') +# self.check_invalid_constraint('libelf@0:2.5%gcc@4.8:4.9', 'libelf@2.1:3%gcc@4.5:4.7') - self.check_invalid_constraint('libelf+debug', 'libelf~debug') - self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') - self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') +# self.check_invalid_constraint('libelf+debug', 'libelf~debug') +# self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') +# self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') - self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') - self.check_invalid_constraint('libelf target=default_target os=default_os', - 'libelf target=x86_64 os=ubuntu') +# self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') + platform = spack.architecture.sys_type() + if len(platform.operating_sys.keys()) > 1 or len(platform.targets.keys()) > 1: + os1 = platform.operating_sys.keys()[0] + os2 = platform.operating_sys.keys()[-1] + target1 = platform.targets.keys()[0] + target2 = platform.targets.keys()[-1] + self.check_invalid_constraint('libelf target=%s os=%s' % (target1, os1), + 'libelf target=%s os=%s' % (target2, os2)) def test_constrain_changed(self): self.check_constrain_changed('libelf', '@1.0') @@ -447,6 +441,7 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') platform = spack.architecture.sys_type() default_target = platform.target('default_target').name + print default_target self.check_constrain_changed('libelf^foo', 'libelf^foo target='+default_target) -- cgit v1.2.3-60-g2f50 From 19c8a52fe1c25639c47620bc07f0824c59c921f7 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 19 May 2016 09:53:49 -0700 Subject: fixed spec syntax test --- lib/spack/spack/test/spec_syntax.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index c4e4c9cdfe..ae19de177b 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -58,7 +58,7 @@ class SpecSyntaxTest(unittest.TestCase): # ================================================================================ # Parse checks # ================================================================================ - def check_parse(self, expected, spec=None): + def check_parse(self, expected, spec=None, remove_arch=True): """Assert that the provided spec is able to be parsed. If this is called with one argument, it assumes that the string is canonical (i.e., no spaces and ~ instead of - for variants) and that it @@ -70,6 +70,13 @@ class SpecSyntaxTest(unittest.TestCase): if spec is None: spec = expected output = spack.spec.parse(spec) + + # Remove architectures that get added by parser. + if remove_arch: + for spec in output: + for s in spec.traverse(): + s.architecture = None + parsed = (" ".join(str(spec) for spec in output)) self.assertEqual(expected, parsed) -- cgit v1.2.3-60-g2f50 From 36450b9bc9f41ac0f70696352327d8f07d4ee3e3 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 19 May 2016 14:25:50 -0700 Subject: Adding link_paths to craype compilers --- lib/spack/spack/compilers/craype.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/craype.py b/lib/spack/spack/compilers/craype.py index e8ae284f5b..4ba8b110ec 100644 --- a/lib/spack/spack/compilers/craype.py +++ b/lib/spack/spack/compilers/craype.py @@ -47,10 +47,11 @@ class Craype(Compiler): PrgEnv = 'PrgEnv-cray' PrgEnv_compiler = 'craype' -# @property -# def cxx11_flag(self): -# return "-hstd=c++11" - + link_paths = { 'cc' : 'cc', + 'cxx' : 'c++', + 'f77' : 'f77', + 'fc' : 'fc'} + @classmethod def default_version(cls, comp): return get_compiler_version(comp, r'([Vv]ersion).*(\d+(\.\d+)+)') -- cgit v1.2.3-60-g2f50 From 175a042fd332c13200fb30f7f834541c3bbaa6c4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 19 May 2016 16:42:13 -0700 Subject: Changed rpaths method so that it only grabs rpaths using the compiler module name instead of PrgEnv-x module name --- lib/spack/spack/build_environment.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 7aeea5c672..4f4ca26cbc 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -113,7 +113,6 @@ class MakeExecutable(Executable): return super(MakeExecutable, self).__call__(*args, **kwargs) - def load_module(mod): """Takes a module name and removes modules until it is possible to load that module. It then loads the provided module. Depends on the @@ -145,7 +144,6 @@ def get_path_from_module(mod): # Read the module text = modulecmd('show', mod, output=str, error=str).split('\n') - # If it lists its package directory, return that for line in text: if line.find(mod.upper()+'_DIR') >= 0: @@ -166,15 +164,14 @@ def get_path_from_module(mod): # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that for line in text: - if line.find('LD_LIBRARY_PATH') >= 0: + if line.find('LD_LIBRARY_PATH') >= 0: words = line.split() path = words[2] return path[:path.find('/lib')] - # Unable to find module path return None -def set_compiler_environment_variables(pkg): +def set_compiler_environment_variables(pkg, env): assert(pkg.spec.concrete) compiler = pkg.compiler flags = pkg.spec.compiler_flags @@ -276,14 +273,12 @@ def set_build_environment_variables(pkg, env): env.set(SPACK_DEBUG_LOG_DIR, spack.spack_working_dir) # Add any pkgconfig directories to PKG_CONFIG_PATH - pkg_config_dirs = [] - for p in dep_prefixes: - for maybe in ('lib', 'lib64', 'share'): - pcdir = join_path(p, maybe, 'pkgconfig') + for pre in dep_prefixes: + for directory in ('lib', 'lib64', 'share'): + pcdir = join_path(pre, directory, 'pkgconfig') if os.path.isdir(pcdir): - pkg_config_dirs.append(pcdir) - - env.prepend_path('PKG_CONFIG_PATH', pkg_config_dirs) + #pkg_config_dirs.append(pcdir) + env.prepend_path('PKG_CONFIG_PATH',pcdir) if pkg.spec.architecture.target.module_name: load_module(pkg.spec.architecture.target.module_name) @@ -369,8 +364,9 @@ def get_rpaths(pkg): if os.path.isdir(d.prefix.lib)) rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) - for mod in pkg.spec.compiler.modules: - rpaths.append(get_path_for_module(mod)) + # Second module is our compiler mod name. We use that to get rpaths from + # module show output. + rpaths.append(get_path_from_module(pkg.compiler.modules[1])) return rpaths -- cgit v1.2.3-60-g2f50 From c3767d593d04f990b4344540c95e07f172d54c42 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 20 May 2016 15:48:13 -0700 Subject: compiler list is sorted in descending order, in order to get most up to date version we check from the beginning of the list --- lib/spack/spack/concretize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index f38afd38dc..1c451296c6 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -302,10 +302,10 @@ class DefaultConcretizer(object): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - index = len(matches)-1 + index = 0 while not _proper_compiler_style(matches[index], spec.architecture): - index -= 1 - if index == 0: + index += 1 + if index == len(matches) - 1: raise NoValidVersionError(spec) spec.compiler = matches[index].copy() assert(spec.compiler.concrete) -- cgit v1.2.3-60-g2f50 From 6c352132d0dd2c0f05270fa7888d47ca37547674 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 23 May 2016 14:28:00 -0700 Subject: Started writing documentation for architecture. Might need a couple more drafts --- lib/spack/spack/architecture.py | 52 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 41778795c3..45b4cc228d 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,6 +22,58 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +""" +This module contains all the elements that are required to create an +architecture object. These include, the target processor, the operating system, +and the architecture platform (i.e. cray, darwin, linux, bgq, etc) classes. + +On a multiple architecture machine, the architecture spec field can be set to +build a package against any target and operating system that is present on the +platform. On Cray platforms or any other architecture that has different front and +back end environments, the operating system will determine the method of compiler +detection. + +There are two different types of compiler detection: + 1. Through the $PATH env variable (front-end detection) + 2. Through the tcl module system. (back-end detection) + +Depending on which operating system is specified, the compiler will be detected +using one of those methods. + +For platforms such as linux and darwin, the operating system is autodetected and +the target is set to be x86_64. + +The command line syntax for specifying an architecture is as follows: + + target= os= + +If the user wishes to use the defaults, either target or os can be left out of +the command line and Spack will concretize using the default. These defaults are +set in the 'platforms/' directory which contains the different subclasses for +platforms. If the machine has multiple architectures, the user can +also enter front-end, or fe or back-end or be. These settings will concretize +to their respective front-end and back-end targets and operating systems. +Additional platforms can be added by creating a subclass of Platform +and adding it inside the platform directory. + +Platforms are an abstract class that are extended by subclasses. If the user +wants to add a new type of platform (such as cray_xe), they can create a subclass +and set all the class attributes such as priority, front_target ,back_target, +front_os, back_os. Platforms also contain a priority class attribute. A lower +number signifies higher priority. These numbers are arbitrarily set and can be +changed though often there isn't much need unless a new platform is added and +the user wants that to be detected first. + +Targets are created inside the platform subclasses. Most architecture (like linux, +and darwin) will have only one target (x86_64) but in the case of Cray machines, +there is both a frontend and backend processor. The user can specify which targets +are present on front-end and back-end architecture + +Depending on the platform, operating systems are either auto-detected or are +set. The user can set the front-end and back-end operating setting by the class +attributes front_os and back_os. The operating system as described earlier, will +be responsible for compiler detection. +""" import os from collections import namedtuple import imp -- cgit v1.2.3-60-g2f50 From d7612e7aaace3b0fa483f17840e20cde5b28f430 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 23 May 2016 16:54:41 -0700 Subject: Fixed errors caught by spec and concretize tests --- lib/spack/spack/architecture.py | 78 ++++++++++++++++++++++++++------ lib/spack/spack/concretize.py | 23 +++++++--- lib/spack/spack/spec.py | 81 ++++++++++++++++++++++++++++------ lib/spack/spack/test/architecture.py | 2 + lib/spack/spack/test/concretize.py | 4 +- lib/spack/spack/test/spec_semantics.py | 14 +++--- lib/spack/spack/test/spec_syntax.py | 6 --- 7 files changed, 160 insertions(+), 48 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 41778795c3..e4b3dbf9c7 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -105,12 +105,25 @@ class Platform(object): self.operating_sys = {} self.name = name + def to_dict(self): + n = {} + n['targets'] = dict((name, target.to_dict()) for (name, target) in self.targets.items()) + n['operating_systems'] = dict((name, os.to_dict()) for (name, os) in self.operating_sys.items()) + n['priority'] = self.priority + n['default_front_end_target'] = self.front_end + n['default_back_end_target'] = self.back_end + n['default_target'] = self.default + n['default_front_end_os'] = self.front_os + n['default_back_end_os'] = self.back_os + n['default_os'] = self.default_os + return {self.name: n} + def add_target(self, name, target): """Used by the platform specific subclass to list available targets. Raises an error if the platform specifies a name that is reserved by spack as an alias. """ - if name in ['front_end', 'fe', 'back_end', 'be', 'default']: + if name in ['frontend', 'fe', 'backend', 'be', 'default_target']: raise ValueError( "%s is a spack reserved alias " "and cannot be the name of a target" % name) @@ -135,6 +148,10 @@ class Platform(object): """ Add the operating_system class object into the platform.operating_sys dictionary """ + if name in ['frontend', 'fe', 'backend', 'be', 'default_os']: + raise ValueError( + "%s is a spack reserved alias " + "and cannot be the name of an OS" % name) self.operating_sys[name] = os_class def operating_system(self, name): @@ -284,15 +301,18 @@ class OperatingSystem(object): class Arch(object): "Architecture is now a class to help with setting attributes" - def __init__(self, platform_os=None, target=None): - self.platform = sys_type() - if platform_os: - platform_os = self.platform.operating_system(platform_os) + def __init__(self, platform=None, platform_os=None, target=None): + self.platform = platform + if platform and platform_os: + platform_os = self.platform.operating_system(platform_os) self.platform_os = platform_os - if target: + if platform and target: target = self.platform.target(target) self.target = target + # Hooks for parser to use when platform is set after target or os + self.target_string = None + self.os_string = None @property def concrete(self): @@ -302,16 +322,19 @@ class Arch(object): def __str__(self): - if self.platform.name == 'darwin': - os_name = self.platform_os.name + if self.platform or self.platform_os or self.target: + if self.platform.name == 'darwin': + os_name = self.platform_os.name + else: + os_name = str(self.platform_os) + + return (str(self.platform) +"-"+ + os_name + "-" + str(self.target)) else: - os_name = str(self.platform_os) - - return (str(self.platform) +"-"+ - os_name + "-" + str(self.target)) + return '' def _cmp_key(self): - platform = self.platform.name + platform = self.platform.name if isinstance(self.platform, Platform) else self.platform os = self.platform_os.name if isinstance(self.platform_os, OperatingSystem) else self.platform_os target = self.target.name if isinstance(self.target, Target) else self.target return (platform, os, target) @@ -322,7 +345,7 @@ class Arch(object): platform_os = self.platform_os target = self.target - d['platform'] = self.platform.name + d['platform'] = self.platform.to_dict() if self.platform else None d['platform_os'] = self.platform_os.to_dict() if self.platform_os else None d['target'] = self.target.to_dict() if self.target else None @@ -350,6 +373,27 @@ def _operating_system_from_dict(os_dict): operating_system.version = os_dict['version'] return operating_system +def _platform_from_dict(platform_dict): + """ Constructs a platform from a dictionary. """ + platform = Platform.__new__(Platform) + name, p_dict = platform_dict.items()[0] + platform.name = name + platform.targets = {} + for name, t_dict in p_dict['targets'].items(): + platform.add_target(name, _target_from_dict(t_dict)) + platform.operating_sys = {} + for name, o_dict in p_dict['operating_systems'].items(): + platform.add_operating_system(name, _operating_system_from_dict(o_dict)) + platform.priority = p_dict['priority'] + platform.front_end = p_dict['default_front_end_target'] + platform.back_end = p_dict['default_back_end_target'] + platform.default = p_dict['default_target'] + platform.front_os = p_dict['default_front_end_os'] + platform.back_os = p_dict['default_back_end_os'] + platform.default_os = p_dict['default_os'] + + return platform + def arch_from_dict(d): """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict helper methods to recreate the arch tuple from the dictionary read from @@ -359,14 +403,20 @@ def arch_from_dict(d): if d is None: return None + platform_dict = d['platform'] os_dict = d['platform_os'] target_dict = d['target'] + platform = _platform_from_dict(platform_dict) if platform_dict else None target = _target_from_dict(target_dict) if os_dict else None platform_os = _operating_system_from_dict(os_dict) if os_dict else None + arch.platform = platform arch.target = target arch.platform_os = platform_os + arch.os_string = None + arch.target_string = None + return arch @memoized diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index f38afd38dc..1f5c07779e 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -198,7 +198,7 @@ class DefaultConcretizer(object): spec.architecture.platform_os = spec.root.architecture.platform_os else: spec.architecture.platform_os = spec.architecture.platform.operating_system('default_os') - return True #changed + return True #changed def _concretize_target(self, spec): platform = spec.architecture.platform @@ -210,7 +210,19 @@ class DefaultConcretizer(object): spec.architecture.target = spec.root.architecture.target else: spec.architecture.target = spec.architecture.platform.target('default_target') - return True #changed + print spec.architecture, spec.architecture.platform, spec.architecture.platform_os, spec.architecture.target + return True #changed + + def _concretize_platform(self, spec): + if spec.architecture.platform is not None and isinstance( + spec.architecture.platform, spack.architecture.Platform): + return False + if spec.root.architecture and spec.root.architecture.platform: + if isinstance(spec.root.architecture.platform,spack.architecture.Platform): + spec.architecture.platform = spec.root.architecture.platform + else: + spec.architecture.platform = spack.architecture.sys_type() + return True #changed? def concretize_architecture(self, spec): """If the spec is empty provide the defaults of the platform. If the @@ -227,10 +239,11 @@ class DefaultConcretizer(object): # Set the architecture to all defaults spec.architecture = spack.architecture.Arch() return True - + # Concretize the operating_system and target based of the spec - ret = any((self._concretize_operating_system(spec), - self._concretize_target(spec))) + ret = any((self._concretize_platform(spec), + self._concretize_operating_system(spec), + self._concretize_target(spec))) return ret diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index f88475d1c8..d15598405f 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -99,6 +99,7 @@ import sys import itertools import hashlib import base64 +import imp from StringIO import StringIO from operator import attrgetter import yaml @@ -107,6 +108,7 @@ from yaml.error import MarkedYAMLError import llnl.util.tty as tty from llnl.util.lang import * from llnl.util.tty.color import * +from llnl.util.filesystem import join_path import spack import spack.architecture @@ -119,6 +121,7 @@ from spack.cmd.find import display_specs from spack.version import * from spack.util.string import * from spack.util.prefix import Prefix +from spack.util.naming import mod_to_class from spack.virtual import ProviderIndex from spack.build_environment import get_path_from_module, load_module @@ -535,10 +538,24 @@ class Spec(object): Known flags currently include "arch" """ valid_flags = FlagMap.valid_compiler_flags() - if name == 'os' or name == 'operating_system': - self._set_os(value) +# if name == 'arch' or name == 'architecture': +# platform, op_sys, target = value.split('-') +# print platform, op_sys, target, '+++++++' +# self._set_platform(platform) +# self._set_os(op_sys) +# self._set_target(target) + if name == 'platform': + self._set_platform(value) + elif name == 'os' or name == 'operating_system': + if self.architecture.platform: + self._set_os(value) + else: + self.architecture.os_string = value elif name == 'target': - self._set_target(value) + if self.architecture.platform: + self._set_target(value) + else: + self.architecture.target_string = value elif name in valid_flags: assert(self.compiler_flags is not None) self.compiler_flags[name] = value.split() @@ -551,6 +568,39 @@ class Spec(object): "Spec for '%s' cannot have two compilers." % self.name) self.compiler = compiler + def _set_platform(self, value): + """Called by the parser to set the architecture platform""" + if isinstance(value, basestring): + mod_path = spack.platform_path + mod_string = 'spack.platformss' + names = list_modules(mod_path) + if value in names: + # Create a platform object from the name + mod_name = mod_string + value + path = join_path(mod_path, value) + '.py' + mod = imp.load_source(mod_name, path) + class_name = mod_to_class(value) + if not hasattr(mod, class_name): + tty.die('No class %s defined in %s' % (class_name, mod_name)) + cls = getattr(mod, class_name) + if not inspect.isclass(cls): + tty.die('%s.%s is not a class' % (mod_name, class_name)) + platform = cls() + else: + tty.die("No platform class %s defined." % value) + else: + # The value is a platform + platform = value + + self.architecture.platform = platform + + # Set os and target if we previously got strings for them + if self.architecture.os_string: + self._set_os(self.architecture.os_string) + self.architecture.os_string = None + if self.architecture.target_string: + self._set_target(self.architecture.target_string) + self.architecture.target_string = None def _set_os(self, value): """Called by the parser to set the architecture operating system""" @@ -1016,6 +1066,7 @@ class Spec(object): changed = True spec.dependencies = DependencyMap() replacement.dependencies = DependencyMap() + replacement.architecture = self.architecture # TODO: could this and the stuff in _dup be cleaned up? def feq(cfield, sfield): @@ -1426,7 +1477,6 @@ class Spec(object): other.variants[v]) # TODO: Check out the logic here - print self.architecture, other.architecture, "^^^^^^^^^^^^^^^^^^^^^^^" if self.architecture is not None and other.architecture is not None: if self.architecture.platform is not None and other.architecture.platform is not None: if self.architecture.platform != other.architecture.platform: @@ -1831,8 +1881,7 @@ class Spec(object): self.variants, self.architecture, self.compiler, - self.compiler_flags, - self.dag_hash()) + self.compiler_flags) def eq_node(self, other): @@ -1946,7 +1995,7 @@ class Spec(object): if self.variants: write(fmt % str(self.variants), c) elif c == '=': - if self.architecture: + if self.architecture and str(self.architecture): write(fmt % (' arch' + c + str(self.architecture)), c) elif c == '#': out.write('-' + fmt % (self.dag_hash(7))) @@ -2004,7 +2053,7 @@ class Spec(object): if self.variants: write(fmt % str(self.variants), '+') elif named_str == 'ARCHITECTURE': - if self.architecture: + if self.architecture and str(self.architecture): write(fmt % str(self.architecture), ' arch=') elif named_str == 'SHA1': if self.dependencies: @@ -2054,13 +2103,13 @@ class Spec(object): self.variants, other.variants) #Target - if self.target != other.target: - return spack.pkgsort.target_compare(pkgname, - self.target, other.target) + if self.architecture != other.architecture: + return spack.pkgsort.architecture_compare(pkgname, + self.architecture, other.architecture) #Dependency is not configurable - if self.dep_hash() != other.dep_hash(): - return -1 if self.dep_hash() < other.dep_hash() else 1 + if self.dependencies != other.dependencies: + return -1 if self.dependencies < other.dependencies else 1 #Equal specs return 0 @@ -2181,6 +2230,11 @@ class SpecParser(spack.parse.Parser): raise SpecParseError(e) + # If the spec has an os or a target and no platform, give it the default platform + for spec in specs: + for s in spec.traverse(): + if s.architecture.os_string or s.architecture.target_string: + s._set_platform(spack.architecture.sys_type()) return specs @@ -2401,7 +2455,6 @@ class SpecError(spack.error.SpackError): def __init__(self, message): super(SpecError, self).__init__(message) - class SpecParseError(SpecError): """Wrapper for ParseError for when we're parsing specs.""" def __init__(self, parse_error): diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 2927e468a0..f5b1068435 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -26,12 +26,14 @@ class ArchitectureTest(MockPackagesTest): def test_dict_functions_for_architecture(self): arch = Arch() + arch.platform = spack.architecture.sys_type() arch.platform_os = arch.platform.operating_system('default_os') arch.target = arch.platform.target('default_target') d = arch.to_dict() new_arch = spack.architecture.arch_from_dict(d) + self.assertEqual(arch, new_arch) self.assertTrue( isinstance(arch, Arch) ) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 49281b9a0c..963481054e 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -55,8 +55,8 @@ class ConcretizeTest(MockPackagesTest): if abstract.compiler and abstract.compiler.concrete: self.assertEqual(abstract.compiler, concrete.compiler) - if abstract.architecture and abstract.architecture.target.concrete: - self.assertEqual(abstract.target, concrete.target) + if abstract.architecture and abstract.architecture.concrete: + self.assertEqual(abstract.architecture, concrete.architecture) def check_concretize(self, abstract_spec): diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 9bd32a3d10..45c89100d4 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -383,14 +383,14 @@ class SpecSematicsTest(MockPackagesTest): def test_invalid_constraint(self): -# self.check_invalid_constraint('libelf@0:2.0', 'libelf@2.1:3') -# self.check_invalid_constraint('libelf@0:2.5%gcc@4.8:4.9', 'libelf@2.1:3%gcc@4.5:4.7') + self.check_invalid_constraint('libelf@0:2.0', 'libelf@2.1:3') + self.check_invalid_constraint('libelf@0:2.5%gcc@4.8:4.9', 'libelf@2.1:3%gcc@4.5:4.7') -# self.check_invalid_constraint('libelf+debug', 'libelf~debug') -# self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') -# self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') + self.check_invalid_constraint('libelf+debug', 'libelf~debug') + self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') + self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') -# self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') + self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') platform = spack.architecture.sys_type() if len(platform.operating_sys.keys()) > 1 or len(platform.targets.keys()) > 1: os1 = platform.operating_sys.keys()[0] @@ -439,9 +439,9 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_changed('libelf^foo%gcc', 'libelf^foo%gcc@4.5') self.check_constrain_changed('libelf^foo', 'libelf^foo+debug') self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') + platform = spack.architecture.sys_type() default_target = platform.target('default_target').name - print default_target self.check_constrain_changed('libelf^foo', 'libelf^foo target='+default_target) diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index ae19de177b..4a534d7b5c 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -71,12 +71,6 @@ class SpecSyntaxTest(unittest.TestCase): spec = expected output = spack.spec.parse(spec) - # Remove architectures that get added by parser. - if remove_arch: - for spec in output: - for s in spec.traverse(): - s.architecture = None - parsed = (" ".join(str(spec) for spec in output)) self.assertEqual(expected, parsed) -- cgit v1.2.3-60-g2f50 From 1da6bbd14667b50f8c970022614baf39e059a779 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 23 May 2016 17:25:54 -0700 Subject: Fixed a couple more bugs --- lib/spack/spack/build_environment.py | 3 ++- lib/spack/spack/concretize.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 4f4ca26cbc..073f73a91a 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -366,7 +366,8 @@ def get_rpaths(pkg): if os.path.isdir(d.prefix.lib64)) # Second module is our compiler mod name. We use that to get rpaths from # module show output. - rpaths.append(get_path_from_module(pkg.compiler.modules[1])) + if pkg.compiler.modules and len(pkg.compiler.modules > 1): + rpaths.append(get_path_from_module(pkg.compiler.modules[1])) return rpaths diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 7ea8ea6928..7a24c1135e 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -210,7 +210,6 @@ class DefaultConcretizer(object): spec.architecture.target = spec.root.architecture.target else: spec.architecture.target = spec.architecture.platform.target('default_target') - print spec.architecture, spec.architecture.platform, spec.architecture.platform_os, spec.architecture.target return True #changed def _concretize_platform(self, spec): -- cgit v1.2.3-60-g2f50 From 513aae5ef8d610644e8f2d2c855e92a080573f5b Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 23 May 2016 17:39:06 -0700 Subject: fixed bug where earlier test was blowing away environment, causing an error on which('modulecmd') --- lib/spack/spack/test/environment.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index ded1539e18..a0d959db2f 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -24,17 +24,20 @@ ############################################################################## import unittest import os +import copy from spack.environment import EnvironmentModifications class EnvironmentTest(unittest.TestCase): def setUp(self): - os.environ.clear() os.environ['UNSET_ME'] = 'foo' os.environ['EMPTY_PATH_LIST'] = '' os.environ['PATH_LIST'] = '/path/second:/path/third' os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' + def tearDown(self): + pass + def test_set(self): env = EnvironmentModifications() env.set('A', 'dummy value') -- cgit v1.2.3-60-g2f50 From 3a68dd2011488acdff4e3c8cb4690718e87623e6 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 25 May 2016 12:22:16 -0700 Subject: (hopefully) final fixes of the merge --- lib/spack/spack/architecture.py | 13 +++++++++++-- lib/spack/spack/spec.py | 9 +++++++++ lib/spack/spack/test/modules.py | 19 +++++++++++-------- lib/spack/spack/test/spec_dag.py | 4 ++-- lib/spack/spack/test/spec_semantics.py | 17 +++++------------ 5 files changed, 38 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 05dc969ebe..e9dd489e54 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -233,8 +233,17 @@ class Platform(object): return self.name def _cmp_key(self): - return (self.name, (_cmp_key(t) for t in self.targets.values()), - (_cmp_key(o) for o in self.operating_sys.values())) + t_keys = ''.join(str(t._cmp_key()) for t in sorted(self.targets.values())) + o_keys = ''.join(str(o._cmp_key()) for o in sorted(self.operating_sys.values())) + return (self.name, + self.default, + self.front_end, + self.back_end, + self.default_os, + self.front_os, + self.back_os, + t_keys, + o_keys) @key_ordering class OperatingSystem(object): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index d15598405f..5d82af4af6 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1649,9 +1649,18 @@ class Spec(object): # TODO: Need to make sure that comparisons can be made via classes if self.architecture and other.architecture: + print self.architecture, other.architecture if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or (self.architecture.platform_os and other.architecture.platform_os and self.architecture.platform_os != other.architecture.platform_os) or (self.architecture.target and other.architecture.target and self.architecture.target != other.architecture.target)): + d1 = self.architecture.platform.to_dict() + d2 = other.architecture.platform.to_dict() + print d1 + print d2 + print d1==d2 + print self.architecture.platform == other.architecture.platform + print self.architecture.platform._cmp_key() + print other.architecture.platform._cmp_key() return False elif strict and ((other.architecture and not self.architecture) or (other.architecture.platform and not self.architecture.platform) or diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index c73badf8f2..582e067860 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -73,7 +73,7 @@ configuration_alter_environment = { 'all': { 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']} }, - 'arch=x86-linux': { + 'platform=test target=x86_64': { 'environment': {'set': {'FOO': 'foo'}, 'unset': ['BAR']} } @@ -116,6 +116,7 @@ class TclTests(MockPackagesTest): def get_modulefile_content(self, spec): spec.concretize() + print spec, '&&&&&' generator = spack.modules.TclModule(spec) generator.write() content = FILE_REGISTRY[generator.file_name].split('\n') @@ -123,27 +124,28 @@ class TclTests(MockPackagesTest): def test_simple_case(self): spack.modules.CONFIGURATION = configuration_autoload_direct - spec = spack.spec.Spec('mpich@3.0.4 arch=x86-linux') + spec = spack.spec.Spec('mpich@3.0.4') content = self.get_modulefile_content(spec) self.assertTrue('module-whatis "mpich @3.0.4"' in content) def test_autoload(self): spack.modules.CONFIGURATION = configuration_autoload_direct - spec = spack.spec.Spec('mpileaks arch=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2) self.assertEqual(len([x for x in content if 'module load ' in x]), 2) spack.modules.CONFIGURATION = configuration_autoload_all - spec = spack.spec.Spec('mpileaks arch=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5) self.assertEqual(len([x for x in content if 'module load ' in x]), 5) def test_alter_environment(self): spack.modules.CONFIGURATION = configuration_alter_environment - spec = spack.spec.Spec('mpileaks arch=x86-linux') + spec = spack.spec.Spec('mpileaks platform=test target=x86_64') content = self.get_modulefile_content(spec) + print content self.assertEqual( len([x for x in content @@ -152,8 +154,9 @@ class TclTests(MockPackagesTest): len([x for x in content if 'setenv FOO "foo"' in x]), 1) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 1) - spec = spack.spec.Spec('libdwarf arch=x64-linux') + spec = spack.spec.Spec('libdwarf %clang platform=test target=x86_32') content = self.get_modulefile_content(spec) + print content self.assertEqual( len([x for x in content @@ -164,14 +167,14 @@ class TclTests(MockPackagesTest): def test_blacklist(self): spack.modules.CONFIGURATION = configuration_blacklist - spec = spack.spec.Spec('mpileaks arch=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) self.assertEqual(len([x for x in content if 'module load ' in x]), 1) def test_conflicts(self): spack.modules.CONFIGURATION = configuration_conflicts - spec = spack.spec.Spec('mpileaks arch=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual( len([x for x in content if x.startswith('conflict')]), 2) diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 99a986a528..712f07ac4d 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -244,8 +244,8 @@ class SpecDagTest(MockPackagesTest): def test_unsatisfiable_architecture(self): platform = spack.architecture.sys_type() - self.set_pkg_dep('mpileaks', 'mpich target=%s' % platform.target('default_target')) - spec = Spec('mpileaks ^mpich target=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') + self.set_pkg_dep('mpileaks', 'mpich platform=test target=be') + spec = Spec('mpileaks ^mpich platform=test target=fe ^callpath ^dyninst ^libelf ^libdwarf') self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 45c89100d4..aa99ad71ed 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -141,10 +141,9 @@ class SpecSematicsTest(MockPackagesTest): def test_satisfies_architecture(self): platform = spack.architecture.sys_type() - if platform.name == 'crayxc': - self.check_satisfies('foo target=frontend os=frontend', 'target=frontend os=frontend') - self.check_satisfies('foo target=backend os=backend', 'target=backend', 'os=backend') - self.check_satisfies('foo target=default_target os=default_os','target=default_target os=default_os') + self.check_satisfies('foo platform=test target=frontend os=frontend', 'platform=test target=frontend os=frontend') + self.check_satisfies('foo platform=test target=backend os=backend', 'platform=test target=backend', 'platform=test os=backend') + self.check_satisfies('foo platform=test target=default_target os=default_os','platform=test target=default_target os=default_os') #ifdef NEW @@ -391,14 +390,8 @@ class SpecSematicsTest(MockPackagesTest): self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') - platform = spack.architecture.sys_type() - if len(platform.operating_sys.keys()) > 1 or len(platform.targets.keys()) > 1: - os1 = platform.operating_sys.keys()[0] - os2 = platform.operating_sys.keys()[-1] - target1 = platform.targets.keys()[0] - target2 = platform.targets.keys()[-1] - self.check_invalid_constraint('libelf target=%s os=%s' % (target1, os1), - 'libelf target=%s os=%s' % (target2, os2)) + self.check_invalid_constraint('libelf platform=test target=be os=be', + 'libelf target=fe os=fe') def test_constrain_changed(self): self.check_constrain_changed('libelf', '@1.0') -- cgit v1.2.3-60-g2f50 From 0f40174723612c60ca7bdc0ed0ce1e96daa643bd Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 25 May 2016 12:28:59 -0700 Subject: added test platform --- lib/spack/spack/platforms/test.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 lib/spack/spack/platforms/test.py (limited to 'lib') diff --git a/lib/spack/spack/platforms/test.py b/lib/spack/spack/platforms/test.py new file mode 100644 index 0000000000..3a4f9d5f2d --- /dev/null +++ b/lib/spack/spack/platforms/test.py @@ -0,0 +1,28 @@ +import subprocess +from spack.architecture import Platform, Target +from spack.operating_systems.linux_distro import LinuxDistro +from spack.operating_systems.cnl import Cnl + + +class Test(Platform): + priority = 1000000 + front_end = 'x86_32' + back_end = 'x86_64' + default = 'x86_64' + + back_os = 'CNL' + default_os = 'CNL' + + def __init__(self): + super(Test, self).__init__('test') + self.add_target(self.default, Target(self.default)) + self.add_target(self.front_end, Target(self.front_end)) + + self.add_operating_system(self.default_os, Cnl()) + linux_dist = LinuxDistro() + self.front_os = linux_dist.name + self.add_operating_system(self.front_os, linux_dist) + + @classmethod + def detect(self): + return True -- cgit v1.2.3-60-g2f50 From 8f99334f11edd4b56f3e9bc88d2f8666cf0a37a1 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 25 May 2016 16:06:14 -0700 Subject: Fixed test config to work on yosemite --- lib/spack/spack/architecture.py | 38 +++++++++++++++--------------- lib/spack/spack/test/mock_packages_test.py | 24 ++++++++++++++++++- 2 files changed, 42 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e9dd489e54..5d16a2f5f5 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,26 +22,26 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -""" -This module contains all the elements that are required to create an -architecture object. These include, the target processor, the operating system, -and the architecture platform (i.e. cray, darwin, linux, bgq, etc) classes. +""" +This module contains all the elements that are required to create an +architecture object. These include, the target processor, the operating system, +and the architecture platform (i.e. cray, darwin, linux, bgq, etc) classes. -On a multiple architecture machine, the architecture spec field can be set to -build a package against any target and operating system that is present on the +On a multiple architecture machine, the architecture spec field can be set to +build a package against any target and operating system that is present on the platform. On Cray platforms or any other architecture that has different front and back end environments, the operating system will determine the method of compiler -detection. +detection. -There are two different types of compiler detection: +There are two different types of compiler detection: 1. Through the $PATH env variable (front-end detection) 2. Through the tcl module system. (back-end detection) Depending on which operating system is specified, the compiler will be detected -using one of those methods. +using one of those methods. For platforms such as linux and darwin, the operating system is autodetected and -the target is set to be x86_64. +the target is set to be x86_64. The command line syntax for specifying an architecture is as follows: @@ -52,27 +52,27 @@ the command line and Spack will concretize using the default. These defaults are set in the 'platforms/' directory which contains the different subclasses for platforms. If the machine has multiple architectures, the user can also enter front-end, or fe or back-end or be. These settings will concretize -to their respective front-end and back-end targets and operating systems. +to their respective front-end and back-end targets and operating systems. Additional platforms can be added by creating a subclass of Platform -and adding it inside the platform directory. +and adding it inside the platform directory. Platforms are an abstract class that are extended by subclasses. If the user wants to add a new type of platform (such as cray_xe), they can create a subclass -and set all the class attributes such as priority, front_target ,back_target, +and set all the class attributes such as priority, front_target ,back_target, front_os, back_os. Platforms also contain a priority class attribute. A lower number signifies higher priority. These numbers are arbitrarily set and can be changed though often there isn't much need unless a new platform is added and -the user wants that to be detected first. +the user wants that to be detected first. Targets are created inside the platform subclasses. Most architecture (like linux, and darwin) will have only one target (x86_64) but in the case of Cray machines, there is both a frontend and backend processor. The user can specify which targets are present on front-end and back-end architecture -Depending on the platform, operating systems are either auto-detected or are +Depending on the platform, operating systems are either auto-detected or are set. The user can set the front-end and back-end operating setting by the class attributes front_os and back_os. The operating system as described earlier, will -be responsible for compiler detection. +be responsible for compiler detection. """ import os from collections import namedtuple @@ -216,7 +216,7 @@ class Platform(object): return self.operating_sys.get(name, None) - + @classmethod def detect(self): """ Subclass is responsible for implementing this method. @@ -235,7 +235,7 @@ class Platform(object): def _cmp_key(self): t_keys = ''.join(str(t._cmp_key()) for t in sorted(self.targets.values())) o_keys = ''.join(str(o._cmp_key()) for o in sorted(self.operating_sys.values())) - return (self.name, + return (self.name, self.default, self.front_end, self.back_end, @@ -385,7 +385,7 @@ class Arch(object): def __str__(self): if self.platform or self.platform_os or self.target: if self.platform.name == 'darwin': - os_name = self.platform_os.name + os_name = self.platform_os.name if self.platform_os else "None" else: os_name = str(self.platform_os) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 79dbdb89a6..0cdf01f1b9 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -69,6 +69,17 @@ compilers: f77: None fc: None modules: 'None' + clang3.3OSX: + spec: clang@3.3 + operating_system: + name: yosemite + version: '10.10' + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' gcc4.5.0CNL: paths: cc: /path/to/gcc @@ -98,10 +109,21 @@ compilers: f77: /path/to/gfortran fc: /path/to/gfortran operating_system: - name: RHL + name: redhat version: '6.7' spec: gcc@4.5.0 modules: 'None' + gcc4.5.0OSX: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: + name: yosemite + version: '10.10' + spec: gcc@4.5.0 + modules: 'None' """ mock_packages_config = """\ -- cgit v1.2.3-60-g2f50 From c30fe932d96b4a6fbe8845a2f6bdbfb7567ab1d2 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 25 May 2016 16:34:45 -0700 Subject: fixed mock config to use truncated versions of linux distro versions --- lib/spack/spack/test/mock_packages_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 0cdf01f1b9..23b1cf52cc 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -62,7 +62,7 @@ compilers: spec: clang@3.3 operating_system: name: redhat - version: '6.7' + version: '6' paths: cc: /path/to/clang cxx: /path/to/clang++ @@ -110,7 +110,7 @@ compilers: fc: /path/to/gfortran operating_system: name: redhat - version: '6.7' + version: '6' spec: gcc@4.5.0 modules: 'None' gcc4.5.0OSX: -- cgit v1.2.3-60-g2f50 From 3b675d8b700369a5b5282e56ecaf6c559e592551 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 26 May 2016 11:09:14 -0700 Subject: Changed comparison operator so that we don't get caught in an infinite loop --- lib/spack/spack/concretize.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 7a24c1135e..e30cf329f1 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -330,6 +330,8 @@ class DefaultConcretizer(object): compiler is used, defaulting to no compiler flags in the spec. Default specs set at the compiler level will still be added later. """ + + if not spec.architecture.platform_os: #Although this usually means changed, this means awaiting other changes return True @@ -340,9 +342,9 @@ class DefaultConcretizer(object): nearest = next(p for p in spec.traverse(direction='parents') if ((p.compiler == spec.compiler and p is not spec) and flag in p.compiler_flags)) - if ((not flag in spec.compiler_flags) or - sorted(spec.compiler_flags[flag]) != sorted(nearest.compiler_flags[flag])): - if flag in spec.compiler_flags: + if not flag in spec.compiler_flags or \ + not (sorted(spec.compiler_flags[flag]) >= sorted(nearest.compiler_flags[flag])): + if flag in spec.compiler_flag: spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | set(nearest.compiler_flags[flag])) else: -- cgit v1.2.3-60-g2f50 From b968603a57a4085b05318c295443d370f4b4c270 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 26 May 2016 11:14:05 -0700 Subject: made yaml format backwards compatible --- lib/spack/spack/architecture.py | 38 ++++++++++++++---------- lib/spack/spack/operating_systems/pre_v1.py | 17 +++++++++++ lib/spack/spack/platforms/spack_compatibility.py | 23 ++++++++++++++ 3 files changed, 62 insertions(+), 16 deletions(-) create mode 100644 lib/spack/spack/operating_systems/pre_v1.py create mode 100644 lib/spack/spack/platforms/spack_compatibility.py (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 5d16a2f5f5..86733ba418 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -90,7 +90,7 @@ from spack.util.naming import mod_to_class from spack.util.environment import get_path from spack.util.multiproc import parmap import spack.error as serr - +import spack.platforms class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): @@ -462,21 +462,27 @@ def arch_from_dict(d): """ arch = Arch() - if d is None: - return None - platform_dict = d['platform'] - os_dict = d['platform_os'] - target_dict = d['target'] - - platform = _platform_from_dict(platform_dict) if platform_dict else None - target = _target_from_dict(target_dict) if os_dict else None - platform_os = _operating_system_from_dict(os_dict) if os_dict else None - arch.platform = platform - arch.target = target - arch.platform_os = platform_os - - arch.os_string = None - arch.target_string = None + if isinstance(d, basestring): + # We have an old spec using a string for the architecture + arch.platform = spack.platforms.spack_compatibility.SpackCompatibility() + arch.platform_os = arch.platform.operating_system('default') + arch.target = Target(d) + + arch.os_string = None + arch.target_string = None + else: + if d is None: + return None + platform_dict = d['platform'] + os_dict = d['platform_os'] + target_dict = d['target'] + + arch.platform = _platform_from_dict(platform_dict) if platform_dict else None + arch.target = _target_from_dict(target_dict) if os_dict else None + arch.platform_os = _operating_system_from_dict(os_dict) if os_dict else None + + arch.os_string = None + arch.target_string = None return arch diff --git a/lib/spack/spack/operating_systems/pre_v1.py b/lib/spack/spack/operating_systems/pre_v1.py new file mode 100644 index 0000000000..71ee557ac1 --- /dev/null +++ b/lib/spack/spack/operating_systems/pre_v1.py @@ -0,0 +1,17 @@ +import re +import os + +from spack.architecture import OperatingSystem + + +class PreV1(OperatingSystem): + """ Compute Node Linux (CNL) is the operating system used for the Cray XC + series super computers. It is a very stripped down version of GNU/Linux. + Any compilers found through this operating system will be used with + modules. If updated, user must make sure that version and name are + updated to indicate that OS has been upgraded (or downgraded) + """ + def __init__(self): + name = 'PreVersion1.0' + version = '1.0' + super(PreV1, self).__init__(name, version) diff --git a/lib/spack/spack/platforms/spack_compatibility.py b/lib/spack/spack/platforms/spack_compatibility.py new file mode 100644 index 0000000000..86cbfb3044 --- /dev/null +++ b/lib/spack/spack/platforms/spack_compatibility.py @@ -0,0 +1,23 @@ +import subprocess +from spack.architecture import Platform +from spack.operating_systems.pre_v1 + +class SpackCompatibility(Platform): + priority = 9999 + + # We don't use the normal target getters for this platform + # Instead, targets are added directly when parsing the yaml + + # OS is the spack backwards compatibility os. + front_os = 'PreVersion1.0' + back_os = 'PreVersion1.0' + default_os = 'PreVersion1.0' + + def __init__(self): + super(SpackCompatibility, self).__init__('spack_compatibility') + sc_os = spack.operating_systems.pre_v1.PreV1() + self.add_operating_system(sc_os.name, sc_os) + + @classmethod + def detect(self): + return True -- cgit v1.2.3-60-g2f50 From c0661744cccffc6ca11a82cea840e7be63736d37 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 26 May 2016 11:38:40 -0700 Subject: fixed compatibility work --- lib/spack/spack/architecture.py | 5 ++--- lib/spack/spack/operating_systems/pre_v1.py | 17 ----------------- lib/spack/spack/platforms/spack_compatibility.py | 23 ----------------------- 3 files changed, 2 insertions(+), 43 deletions(-) delete mode 100644 lib/spack/spack/operating_systems/pre_v1.py delete mode 100644 lib/spack/spack/platforms/spack_compatibility.py (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 86733ba418..a0ef4f14da 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -90,7 +90,6 @@ from spack.util.naming import mod_to_class from spack.util.environment import get_path from spack.util.multiproc import parmap import spack.error as serr -import spack.platforms class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): @@ -464,8 +463,8 @@ def arch_from_dict(d): if isinstance(d, basestring): # We have an old spec using a string for the architecture - arch.platform = spack.platforms.spack_compatibility.SpackCompatibility() - arch.platform_os = arch.platform.operating_system('default') + arch.platform = Platform('spack_compatibility') + arch.platform_os = OperatingSystem('pre_version', '1.0') arch.target = Target(d) arch.os_string = None diff --git a/lib/spack/spack/operating_systems/pre_v1.py b/lib/spack/spack/operating_systems/pre_v1.py deleted file mode 100644 index 71ee557ac1..0000000000 --- a/lib/spack/spack/operating_systems/pre_v1.py +++ /dev/null @@ -1,17 +0,0 @@ -import re -import os - -from spack.architecture import OperatingSystem - - -class PreV1(OperatingSystem): - """ Compute Node Linux (CNL) is the operating system used for the Cray XC - series super computers. It is a very stripped down version of GNU/Linux. - Any compilers found through this operating system will be used with - modules. If updated, user must make sure that version and name are - updated to indicate that OS has been upgraded (or downgraded) - """ - def __init__(self): - name = 'PreVersion1.0' - version = '1.0' - super(PreV1, self).__init__(name, version) diff --git a/lib/spack/spack/platforms/spack_compatibility.py b/lib/spack/spack/platforms/spack_compatibility.py deleted file mode 100644 index 86cbfb3044..0000000000 --- a/lib/spack/spack/platforms/spack_compatibility.py +++ /dev/null @@ -1,23 +0,0 @@ -import subprocess -from spack.architecture import Platform -from spack.operating_systems.pre_v1 - -class SpackCompatibility(Platform): - priority = 9999 - - # We don't use the normal target getters for this platform - # Instead, targets are added directly when parsing the yaml - - # OS is the spack backwards compatibility os. - front_os = 'PreVersion1.0' - back_os = 'PreVersion1.0' - default_os = 'PreVersion1.0' - - def __init__(self): - super(SpackCompatibility, self).__init__('spack_compatibility') - sc_os = spack.operating_systems.pre_v1.PreV1() - self.add_operating_system(sc_os.name, sc_os) - - @classmethod - def detect(self): - return True -- cgit v1.2.3-60-g2f50 From 88bec814eb45ff4997229e092f234d8ce7c76664 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 26 May 2016 12:10:41 -0700 Subject: Allow 'arch=' syntax for specs to allow copying from output --- lib/spack/spack/concretize.py | 2 +- lib/spack/spack/platforms/cray_xc.py | 12 ++++++------ lib/spack/spack/platforms/linux.py | 8 ++++---- lib/spack/spack/spec.py | 18 +++++++++++------- lib/spack/spack/test/spec_semantics.py | 1 - 5 files changed, 22 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index e30cf329f1..5d9715feed 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -344,7 +344,7 @@ class DefaultConcretizer(object): and flag in p.compiler_flags)) if not flag in spec.compiler_flags or \ not (sorted(spec.compiler_flags[flag]) >= sorted(nearest.compiler_flags[flag])): - if flag in spec.compiler_flag: + if flag in spec.compiler_flags: spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | set(nearest.compiler_flags[flag])) else: diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index 4843a47c62..e710303e23 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -9,16 +9,16 @@ class CrayXc(Platform): back_end = 'ivybridge' default = 'ivybridge' - front_os = "SuSE" - back_os = "CNL" - default_os = "CNL" + front_os = "SuSE11" + back_os = "CNL10" + default_os = "CNL10" def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better if we use CRAY_CPU_TARGET as the default. This will ensure that if we're on a XC-40 or XC-30 then we can detect the target ''' - super(CrayXc, self).__init__('crayxc') + super(CrayXc, self).__init__('cray_xc') # Handle the default here so we can check for a key error if 'CRAY_CPU_TARGET' in os.environ: @@ -37,8 +37,8 @@ class CrayXc(Platform): self.add_target('haswell', Target('haswell','craype-haswell')) - self.add_operating_system('SuSE', LinuxDistro()) - self.add_operating_system('CNL', Cnl()) + self.add_operating_system('SuSE11', LinuxDistro()) + self.add_operating_system('CNL10', Cnl()) @classmethod def detect(self): diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 18050ac79e..ebe018812a 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -12,10 +12,10 @@ class Linux(Platform): super(Linux, self).__init__('linux') self.add_target(self.default, Target(self.default)) linux_dist = LinuxDistro() - self.default_os = linux_dist.name - self.front_os = linux_dist.name - self.back_os = linux_dist.name - self.add_operating_system(linux_dist.name, linux_dist) + self.default_os = str(linux_dist) + self.front_os = self.default_os + self.back_os = self.default._os + self.add_operating_system(str(linux_dist), linux_dist) @classmethod def detect(self): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 5d82af4af6..58d76fd7a9 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -538,13 +538,17 @@ class Spec(object): Known flags currently include "arch" """ valid_flags = FlagMap.valid_compiler_flags() -# if name == 'arch' or name == 'architecture': -# platform, op_sys, target = value.split('-') -# print platform, op_sys, target, '+++++++' -# self._set_platform(platform) -# self._set_os(op_sys) -# self._set_target(target) - if name == 'platform': + if name == 'arch' or name == 'architecture': + platform, op_sys, target = value.split('-') + assert(self.architecture.platform is None) + assert(self.architecture.platform_os is None) + assert(self.architecture.target is None) + assert(self.architecture.os_string is None) + assert(self.architecture.target_string is None) + self._set_platform(platform) + self._set_os(op_sys) + self._set_target(target) + elif name == 'platform': self._set_platform(value) elif name == 'os' or name == 'operating_system': if self.architecture.platform: diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index aa99ad71ed..0b2111bfe8 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -402,7 +402,6 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_changed('libelf', '~debug') self.check_constrain_changed('libelf', 'debug=2') self.check_constrain_changed('libelf', 'cppflags="-O3"') - self.check_constrain_changed('libelf', ' arch=bgqos_0') platform = spack.architecture.sys_type() self.check_constrain_changed('libelf', 'target='+platform.target('default_target').name) -- cgit v1.2.3-60-g2f50 From 189e8b3f349c8102772fa995fd8ee79df3c242c0 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 26 May 2016 13:17:01 -0700 Subject: fixed compiler config init logic --- lib/spack/spack/compilers/__init__.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 8fe7a17116..ea79f655b0 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -102,15 +102,18 @@ def get_compiler_config(scope=None): # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. # if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: - if scope is None: - # We know no compilers were configured in any scope. - init_compiler_config() - elif scope == 'user': - # Check the site config and update the user config if - # nothing is configured at the site level. - site_config = spack.config.get_config('compilers', scope='site') - if not site_config: + if not config: + if scope is None: + # We know no compilers were configured in any scope. init_compiler_config() + config = spack.config.get_config('compilers', scope=scope) + elif scope == 'user': + # Check the site config and update the user config if + # nothing is configured at the site level. + site_config = spack.config.get_config('compilers', scope='site') + if not site_config: + init_compiler_config() + config = spack.config.get_config('compilers', scope=scope) return config -- cgit v1.2.3-60-g2f50 From 9c1da4fa4cdfe32d8a7a0026f457066b0cac7e8e Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 26 May 2016 13:58:17 -0700 Subject: Corrected parantheses wrapping --- lib/spack/spack/build_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 073f73a91a..4483b9d132 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -366,7 +366,7 @@ def get_rpaths(pkg): if os.path.isdir(d.prefix.lib64)) # Second module is our compiler mod name. We use that to get rpaths from # module show output. - if pkg.compiler.modules and len(pkg.compiler.modules > 1): + if pkg.compiler.modules and len(pkg.compiler.modules) > 1: rpaths.append(get_path_from_module(pkg.compiler.modules[1])) return rpaths -- cgit v1.2.3-60-g2f50 From a2f90453f4d1ac1372f717911bd08e33d801ff31 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 26 May 2016 20:30:05 -0700 Subject: Fixed backwards compatibility for compilers.yaml config file --- bin/spack | 3 ++ lib/spack/spack/yaml_version_check.py | 55 +++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 lib/spack/spack/yaml_version_check.py (limited to 'lib') diff --git a/bin/spack b/bin/spack index 3544feb10a..9b1276a866 100755 --- a/bin/spack +++ b/bin/spack @@ -138,6 +138,9 @@ def main(): import spack.util.debug as debug debug.register_interrupt_handler() + from spack.yaml_version_check import check_yaml_versions + check_yaml_versions() + spack.spack_working_dir = working_dir if args.mock: from spack.repository import RepoPath diff --git a/lib/spack/spack/yaml_version_check.py b/lib/spack/spack/yaml_version_check.py new file mode 100644 index 0000000000..7e7bae4edf --- /dev/null +++ b/lib/spack/spack/yaml_version_check.py @@ -0,0 +1,55 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Yaml Version Check is a module for ensuring that config file +formats are compatible with the current version of Spack.""" +import os.path +import os +import llnl.util.tty as tty +import spack.util.spack_yaml as syaml +import spack.config + + +def check_yaml_versions(): + check_compiler_yaml_version() + +def check_compiler_yaml_version(): + config_scopes = spack.config.config_scopes + for scope in config_scopes.values(): + file_name = os.path.join(scope.path, 'compilers.yaml') + data = None + if os.path.isfile(file_name): + with open(file_name) as f: + data = syaml.load(f) + + if data: + compilers = data['compilers'].items() + if len(compilers) > 0: + if 'operating_system' not in compilers[0][1]: + new_file = os.path.join(scope.path, '_old_compilers.yaml') + tty.warn('%s in out of date compilers format. ' + 'Moved to %s. Spack automatically generate ' + 'a compilers config file ' + % (file_name, new_file)) + os.rename(file_name, new_file) -- cgit v1.2.3-60-g2f50 From 023504ed645fecc7e7c12948dac890b65124c880 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 27 May 2016 12:26:59 -0700 Subject: Fixed the external modules bug --- lib/spack/spack/spec.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 58d76fd7a9..bbc1abfa9e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1065,7 +1065,7 @@ class Spec(object): continue # If replacement is external then trim the dependencies - if replacement.external: + if replacement.external or replacement.external_module: if (spec.dependencies): changed = True spec.dependencies = DependencyMap() @@ -1082,7 +1082,8 @@ class Spec(object): feq(replacement.architecture, spec.architecture) and feq(replacement.dependencies, spec.dependencies) and feq(replacement.variants, spec.variants) and - feq(replacement.external, spec.external)): + feq(replacement.external, spec.external) and + feq(replacement.external_module, spec.external_module)): continue # Refine this spec to the candidate. This uses @@ -1653,18 +1654,9 @@ class Spec(object): # TODO: Need to make sure that comparisons can be made via classes if self.architecture and other.architecture: - print self.architecture, other.architecture if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or (self.architecture.platform_os and other.architecture.platform_os and self.architecture.platform_os != other.architecture.platform_os) or (self.architecture.target and other.architecture.target and self.architecture.target != other.architecture.target)): - d1 = self.architecture.platform.to_dict() - d2 = other.architecture.platform.to_dict() - print d1 - print d2 - print d1==d2 - print self.architecture.platform == other.architecture.platform - print self.architecture.platform._cmp_key() - print other.architecture.platform._cmp_key() return False elif strict and ((other.architecture and not self.architecture) or (other.architecture.platform and not self.architecture.platform) or -- cgit v1.2.3-60-g2f50 From f49644cdea3b00b057f33bf0c609e3a83eb3cd65 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 27 May 2016 12:37:58 -0700 Subject: Fixed typo in linux platform --- lib/spack/spack/platforms/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index ebe018812a..4d8adac384 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -14,7 +14,7 @@ class Linux(Platform): linux_dist = LinuxDistro() self.default_os = str(linux_dist) self.front_os = self.default_os - self.back_os = self.default._os + self.back_os = self.default_os self.add_operating_system(str(linux_dist), linux_dist) @classmethod -- cgit v1.2.3-60-g2f50 From becec8ac7e17a9d56f1d5b0640c1f56be8f64a7f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 10:12:53 -0700 Subject: Added external module check in normalize to avoid infinite loop --- lib/spack/spack/spec.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index a505b3c12e..f8bad3ff03 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1002,11 +1002,12 @@ class Spec(object): """ # Make an index of stuff this spec already provides self_index = ProviderIndex(self.traverse(), restrict=True) - changed = False done = False + while not done: done = True + for spec in list(self.traverse()): replacement = None if spec.virtual: @@ -1329,7 +1330,7 @@ class Spec(object): # if we descend into a virtual spec, there's nothing more # to normalize. Concretize will finish resolving it later. - if self.virtual or self.external: + if self.virtual or self.external or self.external_module: return False # Combine constraints from package deps with constraints from @@ -1376,7 +1377,6 @@ class Spec(object): # Ensure first that all packages & compilers in the DAG exist. self.validate_names() - # Get all the dependencies into one DependencyMap spec_deps = self.flat_dependencies(copy=False) -- cgit v1.2.3-60-g2f50 From 98a4a9f2cc67c141deb7903f577c60c21912667c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 10:13:22 -0700 Subject: Adding test for external modules --- lib/spack/spack/test/concretize.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 963481054e..77ff026c7a 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -253,6 +253,13 @@ class ConcretizeTest(MockPackagesTest): self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) + def test_external_package_module(self): + spec = Spec('externalmodule') + spec.concretize() + self.assertEqual(spec['externalmodule'].external_module, 'external-module') + self.assertFalse('externalprereq' in spec) + self.assertTrue(spec['externalmodule'].compiler.satisfies('gcc')) + def test_nobuild_package(self): got_error = False spec = Spec('externaltool%clang') -- cgit v1.2.3-60-g2f50 From f0a9c45207eabc81b5f50b68f6dc02f807f1a221 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 10:13:57 -0700 Subject: Adding external to mock config --- lib/spack/spack/test/mock_packages_test.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 23b1cf52cc..5561027aac 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -137,6 +137,10 @@ packages: paths: externalvirtual@2.0%clang@3.3: /path/to/external_virtual_clang externalvirtual@1.0%gcc@4.5.0: /path/to/external_virtual_gcc + externalmodule: + buildable: False + modules: + externalmodule@1.0%gcc@4.5.0: external-module """ class MockPackagesTest(unittest.TestCase): -- cgit v1.2.3-60-g2f50 From 69585cb6e3b391d2771387830067d893f2e7ec32 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 12:39:43 -0700 Subject: Added elcap compilers --- lib/spack/spack/test/mock_packages_test.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 5561027aac..f02bfb7a07 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -124,6 +124,28 @@ compilers: version: '10.10' spec: gcc@4.5.0 modules: 'None' + gcc4.5.0ELCAP: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: + name: elcapitan + version: '10.11' + spec: gcc@4.5.0 + modules: 'None' + clang3.3ELCAP: + spec: clang@3.3 + operating_system: + name: elcapitan + version: '10.11' + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' """ mock_packages_config = """\ -- cgit v1.2.3-60-g2f50 From 22ca72e7b9870dddfd66760658a9221f4f24ffe3 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 12:40:29 -0700 Subject: Added conditional skipIf to test_external_modules, darwin machines don't have tcl modules so skip this test and let user know why --- lib/spack/spack/test/concretize.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 77ff026c7a..cf2daa4be2 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -23,6 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import spack +import spack.architecture from spack.spec import Spec, CompilerSpec from spack.version import ver from spack.concretize import find_spec @@ -252,7 +253,8 @@ class ConcretizeTest(MockPackagesTest): self.assertFalse('externalprereq' in spec) self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) - + @unittest.skipIf(spack.architecture.sys_type().name == 'darwin', + "No tcl modules on darwin machines") def test_external_package_module(self): spec = Spec('externalmodule') spec.concretize() @@ -269,7 +271,7 @@ class ConcretizeTest(MockPackagesTest): got_error = True self.assertTrue(got_error) - + def test_external_and_virtual(self): spec = Spec('externaltest') spec.concretize() -- cgit v1.2.3-60-g2f50 From f96c97902452761cbf2bff64d3efa1240df70771 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 30 May 2016 13:05:49 -0700 Subject: Added a conditional to check if package is external --- lib/spack/spack/concretize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 5d9715feed..58811c3419 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -84,7 +84,8 @@ class DefaultConcretizer(object): raise NoBuildError(spec) def cmp_externals(a, b): - if a.name != b.name: + if a.name != b.name and (not a.external or a.external_module and + not b.external and b.external_module): # We're choosing between different providers, so # maintain order from provider sort return candidates.index(a) - candidates.index(b) -- cgit v1.2.3-60-g2f50 From 08a9d80ebcd239790d28904787ea5f6a9bf5fa5d Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 30 May 2016 13:07:20 -0700 Subject: Changed platform to py_platform to avoid name collisions. The naming collisions resulted in an error on linux machines --- lib/spack/spack/test/architecture.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index f5b1068435..3d08269dee 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -3,7 +3,7 @@ """ import unittest import os -import platform +import platform as py_platform import spack from spack.architecture import * from spack.spec import * @@ -61,9 +61,9 @@ class ArchitectureTest(MockPackagesTest): my_platform_class = CrayXc() elif os.path.exists('/bgsys'): my_platform_class = Bgq() - elif 'Linux' in platform.system(): + elif 'Linux' in py_platform.system(): my_platform_class = Linux() - elif 'Darwin' in platform.system(): + elif 'Darwin' in py_platform.system(): my_platform_class = Darwin() self.assertEqual(str(output_platform_class), str(my_platform_class)) -- cgit v1.2.3-60-g2f50 From 716991775d6f6541e2acf41d51ff528e4c3412c7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 30 May 2016 13:09:41 -0700 Subject: Added extra check on linux machines, since most linux machines don't have module system --- lib/spack/spack/test/concretize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index cf2daa4be2..67b20b4084 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -253,8 +253,9 @@ class ConcretizeTest(MockPackagesTest): self.assertFalse('externalprereq' in spec) self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) - @unittest.skipIf(spack.architecture.sys_type().name == 'darwin', - "No tcl modules on darwin machines") + @unittest.skipIf(spack.architecture.sys_type().name == 'darwin' or + spack.architecture.sys_type().name == 'linux', + "No tcl modules on darwin/linux machines") def test_external_package_module(self): spec = Spec('externalmodule') spec.concretize() -- cgit v1.2.3-60-g2f50 From b14ba3125033dc93afb9662ba87c2ca0acbbbde4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 30 May 2016 13:22:08 -0700 Subject: Quick fix for mock compilers.yaml to work on linux machines. --- lib/spack/spack/test/mock_packages_test.py | 35 +++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index f02bfb7a07..561f319b16 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -34,8 +34,41 @@ from ordereddict_backport import OrderedDict from spack.repository import RepoPath from spack.spec import Spec +platform = spack.architecture.sys_type() + +linux_os_name = 'debian' +linux_os_version = '6' + +if platform.name == 'linux': + global linux_os_name, linux_os_version + linux_os = platform.operating_system("default_os") + linux_os_name = linux_os.name + linux_os_version = linux_os.version + mock_compiler_config = """\ compilers: + clang3.3GENLINUX: + spec: clang@3.3 + operating_system: + name: {0} + version: '{1}' + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' + gcc4.5GENLINUX: + spec: gcc@4.5.0 + operating_system: + name: {0} + version: '{1}' + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: None + fc: None + modules: 'None' clang3.3CNL: spec: clang@3.3 operating_system: @@ -146,7 +179,7 @@ compilers: f77: None fc: None modules: 'None' -""" +""".format(linux_os_name, linux_os_version) mock_packages_config = """\ packages: -- cgit v1.2.3-60-g2f50 From 24ee32d7b0cb209f8a17f6df7e8715fa508d0f5d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 30 May 2016 22:02:22 -0700 Subject: More flexible reading of specs from DB, formatting. --- lib/spack/spack/database.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index e768ddf5fe..38bb7541e0 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -214,9 +214,10 @@ class Database(object): # Add dependencies from other records in the install DB to # form a full spec. - for dep_hash in spec_dict[spec.name]['dependencies'].values(): - child = self._read_spec_from_yaml(dep_hash, installs, hash_key) - spec._add_dependency(child) + if 'dependencies' in spec_dict[spec.name]: + for dep_hash in spec_dict[spec.name]['dependencies'].values(): + child = self._read_spec_from_yaml(dep_hash, installs, hash_key) + spec._add_dependency(child) # Specs from the database need to be marked concrete because # they represent actual installations. @@ -289,7 +290,8 @@ class Database(object): except Exception as e: tty.warn("Invalid database reecord:", "file: %s" % self._index_path, - "hash: %s" % hash_key, "cause: %s" % str(e)) + "hash: %s" % hash_key, + "cause: %s: %s" % (type(e).__name__, str(e))) raise self._data = data -- cgit v1.2.3-60-g2f50 From 7bdf63a0fa87cce6f9237ffae4a8903311327d54 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 31 May 2016 03:01:05 -0700 Subject: Fix some bugs in architecture. --- lib/spack/spack/architecture.py | 14 ++++++-------- lib/spack/spack/operating_systems/linux_distro.py | 13 ++++++++++--- 2 files changed, 16 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index a0ef4f14da..88cebc0943 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -487,25 +487,23 @@ def arch_from_dict(d): @memoized def all_platforms(): - modules = [] - + classes = [] mod_path = spack.platform_path - mod_string = "spack.platformss" + parent_module = "spack.platforms" for name in list_modules(mod_path): - mod_name = mod_string + name - path = join_path(mod_path, name) + ".py" - mod = imp.load_source(mod_name, path) + mod_name = '%s.%s' % (parent_module, name) class_name = mod_to_class(name) + mod = __import__(mod_name, fromlist=[class_name]) if not hasattr(mod, class_name): tty.die('No class %s defined in %s' % (class_name, mod_name)) cls = getattr(mod, class_name) if not inspect.isclass(cls): tty.die('%s.%s is not a class' % (mod_name, class_name)) - modules.append(cls) + classes.append(cls) - return modules + return classes @memoized def sys_type(): diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py index d0f24d842f..2e3c72719b 100644 --- a/lib/spack/spack/operating_systems/linux_distro.py +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -1,3 +1,4 @@ +import re import platform as py_platform from spack.architecture import OperatingSystem @@ -9,7 +10,13 @@ class LinuxDistro(OperatingSystem): platform.dist() """ def __init__(self): - name = py_platform.dist()[0] - version = py_platform.dist()[1].split(".")[0] # Grabs major version from tuple + distname, version, _ = py_platform.linux_distribution( + full_distribution_name=False) - super(LinuxDistro, self).__init__(name, version) + # Grabs major version from tuple on redhat; on other platforms + # grab the first legal identifier in the version field. On + # debian you get things like 'wheezy/sid'; sid means unstable. + # We just record 'wheezy' and don't get quite so detailed. + version = re.split(r'[^\w-]', version)[0] + + super(LinuxDistro, self).__init__(distname, version) -- cgit v1.2.3-60-g2f50 From 24d160e93ed80052ad30f99ce36b34de2872a888 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 3 Jun 2016 15:57:54 -0700 Subject: Changed how architecture looks like in spec.yaml. Now it's only three strings, platform, os, target in spec.yaml. Also did some flake8 formatting changes --- lib/spack/spack/architecture.py | 216 +++++++++++++++++++--------------------- 1 file changed, 105 insertions(+), 111 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index a0ef4f14da..ee0d2173c7 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -29,8 +29,9 @@ and the architecture platform (i.e. cray, darwin, linux, bgq, etc) classes. On a multiple architecture machine, the architecture spec field can be set to build a package against any target and operating system that is present on the -platform. On Cray platforms or any other architecture that has different front and -back end environments, the operating system will determine the method of compiler +platform. On Cray platforms or any other architecture that has different front +and back end environments, the operating system will determine the method of +compiler detection. There are two different types of compiler detection: @@ -40,44 +41,42 @@ There are two different types of compiler detection: Depending on which operating system is specified, the compiler will be detected using one of those methods. -For platforms such as linux and darwin, the operating system is autodetected and -the target is set to be x86_64. +For platforms such as linux and darwin, the operating system is autodetected +and the target is set to be x86_64. The command line syntax for specifying an architecture is as follows: target= os= If the user wishes to use the defaults, either target or os can be left out of -the command line and Spack will concretize using the default. These defaults are -set in the 'platforms/' directory which contains the different subclasses for -platforms. If the machine has multiple architectures, the user can +the command line and Spack will concretize using the default. These defaults +are set in the 'platforms/' directory which contains the different subclasses +for platforms. If the machine has multiple architectures, the user can also enter front-end, or fe or back-end or be. These settings will concretize to their respective front-end and back-end targets and operating systems. Additional platforms can be added by creating a subclass of Platform and adding it inside the platform directory. Platforms are an abstract class that are extended by subclasses. If the user -wants to add a new type of platform (such as cray_xe), they can create a subclass -and set all the class attributes such as priority, front_target ,back_target, -front_os, back_os. Platforms also contain a priority class attribute. A lower -number signifies higher priority. These numbers are arbitrarily set and can be -changed though often there isn't much need unless a new platform is added and -the user wants that to be detected first. - -Targets are created inside the platform subclasses. Most architecture (like linux, -and darwin) will have only one target (x86_64) but in the case of Cray machines, -there is both a frontend and backend processor. The user can specify which targets -are present on front-end and back-end architecture +wants to add a new type of platform (such as cray_xe), they can create a +subclass and set all the class attributes such as priority, front_target, +back_target, front_os, back_os. Platforms also contain a priority class +attribute. A lower number signifies higher priority. These numbers are +arbitrarily set and can be changed though often there isn't much need unless a +new platform is added and the user wants that to be detected first. + +Targets are created inside the platform subclasses. Most architecture +(like linux, and darwin) will have only one target (x86_64) but in the case of +Cray machines, there is both a frontend and backend processor. The user can +specify which targets are present on front-end and back-end architecture Depending on the platform, operating systems are either auto-detected or are set. The user can set the front-end and back-end operating setting by the class -attributes front_os and back_os. The operating system as described earlier, will -be responsible for compiler detection. +attributes front_os and back_os. The operating system as described earlier, +will be responsible for compiler detection. """ import os -from collections import namedtuple import imp -import platform as py_platform import inspect from llnl.util.lang import memoized, list_modules, key_ordering @@ -91,6 +90,7 @@ from spack.util.environment import get_path from spack.util.multiproc import parmap import spack.error as serr + class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): super(InvalidSysTypeError, self).__init__( @@ -128,12 +128,6 @@ class Target(object): def __str__(self): return self.name - def to_dict(self): - d = {} - d['name'] = self.name - d['module_name'] = self.module_name - - return d @key_ordering class Platform(object): @@ -142,7 +136,7 @@ class Platform(object): is returned """ - priority = None # Subclass needs to set this number. This controls order in which platform is detected. + priority = None # Subclass sets number. Controls detection order front_end = None back_end = None default = None # The default back end target. On cray ivybridge @@ -156,19 +150,6 @@ class Platform(object): self.operating_sys = {} self.name = name - def to_dict(self): - n = {} - n['targets'] = dict((name, target.to_dict()) for (name, target) in self.targets.items()) - n['operating_systems'] = dict((name, os.to_dict()) for (name, os) in self.operating_sys.items()) - n['priority'] = self.priority - n['default_front_end_target'] = self.front_end - n['default_back_end_target'] = self.back_end - n['default_target'] = self.default - n['default_front_end_os'] = self.front_os - n['default_back_end_os'] = self.back_os - n['default_os'] = self.default_os - return {self.name: n} - def add_target(self, name, target): """Used by the platform specific subclass to list available targets. Raises an error if the platform specifies a name @@ -215,7 +196,6 @@ class Platform(object): return self.operating_sys.get(name, None) - @classmethod def detect(self): """ Subclass is responsible for implementing this method. @@ -232,8 +212,10 @@ class Platform(object): return self.name def _cmp_key(self): - t_keys = ''.join(str(t._cmp_key()) for t in sorted(self.targets.values())) - o_keys = ''.join(str(o._cmp_key()) for o in sorted(self.operating_sys.values())) + t_keys = ''.join(str(t._cmp_key()) for t in + sorted(self.targets.values())) + o_keys = ''.join(str(o._cmp_key()) for o in + sorted(self.operating_sys.values())) return (self.name, self.default, self.front_end, @@ -244,6 +226,7 @@ class Platform(object): t_keys, o_keys) + @key_ordering class OperatingSystem(object): """ Operating System will be like a class similar to platform extended @@ -265,7 +248,6 @@ class OperatingSystem(object): def _cmp_key(self): return (self.name, self.version) - def find_compilers(self, *paths): """ Return a list of compilers found in the suppied paths. @@ -293,11 +275,13 @@ class OperatingSystem(object): # compiler. We can spawn a bunch of parallel searches to reduce # the overhead of spelunking all these directories. types = spack.compilers.all_compiler_types() - compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *filtered_path), types) + compiler_lists = parmap(lambda cmp_cls: + self.find_compiler(cmp_cls, *filtered_path), + types) # ensure all the version calls we made are cached in the parent # process, as well. This speeds up Spack a lot. - clist = reduce(lambda x,y: x+y, compiler_lists) + clist = reduce(lambda x, y: x+y, compiler_lists) return clist def find_compiler(self, cmp_cls, *path): @@ -338,7 +322,7 @@ class OperatingSystem(object): # prefer the one with more compilers. prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc] - newcount = len([p for p in paths if p is not None]) + newcount = len([p for p in paths if p is not None]) prevcount = len([p for p in prev_paths if p is not None]) # Don't add if it's not an improvement over prev compiler. @@ -349,14 +333,7 @@ class OperatingSystem(object): return list(compilers.values()) - def to_dict(self): - d = {} - d['name'] = self.name - d['version'] = self.version - return d - -#NOTE: Key error caused because Architecture has no comparison method @key_ordering class Arch(object): "Architecture is now a class to help with setting attributes" @@ -376,10 +353,11 @@ class Arch(object): @property def concrete(self): - return all( (self.platform is not None, isinstance(self.platform, Platform), - self.platform_os is not None, isinstance(self.platform_os, OperatingSystem), - self.target is not None, isinstance(self.target, Target) ) ) - + return all((self.platform is not None, + isinstance(self.platform, Platform), + self.platform_os is not None, + isinstance(self.platform_os, OperatingSystem), + self.target is not None, isinstance(self.target, Target))) def __str__(self): if self.platform or self.platform_os or self.target: @@ -388,71 +366,75 @@ class Arch(object): else: os_name = str(self.platform_os) - return (str(self.platform) +"-"+ + return (str(self.platform) + "-" + os_name + "-" + str(self.target)) else: return '' def _cmp_key(self): - platform = self.platform.name if isinstance(self.platform, Platform) else self.platform - os = self.platform_os.name if isinstance(self.platform_os, OperatingSystem) else self.platform_os - target = self.target.name if isinstance(self.target, Target) else self.target - return (platform, os, target) + if isinstance(self.platform, Platform): + platform = self.platform.name + else: + platform = self.platform + if isinstance(self.platform_os, OperatingSystem): + platform_os = self.platform_os.name + else: + platform_os = self.platform_os + if isinstance(self.target, Target): + target = self.target.name + else: + target = self.target + print (platform, platform_os, target) + return (platform, platform_os, target) def to_dict(self): d = {} - platform = self.platform - platform_os = self.platform_os - target = self.target - - d['platform'] = self.platform.to_dict() if self.platform else None - d['platform_os'] = self.platform_os.to_dict() if self.platform_os else None - d['target'] = self.target.to_dict() if self.target else None + d['platform'] = str(self.platform) if self.platform else None + d['platform_os'] = str(self.platform_os) if self.platform_os else None + d['target'] = str(self.target) if self.target else None return d -def _target_from_dict(target_dict): +def _target_from_dict(target_name, platform=None): """ Creates new instance of target and assigns all the attributes of that target from the dictionary """ - target = Target.__new__(Target) - target.name = target_dict['name'] - target.module_name = target_dict['module_name'] - if 'platform_name' in target_dict: - target.platform_name = target_dict['platform_name'] - return target - -def _operating_system_from_dict(os_dict): + if not platform: + platform = sys_type() + return platform.target(target_name) + + +def _operating_system_from_dict(os_name, platform=None): """ uses platform's operating system method to grab the constructed operating systems that are valid on the platform. """ -# NOTE: Might need a better way to create operating system objects - operating_system = OperatingSystem.__new__(OperatingSystem) - operating_system.name = os_dict['name'] - operating_system.version = os_dict['version'] - return operating_system + if not platform: + platform = sys_type() + if isinstance(os_name, spack.util.spack_yaml.syaml_dict): + name = os_name['name'] + version = os_name['version'] + return platform.operating_system(name+version) + else: + return platform.operating_system(os_name) + -def _platform_from_dict(platform_dict): +def _platform_from_dict(platform_name): """ Constructs a platform from a dictionary. """ - platform = Platform.__new__(Platform) - name, p_dict = platform_dict.items()[0] - platform.name = name - platform.targets = {} - for name, t_dict in p_dict['targets'].items(): - platform.add_target(name, _target_from_dict(t_dict)) - platform.operating_sys = {} - for name, o_dict in p_dict['operating_systems'].items(): - platform.add_operating_system(name, _operating_system_from_dict(o_dict)) - platform.priority = p_dict['priority'] - platform.front_end = p_dict['default_front_end_target'] - platform.back_end = p_dict['default_back_end_target'] - platform.default = p_dict['default_target'] - platform.front_os = p_dict['default_front_end_os'] - platform.back_os = p_dict['default_back_end_os'] - platform.default_os = p_dict['default_os'] - - return platform + platform_path = spack.platform_path + mod_string = "spack.platforms" + + for p in list_modules(platform_path): + if platform_name == p: + mod_name = mod_string + platform_name + path = join_path(platform_path, platform_name) + ".py" + mod = imp.load_source(mod_name, path) + platform_class = mod_to_class(platform_name) + cls = getattr(mod, platform_class) + platform = cls() + return platform + return None + def arch_from_dict(d): """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict @@ -472,19 +454,30 @@ def arch_from_dict(d): else: if d is None: return None - platform_dict = d['platform'] - os_dict = d['platform_os'] - target_dict = d['target'] + platform_name = d['platform'] + os_name = d['platform_os'] + target_name = d['target'] - arch.platform = _platform_from_dict(platform_dict) if platform_dict else None - arch.target = _target_from_dict(target_dict) if os_dict else None - arch.platform_os = _operating_system_from_dict(os_dict) if os_dict else None + if platform_name: + arch.platform = _platform_from_dict(platform_name) + else: + arch.platform = None + if target_name: + arch.target = _target_from_dict(target_name, arch.platform) + else: + arch.target = None + if os_name: + arch.platform_os = _operating_system_from_dict(os_name, + arch.platform) + else: + arch.platform_os = None arch.os_string = None arch.target_string = None return arch + @memoized def all_platforms(): modules = [] @@ -507,6 +500,7 @@ def all_platforms(): return modules + @memoized def sys_type(): """ Gather a list of all available subclasses of platforms. -- cgit v1.2.3-60-g2f50 From 30c204a0a5ca116d2e6ab1ab177e103a5857ca4e Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 3 Jun 2016 16:16:59 -0700 Subject: Made a merge commit earlier with Todd fixing all_platforms, but changed how architecture looks in spec.yaml. Just three strings. Also made some flake8 formatting changes --- lib/spack/spack/architecture.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e74e72d6cd..818731cbf8 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -384,7 +384,6 @@ class Arch(object): target = self.target.name else: target = self.target - print (platform, platform_os, target) return (platform, platform_os, target) def to_dict(self): @@ -421,19 +420,10 @@ def _operating_system_from_dict(os_name, platform=None): def _platform_from_dict(platform_name): """ Constructs a platform from a dictionary. """ - platform_path = spack.platform_path - mod_string = "spack.platforms" - - for p in list_modules(platform_path): - if platform_name == p: - mod_name = mod_string + platform_name - path = join_path(platform_path, platform_name) + ".py" - mod = imp.load_source(mod_name, path) - platform_class = mod_to_class(platform_name) - cls = getattr(mod, platform_class) - platform = cls() - return platform - return None + platform_list = all_platforms() + for p in platform_list: + if platform_name.replace("_", "").lower() == p.__name__.lower(): + return p() def arch_from_dict(d): -- cgit v1.2.3-60-g2f50 From f751d681771499b1da46216f46beb0a074fa73e7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 09:03:32 -0700 Subject: Passing architecture to functions to find compiler. We can use the platform name if provided to find the operating system. Also changed error to operating system instead of target since operating system is now in charge of compiler finding. --- lib/spack/spack/compilers/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index ea79f655b0..d923d77da8 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -220,10 +220,11 @@ def find(compiler_spec, scope=None): @_auto_compiler_spec -def compilers_for_spec(compiler_spec, scope=None): +def compilers_for_spec(compiler_spec, scope=None, **kwargs): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ + platform = kwargs.get("platform", None) config = all_compilers_config(scope) def get_compilers(cspec): @@ -253,7 +254,7 @@ def compilers_for_spec(compiler_spec, scope=None): mods = [] if 'operating_system' in items: - operating_system = spack.architecture._operating_system_from_dict( items['operating_system'] ) + operating_system = spack.architecture._operating_system_from_dict(items['operating_system'], platform) else: operating_system = None @@ -275,12 +276,13 @@ def compilers_for_spec(compiler_spec, scope=None): @_auto_compiler_spec -def compiler_for_spec(compiler_spec, operating_system): +def compiler_for_spec(compiler_spec, arch): """Get the compiler that satisfies compiler_spec. compiler_spec must be concrete.""" + operating_system = arch.platform_os assert(compiler_spec.concrete) - compilers = [c for c in compilers_for_spec(compiler_spec) + compilers = [c for c in compilers_for_spec(compiler_spec, platform=arch.platform) if c.operating_system == operating_system] if len(compilers) < 1: raise NoCompilerForSpecError(compiler_spec, operating_system) @@ -334,7 +336,7 @@ class NoCompilersError(spack.error.SpackError): class NoCompilerForSpecError(spack.error.SpackError): def __init__(self, compiler_spec, target): - super(NoCompilerForSpecError, self).__init__("No compilers for target %s satisfy spec %s" % ( + super(NoCompilerForSpecError, self).__init__("No compilers for operating system %s satisfy spec %s" % ( target, compiler_spec)) class CompilerSpecInsufficientlySpecificError(spack.error.SpackError): -- cgit v1.2.3-60-g2f50 From c7b48f6fef92167b55d901e47a50e3f704fd65ec Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 09:04:24 -0700 Subject: Passing full arch for proper compiler spec so that we can use the platform to get the operating system --- lib/spack/spack/concretize.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 58811c3419..1f37455c77 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -280,10 +280,12 @@ class DefaultConcretizer(object): # Only use a matching compiler if it is of the proper style # Takes advantage of the proper logic already existing in compiler_for_spec # Should think whether this can be more efficient - def _proper_compiler_style(cspec, architecture): - compilers = spack.compilers.compilers_for_spec(cspec) - return filter(lambda c: c.operating_system == - architecture.platform_os, compilers) + def _proper_compiler_style(cspec, arch): + platform = arch.platform + compilers = spack.compilers.compilers_for_spec(cspec, + platform=platform) + return filter(lambda c: c.operating_system == + arch.platform_os, compilers) #return compilers @@ -368,7 +370,7 @@ class DefaultConcretizer(object): # Include the compiler flag defaults from the config files # This ensures that spack will detect conflicts that stem from a change # in default compiler flags. - compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture.platform_os) + compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture) for flag in compiler.flags: if flag not in spec.compiler_flags: spec.compiler_flags[flag] = compiler.flags[flag] -- cgit v1.2.3-60-g2f50 From 5b023bb0a1b2eef9d950cb660b466d2ab0e04349 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 09:05:14 -0700 Subject: Passing through full arch instead of just operating sysystem so that we can use the platform to get the operating system --- lib/spack/spack/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 0edb70e4ca..1a6c289bc7 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -682,7 +682,7 @@ class Package(object): if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") return spack.compilers.compiler_for_spec(self.spec.compiler, - self.spec.architecture.platform_os) + self.spec.architecture) def url_version(self, version): """ -- cgit v1.2.3-60-g2f50 From 864191b6ed9fbd95e549b2f4852dc92113c6a190 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 09:06:58 -0700 Subject: Got rid of unnecessary global keyword --- lib/spack/spack/test/mock_packages_test.py | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 561f319b16..9c586bd8ce 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -40,7 +40,6 @@ linux_os_name = 'debian' linux_os_version = '6' if platform.name == 'linux': - global linux_os_name, linux_os_version linux_os = platform.operating_system("default_os") linux_os_name = linux_os.name linux_os_version = linux_os.version -- cgit v1.2.3-60-g2f50 From 835982faedc0d728356600836caf2dd4690a9c5b Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 09:08:59 -0700 Subject: Changed OS name to CNL10 so that we properly find the compilers for the test platform --- lib/spack/spack/platforms/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/platforms/test.py b/lib/spack/spack/platforms/test.py index 3a4f9d5f2d..8fa2585a7a 100644 --- a/lib/spack/spack/platforms/test.py +++ b/lib/spack/spack/platforms/test.py @@ -10,8 +10,8 @@ class Test(Platform): back_end = 'x86_64' default = 'x86_64' - back_os = 'CNL' - default_os = 'CNL' + back_os = 'CNL10' + default_os = 'CNL10' def __init__(self): super(Test, self).__init__('test') -- cgit v1.2.3-60-g2f50 From 196737c2173a7b5748b3ad59e349611ec4f6b23c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 10:29:19 -0700 Subject: Changed to passing full architecture spec to function rather than just platform_os so we can grab the operating system from the platform getters --- lib/spack/spack/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 89c03a13b7..7718adb72c 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1124,7 +1124,7 @@ class Spec(object): for s in self.traverse(root=False): if s.external_module: - compiler = spack.compilers.compiler_for_spec(s.compiler, s.architecture.platform_os) + compiler = spack.compilers.compiler_for_spec(s.compiler, s.architecture) for mod in compiler.modules: load_module(mod) -- cgit v1.2.3-60-g2f50 From 4925be0bc4b63571eb9a9cb63e65409bebcc2c66 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 7 Jun 2016 13:22:47 -0700 Subject: Load external modules --- lib/spack/spack/build_environment.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 4483b9d132..7c65091d49 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -384,6 +384,13 @@ def parent_class_modules(cls): return result +def load_external_modules(pkg): + """ traverse the spec list and find any specs that have external modules. + """ + for dep in list(pkg.spec.traverse()): + if dep.external_module: + load_module(dep.external_module) + def setup_package(pkg): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() @@ -407,7 +414,7 @@ def setup_package(pkg): set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env) - + load_external_modules(pkg) # traverse in postorder so package can use vars from its dependencies spec = pkg.spec for dspec in pkg.spec.traverse(order='post', root=False): -- cgit v1.2.3-60-g2f50 From 5715799d4ee1c39d11106bb78a034e6affac474c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 7 Jun 2016 14:15:28 -0700 Subject: Added to_dict back into operating system --- lib/spack/spack/architecture.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 818731cbf8..1f25bc14ea 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -195,6 +195,7 @@ class Platform(object): name = self.back_os return self.operating_sys.get(name, None) + @classmethod def detect(self): @@ -333,6 +334,12 @@ class OperatingSystem(object): return list(compilers.values()) + def to_dict(self): + d = {} + d['name'] = self.name + d['version'] = self.version + + return d @key_ordering class Arch(object): @@ -410,7 +417,8 @@ def _operating_system_from_dict(os_name, platform=None): """ if not platform: platform = sys_type() - if isinstance(os_name, spack.util.spack_yaml.syaml_dict): + if isinstance(os_name, spack.util.spack_yaml.syaml_dict) or \ + isinstance(os_name, dict): name = os_name['name'] version = os_name['version'] return platform.operating_system(name+version) -- cgit v1.2.3-60-g2f50 From 992bcac7949883b74a9250d5835ac6d9de28a6c0 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 10 Jun 2016 16:26:02 -0700 Subject: changed compiler yaml format --- lib/spack/spack/compilers/__init__.py | 34 ++++++------ lib/spack/spack/config.py | 24 +++------ lib/spack/spack/test/config.py | 84 +++++++++++++----------------- lib/spack/spack/test/mock_packages_test.py | 72 +++++++++---------------- lib/spack/spack/yaml_version_check.py | 4 +- 5 files changed, 87 insertions(+), 131 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index d923d77da8..3cf8f2297f 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -73,16 +73,14 @@ def _to_dict(compiler): d = {} d['spec'] = str(compiler.spec) d['paths'] = dict( (attr, getattr(compiler, attr, None)) for attr in _path_instance_vars ) - d['operating_system'] = compiler.operating_system.to_dict() + d['operating_system'] = str(compiler.operating_system) d['modules'] = compiler.modules - if not compiler.alias: - yaml_text = yaml.dump( - d, default_flow_style=True, width=sys.maxint) - sha = hashlib.sha1(yaml_text) - compiler.alias = base64.b32encode(sha.digest()).lower()[:8] + if compiler.alias: + d['alias'] = compiler.alias + return { - compiler.alias: d + 'compiler': d } @@ -91,11 +89,11 @@ def get_compiler_config(scope=None): """ def init_compiler_config(): """Compiler search used when Spack has no compilers.""" - config = {} compilers = find_compilers() + compilers_dict = [] for compiler in compilers: - config.update(_to_dict(compiler)) - spack.config.update_config('compilers', config, scope=scope) + compilers_dict.append(_to_dict(compiler)) + spack.config.update_config('compilers', compilers_dict, scope=scope) config = spack.config.get_config('compilers', scope=scope) # Update the configuration if there are currently no compilers @@ -127,7 +125,7 @@ def add_compilers_to_config(compilers, scope=None): """ compiler_config = get_compiler_config(scope) for compiler in compilers: - compiler_config = _to_dict(compiler) + compiler_config.append(_to_dict(compiler)) spack.config.update_config('compilers', compiler_config, scope) @@ -167,8 +165,8 @@ def all_compilers_config(scope=None): def all_compilers(scope=None): # Return compiler specs from the merged config. - return [spack.spec.CompilerSpec(s['spec']) - for s in all_compilers_config(scope).values()] + return [spack.spec.CompilerSpec(s['compiler']['spec']) + for s in all_compilers_config(scope)] def default_compiler(): @@ -230,11 +228,10 @@ def compilers_for_spec(compiler_spec, scope=None, **kwargs): def get_compilers(cspec): compilers = [] - for aka, cmp in config.items(): - if cmp['spec'] != str(cspec): + for items in config: + if items['compiler']['spec'] != str(cspec): continue - items = cmp - alias = aka + items = items['compiler'] if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)): raise InvalidCompilerConfigurationError(cspec) @@ -258,6 +255,9 @@ def compilers_for_spec(compiler_spec, scope=None, **kwargs): else: operating_system = None + + alias = items['alias'] if 'alias' in items else None + flags = {} for f in spack.spec.FlagMap.valid_compiler_flags(): if f in items: diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index ec04c81787..e51016998c 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -146,11 +146,9 @@ section_schemas = { 'additionalProperties': False, 'patternProperties': { 'compilers:?': { # optional colon for overriding site config. - 'type': 'object', - 'default': {}, - 'additionalProperties': False, - 'patternProperties': { - r'\w[\w-]*': { # alias + 'type': 'array', + 'items': { + 'compiler': { 'type': 'object', 'additionalProperties': False, 'required': ['paths', 'spec', 'modules', 'operating_system'], @@ -180,15 +178,10 @@ section_schemas = { {'type' : 'null' }]}, 'ldlibs': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}}}, - 'spec': { 'type': 'string'},#r'\w[\w-]*@\w[\w-]*' - 'operating_system': { - 'type': 'object', - 'required': ['name', 'version'], - 'additionalProperties': False, - 'properties': { - 'name': {'type': 'string'}, - 'version': {'type': 'string'} - }}, + 'spec': { 'type': 'string'}, + 'operating_system': { 'type': 'string'}, + 'alias': { 'anyOf': [ {'type' : 'string'}, + {'type' : 'null' }]}, 'modules': { 'anyOf': [ {'type' : 'string'}, {'type' : 'null' }, {'type': 'array'}, @@ -591,8 +584,7 @@ def _merge_yaml(dest, source): # Source list is prepended (for precedence) if they_are(list): - seen = set(source) - dest[:] = source + [x for x in dest if x not in seen] + dest[:] = source + [x for x in dest if x not in source] return dest # Source dict is merged into dest. diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 8fffc09437..252d77e66b 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -32,8 +32,8 @@ from ordereddict_backport import OrderedDict from spack.test.mock_packages_test import * # Some sample compiler config data -a_comps = { - 'gcc473': { +a_comps = [ + {'compiler': { 'paths': { "cc" : "/gcc473", "cxx": "/g++473", @@ -42,12 +42,9 @@ a_comps = { }, 'modules': None, 'spec': 'gcc@4.7.3', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - }, - 'gcc450': { + 'operating_system': 'CNL10' + }}, + {'compiler': { 'paths': { "cc" : "/gcc450", "cxx": "/g++450", @@ -56,12 +53,9 @@ a_comps = { }, 'modules': None, 'spec': 'gcc@4.5.0', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - }, - 'clang33': { + 'operating_system': 'CNL10' + }}, + {'compiler': { 'paths': { "cc" : "", "cxx": "", @@ -69,15 +63,12 @@ a_comps = { "fc" : '' }, 'modules': None, 'spec': 'clang@3.3', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - } -} - -b_comps = { - 'icc100': { + 'operating_system': 'CNL10' + }} +] + +b_comps = [ + {'compiler': { 'paths': { "cc" : "/icc100", "cxx": "/icp100", @@ -86,12 +77,9 @@ b_comps = { }, 'modules': None, 'spec': 'icc@10.0', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - }, - 'icc111': { + 'operating_system': 'CNL10' + }}, + {'compiler': { 'paths': { "cc" : "/icc111", "cxx": "/icp111", @@ -100,12 +88,9 @@ b_comps = { }, 'modules': None, 'spec': 'icc@11.1', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - }, - 'clang33': { + 'operating_system': 'CNL10' + }}, + {'compiler': { 'paths': { "cc" : "", "cxx": "", @@ -113,12 +98,9 @@ b_comps = { "fc" : '' }, 'modules': None, 'spec': 'clang@3.3', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - } -} + 'operating_system': 'CNL10' + }} +] # Some Sample repo data repos_low = [ "/some/path" ] @@ -143,15 +125,21 @@ class ConfigTest(MockPackagesTest): config = spack.config.get_config('compilers') compiler_list = ['cc', 'cxx', 'f77', 'fc'] param_list = ['modules', 'paths', 'spec', 'operating_system'] - for alias, compiler in config.items(): - if compiler['spec'] in compiler_names: + for compiler in config: + conf = compiler['compiler'] + if conf['spec'] in compiler_names: + comp = None + for c in comps: + if c['compiler']['spec'] == conf['spec']: + comp = c['compiler'] + break + if not comp: + self.fail('Bad config spec') for p in param_list: - expected = comps[alias][p] - actual = config[alias][p] - self.assertEqual(expected, actual) + self.assertEqual(conf[p], comp[p]) for c in compiler_list: - expected = comps[alias]['paths'][c] - actual = config[alias]['paths'][c] + expected = comp['paths'][c] + actual = conf['paths'][c] self.assertEqual(expected, actual) def test_write_list_in_memory(self): diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 9c586bd8ce..a56bd8ebdc 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -46,132 +46,108 @@ if platform.name == 'linux': mock_compiler_config = """\ compilers: - clang3.3GENLINUX: +- compiler: spec: clang@3.3 - operating_system: - name: {0} - version: '{1}' + operating_system: {0}{1} paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None modules: 'None' - gcc4.5GENLINUX: +- compiler: spec: gcc@4.5.0 - operating_system: - name: {0} - version: '{1}' + operating_system: {0}{1} paths: cc: /path/to/gcc cxx: /path/to/g++ f77: None fc: None modules: 'None' - clang3.3CNL: +- compiler: spec: clang@3.3 - operating_system: - name: CNL - version: '10' + operating_system: CNL10 paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None modules: 'None' - clang3.3SUSE: +- compiler: spec: clang@3.3 - operating_system: - name: SuSE - version: '11' + operating_system: SuSE11 paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None modules: 'None' - clang3.3RHL: +- compiler: spec: clang@3.3 - operating_system: - name: redhat - version: '6' + operating_system: redhat6 paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None modules: 'None' - clang3.3OSX: +- compiler: spec: clang@3.3 - operating_system: - name: yosemite - version: '10.10' + operating_system: yosemite paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None modules: 'None' - gcc4.5.0CNL: +- compiler: paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - operating_system: - name: CNL - version: '10' + operating_system: CNL10 spec: gcc@4.5.0 modules: 'None' - gcc4.5.0SUSE: +- compiler: paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - operating_system: - name: SuSE - version: '11' + operating_system: SuSE11 spec: gcc@4.5.0 modules: 'None' - gcc4.5.0RHL: +- compiler: paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - operating_system: - name: redhat - version: '6' + operating_system: redhat6 spec: gcc@4.5.0 modules: 'None' - gcc4.5.0OSX: +- compiler: paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - operating_system: - name: yosemite - version: '10.10' + operating_system: yosemite spec: gcc@4.5.0 modules: 'None' - gcc4.5.0ELCAP: +- compiler: paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - operating_system: - name: elcapitan - version: '10.11' + operating_system: elcapitan spec: gcc@4.5.0 modules: 'None' - clang3.3ELCAP: +- compiler: spec: clang@3.3 - operating_system: - name: elcapitan - version: '10.11' + operating_system: elcapitan paths: cc: /path/to/clang cxx: /path/to/clang++ diff --git a/lib/spack/spack/yaml_version_check.py b/lib/spack/spack/yaml_version_check.py index 7e7bae4edf..5930eefafa 100644 --- a/lib/spack/spack/yaml_version_check.py +++ b/lib/spack/spack/yaml_version_check.py @@ -44,9 +44,9 @@ def check_compiler_yaml_version(): data = syaml.load(f) if data: - compilers = data['compilers'].items() + compilers = data['compilers'] if len(compilers) > 0: - if 'operating_system' not in compilers[0][1]: + if 'operating_system' not in compilers[0]['compiler']: new_file = os.path.join(scope.path, '_old_compilers.yaml') tty.warn('%s in out of date compilers format. ' 'Moved to %s. Spack automatically generate ' -- cgit v1.2.3-60-g2f50 From 2fc9ac4036eb017164cc6ac6f19e50dab5008759 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 15 Jun 2016 16:39:39 -0700 Subject: Two minor fixes --- lib/spack/spack/compilers/__init__.py | 10 +++++----- lib/spack/spack/yaml_version_check.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index ea79f655b0..c76dd252d0 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -74,7 +74,7 @@ def _to_dict(compiler): d['spec'] = str(compiler.spec) d['paths'] = dict( (attr, getattr(compiler, attr, None)) for attr in _path_instance_vars ) d['operating_system'] = compiler.operating_system.to_dict() - d['modules'] = compiler.modules + d['modules'] = compiler.modules if compiler.modules else [] if not compiler.alias: yaml_text = yaml.dump( @@ -141,11 +141,11 @@ def remove_compiler_from_config(compiler_spec, scope=None): - scope: configuration scope to modify. """ compiler_config = get_compiler_config(scope) - matches = [(a,c) for (a,c) in compiler_config.items() if c['spec'] == compiler_spec] + matches = [(a,c) for (a,c) in compiler_config.items() if c['spec'] == compiler_spec] if len(matches) == 1: del compiler_config[matches[0][0]] else: - CompilerSpecInsufficientlySpecificError(compiler_spec) + CompilerSpecInsufficientlySpecificError(compiler_spec) spack.config.update_config('compilers', compiler_config, scope) @@ -234,7 +234,7 @@ def compilers_for_spec(compiler_spec, scope=None): continue items = cmp alias = aka - + if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)): raise InvalidCompilerConfigurationError(cspec) @@ -309,7 +309,7 @@ def all_os_classes(): this platform """ classes = [] - + platform = spack.architecture.sys_type() for os_class in platform.operating_sys.values(): classes.append(os_class) diff --git a/lib/spack/spack/yaml_version_check.py b/lib/spack/spack/yaml_version_check.py index 7e7bae4edf..2d3d78ed39 100644 --- a/lib/spack/spack/yaml_version_check.py +++ b/lib/spack/spack/yaml_version_check.py @@ -46,10 +46,10 @@ def check_compiler_yaml_version(): if data: compilers = data['compilers'].items() if len(compilers) > 0: - if 'operating_system' not in compilers[0][1]: + if (not isinstance(compilers, list)) or 'operating_system' not in compilers[0][1]: new_file = os.path.join(scope.path, '_old_compilers.yaml') - tty.warn('%s in out of date compilers format. ' + tty.warn('%s in out of date compilers format. ' 'Moved to %s. Spack automatically generate ' - 'a compilers config file ' + 'a compilers config file ' % (file_name, new_file)) os.rename(file_name, new_file) -- cgit v1.2.3-60-g2f50 From c90cc465f581514d18fc4fc95649f015cb178079 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 00:54:59 -0700 Subject: Add `__contains__` to arch to satsify old packages. --- lib/spack/spack/architecture.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 1f25bc14ea..df0ab5c14d 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -195,7 +195,7 @@ class Platform(object): name = self.back_os return self.operating_sys.get(name, None) - + @classmethod def detect(self): @@ -206,12 +206,15 @@ class Platform(object): """ raise NotImplementedError() + def __repr__(self): return self.__str__() + def __str__(self): return self.name + def _cmp_key(self): t_keys = ''.join(str(t._cmp_key()) for t in sorted(self.targets.values())) @@ -338,7 +341,6 @@ class OperatingSystem(object): d = {} d['name'] = self.name d['version'] = self.version - return d @key_ordering @@ -378,6 +380,11 @@ class Arch(object): else: return '' + + def __contains__(self, string): + return string in str(self) + + def _cmp_key(self): if isinstance(self.platform, Platform): platform = self.platform.name -- cgit v1.2.3-60-g2f50 From f3d6818d5c09766946652266188c7210c56f027c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 00:55:39 -0700 Subject: be more tolerant when parsing new specs. --- lib/spack/spack/spec.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 7718adb72c..54219ec1b4 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -526,7 +526,12 @@ class Spec(object): """ valid_flags = FlagMap.valid_compiler_flags() if name == 'arch' or name == 'architecture': - platform, op_sys, target = value.split('-') + parts = value.split('-') + if len(parts) == 3: + platform, op_sys, target = parts + else: + platform, op_sys, target = None, None, value + assert(self.architecture.platform is None) assert(self.architecture.platform_os is None) assert(self.architecture.target is None) @@ -596,11 +601,13 @@ class Spec(object): def _set_os(self, value): """Called by the parser to set the architecture operating system""" - self.architecture.platform_os = self.architecture.platform.operating_system(value) + if self.architecture.platform: + self.architecture.platform_os = self.architecture.platform.operating_system(value) def _set_target(self, value): """Called by the parser to set the architecture target""" - self.architecture.target = self.architecture.platform.target(value) + if self.architecture.platform: + self.architecture.target = self.architecture.platform.target(value) def _add_dependency(self, spec): """Called by the parser to add another spec as a dependency.""" @@ -1063,7 +1070,7 @@ class Spec(object): feq(replacement.architecture, spec.architecture) and feq(replacement.dependencies, spec.dependencies) and feq(replacement.variants, spec.variants) and - feq(replacement.external, spec.external) and + feq(replacement.external, spec.external) and feq(replacement.external_module, spec.external_module)): continue # Refine this spec to the candidate. This uses @@ -1468,7 +1475,7 @@ class Spec(object): if self.architecture.target != other.architecture.target: raise UnsatisfiableArchitectureSpecError(self.architecture, other.architecture) - + changed = False if self.compiler is not None and other.compiler is not None: @@ -1619,7 +1626,7 @@ class Spec(object): # Architecture satisfaction is currently just string equality. # If not strict, None means unconstrained. if self.architecture and other.architecture: - if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or + if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or (self.architecture.platform_os and other.architecture.platform_os and self.architecture.platform_os != other.architecture.platform_os) or (self.architecture.target and other.architecture.target and self.architecture.target != other.architecture.target)): return False -- cgit v1.2.3-60-g2f50 From 88b671f8b1a82ec9d9365c296d77e251a821a66c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 02:06:19 -0700 Subject: Just call old os "unknown" --- lib/spack/spack/architecture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index df0ab5c14d..4485bbe840 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -451,7 +451,7 @@ def arch_from_dict(d): if isinstance(d, basestring): # We have an old spec using a string for the architecture arch.platform = Platform('spack_compatibility') - arch.platform_os = OperatingSystem('pre_version', '1.0') + arch.platform_os = OperatingSystem('unknown', '') arch.target = Target(d) arch.os_string = None -- cgit v1.2.3-60-g2f50 From 17b868381f95fa324f7c6327c0977b303975de76 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 02:55:33 -0700 Subject: Fixes #460: Do not show variants by default in spack find. This does two things: 1. By default `spack find` no longer shows variants. You have to supply `-v` to get that 2. This improves the `colify` implementation so that it no longer pads the rightmost column. This avoids the issue where if one spec was too long in the output, *all* specs would have space padding added to that width, and it would look like the output of `spack find` was double spaced. This no longer happens -- the one bad line wraps around and the other lines are now single-spaced when you use `-v` with boost. --- lib/spack/llnl/util/tty/colify.py | 9 +++++++-- lib/spack/spack/cmd/find.py | 19 +++++++++++++------ lib/spack/spack/cmd/uninstall.py | 13 ++++++++++--- 3 files changed, 30 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py index 429ba45882..81a83691d7 100644 --- a/lib/spack/llnl/util/tty/colify.py +++ b/lib/spack/llnl/util/tty/colify.py @@ -198,8 +198,13 @@ def colify(elts, **options): for col in xrange(cols): elt = col * rows + row width = config.widths[col] + cextra(elts[elt]) - fmt = '%%-%ds' % width - output.write(fmt % elts[elt]) + if col < cols - 1: + fmt = '%%-%ds' % width + output.write(fmt % elts[elt]) + else: + # Don't pad the rightmost column (sapces can wrap on + # small teriminals if one line is overlong) + output.write(elts[elt]) output.write("\n") row += 1 diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 9649bc7435..3ec671f93f 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -86,6 +86,11 @@ def setup_parser(subparser): action='store_true', dest='missing', help='Show missing dependencies as well as installed specs.') + subparser.add_argument( + '-v', '--variants', + action='store_true', + dest='variants', + help='Show variants in output (can be long)') subparser.add_argument('-M', '--only-missing', action='store_true', dest='only_missing', @@ -107,6 +112,8 @@ def display_specs(specs, **kwargs): mode = kwargs.get('mode', 'short') hashes = kwargs.get('long', False) namespace = kwargs.get('namespace', False) + flags = kwargs.get('show_flags', False) + variants = kwargs.get('variants', False) hlen = 7 if kwargs.get('very_long', False): @@ -114,10 +121,9 @@ def display_specs(specs, **kwargs): hlen = None nfmt = '.' if namespace else '_' - format_string = '$%s$@$+' % nfmt - flags = kwargs.get('show_flags', False) - if flags: - format_string = '$%s$@$%%+$+' % nfmt + ffmt = '$%+' if flags else '' + vfmt = '$+' if variants else '' + format_string = '$%s$@%s%s' % (nfmt, ffmt, vfmt) # Make a dict with specs keyed by architecture and compiler. index = index_by(specs, ('architecture', 'compiler')) @@ -163,7 +169,7 @@ def display_specs(specs, **kwargs): string = "" if hashes: string += gray_hash(s, hlen) + ' ' - string += s.format('$-%s$@$+' % nfmt, color=True) + string += s.format('$-%s$@%s' % (nfmt, vfmt), color=True) return string @@ -238,4 +244,5 @@ def find(parser, args): long=args.long, very_long=args.very_long, show_flags=args.show_flags, - namespace=args.namespace) + namespace=args.namespace, + variants=args.variants) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 9fdf3045b2..a6f08d09ed 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -39,6 +39,13 @@ error_message = """You can either: b) use spack uninstall -a to uninstall ALL matching specs. """ +# Arguments for display_specs when we find ambiguity +display_args = { + 'long': True, + 'show_flags': True, + 'variants':True +} + def ask_for_confirmation(message): while True: @@ -92,7 +99,7 @@ def concretize_specs(specs, allow_multiple_matches=False, force=False): if not allow_multiple_matches and len(matching) > 1: tty.error("%s matches multiple packages:" % spec) print() - display_specs(matching, long=True, show_flags=True) + display_specs(matching, **display_args) print() has_errors = True @@ -172,7 +179,7 @@ def uninstall(parser, args): tty.error("Will not uninstall %s" % spec.format("$_$@$%@$#", color=True)) print('') print("The following packages depend on it:") - display_specs(lst, long=True) + display_specs(lst, **display_args) print('') has_error = True elif args.dependents: @@ -186,7 +193,7 @@ def uninstall(parser, args): if not args.yes_to_all: tty.msg("The following packages will be uninstalled : ") print('') - display_specs(uninstall_list, long=True, show_flags=True) + display_specs(uninstall_list, **display_args) print('') ask_for_confirmation('Do you want to proceed ? ') -- cgit v1.2.3-60-g2f50