From e9f7d033ff04138905c99cb427e2071cc55cdc16 Mon Sep 17 00:00:00 2001 From: karenyyng Date: Sat, 3 Oct 2015 19:13:08 -0700 Subject: make sure submodule is added in the correct location --- lib/spack/spack/util/python_recipe_parser | 1 + 1 file changed, 1 insertion(+) create mode 160000 lib/spack/spack/util/python_recipe_parser (limited to 'lib') diff --git a/lib/spack/spack/util/python_recipe_parser b/lib/spack/spack/util/python_recipe_parser new file mode 160000 index 0000000000..6777dd98e7 --- /dev/null +++ b/lib/spack/spack/util/python_recipe_parser @@ -0,0 +1 @@ +Subproject commit 6777dd98e75229856d94da6671676b2a66e75986 -- cgit v1.2.3-70-g09d2 From f4e72f33c8ba988507aba667e318d1861c7b2b20 Mon Sep 17 00:00:00 2001 From: karenyyng Date: Mon, 5 Oct 2015 08:24:33 -0700 Subject: added dependencies for py-h5py --- lib/spack/spack/util/python_recipe_parser | 2 +- var/spack/packages/py-h5py/package.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/util/python_recipe_parser b/lib/spack/spack/util/python_recipe_parser index 6777dd98e7..437a62abb3 160000 --- a/lib/spack/spack/util/python_recipe_parser +++ b/lib/spack/spack/util/python_recipe_parser @@ -1 +1 @@ -Subproject commit 6777dd98e75229856d94da6671676b2a66e75986 +Subproject commit 437a62abb3df7212e3ee20269c0089a0a9766fe0 diff --git a/var/spack/packages/py-h5py/package.py b/var/spack/packages/py-h5py/package.py index 6293da5407..04072ca76b 100755 --- a/var/spack/packages/py-h5py/package.py +++ b/var/spack/packages/py-h5py/package.py @@ -13,6 +13,8 @@ class PyH5py(Package): depends_on('hdf5') depends_on('py-numpy') depends_on('py-cython') + depends_on('py-six') + depends_on('py-pkgconfig') def install(self, spec, prefix): python('setup.py', 'configure', '--hdf5=%s' % spec['hdf5'].prefix) -- cgit v1.2.3-70-g09d2 From b6d2a12ceb090692aa2be363b46d7f25486c0245 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 7 Oct 2015 15:57:29 -0700 Subject: Started changing the find in modules method written by Todd --- lib/spack/spack/compiler.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index e7d450ee8b..cbe6d8cd7d 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -272,9 +272,11 @@ class Compiler(object): if cls.PrgEnv: if not cls.PrgEnv_compiler: tty.die('Must supply PrgEnv_compiler with PrgEnv') - - output = _shell('module avail %s' % cls.PrgEnv_compiler) - matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) + + loaded_modules = os.environ["LOADEDMODULES"].split(":") + #output = _shell('module avail %s' % cls.PrgEnv_compiler) + for module in loaded_modules: + match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) for name, version in matches: v = version + '-craype' -- cgit v1.2.3-70-g09d2 From 4f21344e87da327b4166fff1fd2ce32afaa07dbc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 14 Oct 2015 19:41:07 -0700 Subject: Started created the Architecture class for Spack to use --- lib/spack/spack/architecture.py | 73 ++++++++++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 0c4b605e91..829aaa4c1c 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -30,7 +30,7 @@ from llnl.util.lang import memoized import spack import spack.error as serr from spack.version import Version - +from external import yaml class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): @@ -43,41 +43,81 @@ class NoSysTypeError(serr.SpackError): super(NoSysTypeError, self).__init__( "Could not determine sys_type for this machine.") +class Architecture(object): + def __init__(self, *arch_name): + + """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) and a strategy for + searching for that architecture's compiler. The target passed to it should be a dictionary of names and strategies. + """ + self.arch_dict = {} + self.arch_name = arch_name + + def add_arch_strategy(self): + """ Create a dictionary using the tuples of arch_names""" + for n in self.arch_name: + if 'cray' in n.lower(): + self.arch_dict[n] = "MODULES" + if 'linux' in n.lower() or 'x86_64' in n.lower(): + self.arch_dict[n] = "PATH" + else: + self.arch_dict[n] = None def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" + """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" if not hasattr(spack, "sys_type"): - return None + return None elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() + return Architecture(spack.sys_type()) else: - return spack.sys_type - - -def get_sys_type_from_environment(): - """Return $SYS_TYPE or None if it's not defined.""" - return os.environ.get('SYS_TYPE') + return Architecture(spack.sys_type) +# This is livermore dependent. Hard coded for livermore +#def get_sys_type_from_environment(): +# """Return $SYS_TYPE or None if it's not defined.""" +# return os.environ.get('SYS_TYPE') def get_mac_sys_type(): - """Return a Mac OS SYS_TYPE or None if this isn't a mac.""" + """Return a Mac OS SYS_TYPE or None if this isn't a mac. + Front-end config + """ + mac_ver = py_platform.mac_ver()[0] if not mac_ver: return None - return "macosx_%s_%s" % ( - Version(mac_ver).up_to(2), py_platform.machine()) - + return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) + +def get_sys_type_from_uname(): + """ Returns a sys_type from the uname argument + Front-end config + """ + return Architecture(os.uname()[0] + " " + os.uname()[-1]) + +def get_sys_type_from_config_file(): + """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since + The user can specify that the architecture is a cray-xc40 + """ + + home_dir = os.environ["HOME"] + yaml_file = os.path.join(home_dir, ".spack/architecture.yaml") + if os.path.isfile(yaml_file): + with open(yaml_file) as config: + config_dict = config['architecture'] + front_end = config_dict['front'] + back_end = config_dict['back'] + return Architecture(front_end) @memoized def sys_type(): - """Returns a SysType for the current machine.""" + """Returns a SysType for the current machine. Should return output to an + Architecture class + """ methods = [get_sys_type_from_spack_globals, get_sys_type_from_environment, get_mac_sys_type] # search for a method that doesn't return None - sys_type = None + sys_type = (None,None) for method in methods: sys_type = method() if sys_type: break @@ -90,3 +130,4 @@ def sys_type(): raise InvalidSysTypeError(sys_type) return sys_type + -- cgit v1.2.3-70-g09d2 From 29e03ac851f807e9fb5aefb687902e703d208615 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 12:46:44 -0700 Subject: Added __eq__ testing method. Created tests for it in test/ folder --- lib/spack/spack/architecture.py | 56 +++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 829aaa4c1c..97c2cdbd6d 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -57,10 +57,20 @@ class Architecture(object): for n in self.arch_name: if 'cray' in n.lower(): self.arch_dict[n] = "MODULES" - if 'linux' in n.lower() or 'x86_64' in n.lower(): + elif 'linux' in n.lower() or 'x86_64' in n.lower(): self.arch_dict[n] = "PATH" else: self.arch_dict[n] = None + + def get_arch_dict(self): + """ Grab the dictionary from the Architecture class, rather than access the internal Architecture attributes """ + return self.arch_dict + + def __eq__(self, other): + if self.arch_dict != {} and other.arch_dict != {}: + return self.arch_dict == other.arch_dict + else: + return self.arch_name == self.arch_name def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" @@ -95,29 +105,43 @@ def get_sys_type_from_uname(): def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since - The user can specify that the architecture is a cray-xc40 + The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack + is installed. Similar to .spackconfig """ - home_dir = os.environ["HOME"] - yaml_file = os.path.join(home_dir, ".spack/architecture.yaml") - if os.path.isfile(yaml_file): - with open(yaml_file) as config: - config_dict = config['architecture'] - front_end = config_dict['front'] - back_end = config_dict['back'] - return Architecture(front_end) + spack_home_dir = os.environ["HOME"] + "/.spack" + yaml_file = os.path.join(spack_home_dir, "architecture.yaml") + + try: + config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() + arch = config_dict['architecture'] + front = arch['front'] + back = arch['back'] + + except: + print "No architecture.yaml config file found" + + return Architecture(front,back) @memoized def sys_type(): - """Returns a SysType for the current machine. Should return output to an - Architecture class + """Priority of gathering sys-type. + 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 + 2. UNAME + 3. GLOBALS + 4. MAC OSX + Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. + If there is no yaml present then it should move on to the next function and stop immediately once it gets a + arch name + """ - methods = [get_sys_type_from_spack_globals, - get_sys_type_from_environment, - get_mac_sys_type] + methods = [get_sys_type_from_config_file, + get_sys_type_from_uname, + get_sys_type_from_spack_globals, + get_mac_sys_type] # search for a method that doesn't return None - sys_type = (None,None) + sys_type = None for method in methods: sys_type = method() if sys_type: break -- cgit v1.2.3-70-g09d2 From d328f4c3b631e59ed14f52cf38f3d05774100c15 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 12:47:26 -0700 Subject: Test suite for architecture class and functions --- lib/spack/spack/test/architecture.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 lib/spack/spack/test/architecture.py (limited to 'lib') diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py new file mode 100644 index 0000000000..6ff22aaa59 --- /dev/null +++ b/lib/spack/spack/test/architecture.py @@ -0,0 +1,19 @@ +""" Test checks if the architecture class is created correctly and also that + the functions are looking for the correct architecture name +""" +import unittest +import spack +from spack.architecture import * + +class ArchitectureTest(unittest.TestCase): + + def test_Architecture_class(self): + a = Architecture('Cray-XC40') + a.add_arch_strategy() + self.assertEquals(a.get_arch_dict(), {'Cray-XC40': 'MODULES'}) + + def test_get_sys_type_from_config_file(self): + output_arch_class = get_sys_type_from_config_file() + my_arch_class = Architecture('Linux x86_64','Cray-xc40') + + self.assertEqual(output_arch_class, my_arch_class) -- cgit v1.2.3-70-g09d2 From ccdf1057592b48e13ba3b98df9972324e91d0be8 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 12:48:12 -0700 Subject: Commented out a long list of tests to just include my arch test --- lib/spack/spack/test/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 6b3715be6f..ed51fac33a 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -31,7 +31,8 @@ from llnl.util.tty.colify import colify import spack """Names of tests to be included in Spack's test suite""" -test_names = ['versions', +"""test_names = ['architecture', + 'versions', 'url_parse', 'url_substitution', 'packages', @@ -57,7 +58,8 @@ test_names = ['versions', 'optional_deps', 'make_executable', 'configure_guess'] - +""" +test_names = ['architecture'] def list_tests(): """Return names of all tests that can be run for Spack.""" -- cgit v1.2.3-70-g09d2 From fec197ccac94af485745b6b40d8150b4ae030e99 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 15:25:13 -0700 Subject: Fixed the output of sys_type(), might need to add back the error handling part --- lib/spack/spack/architecture.py | 47 ++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 97c2cdbd6d..8f74fbc2e1 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -60,7 +60,7 @@ class Architecture(object): elif 'linux' in n.lower() or 'x86_64' in n.lower(): self.arch_dict[n] = "PATH" else: - self.arch_dict[n] = None + self.arch_dict[n] = "" def get_arch_dict(self): """ Grab the dictionary from the Architecture class, rather than access the internal Architecture attributes """ @@ -72,6 +72,7 @@ class Architecture(object): else: return self.arch_name == self.arch_name + def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" if not hasattr(spack, "sys_type"): @@ -86,6 +87,7 @@ def get_sys_type_from_spack_globals(): # """Return $SYS_TYPE or None if it's not defined.""" # return os.environ.get('SYS_TYPE') + def get_mac_sys_type(): """Return a Mac OS SYS_TYPE or None if this isn't a mac. Front-end config @@ -97,12 +99,14 @@ def get_mac_sys_type(): return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) + def get_sys_type_from_uname(): """ Returns a sys_type from the uname argument Front-end config """ return Architecture(os.uname()[0] + " " + os.uname()[-1]) + def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack @@ -110,21 +114,22 @@ def get_sys_type_from_config_file(): """ spack_home_dir = os.environ["HOME"] + "/.spack" - yaml_file = os.path.join(spack_home_dir, "architecture.yaml") + yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') try: config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() arch = config_dict['architecture'] front = arch['front'] back = arch['back'] + return Architecture(front,back) except: print "No architecture.yaml config file found" - - return Architecture(front,back) + return None + @memoized -def sys_type(): +def sys_type(): # This function is going to give me issues isn't it?? """Priority of gathering sys-type. 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 2. UNAME @@ -133,25 +138,19 @@ def sys_type(): Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. If there is no yaml present then it should move on to the next function and stop immediately once it gets a arch name - """ - methods = [get_sys_type_from_config_file, - get_sys_type_from_uname, - get_sys_type_from_spack_globals, - get_mac_sys_type] - - # search for a method that doesn't return None - sys_type = None - for method in methods: - sys_type = method() - if sys_type: break - - # Couldn't determine the sys_type for this machine. - if sys_type is None: - return "unknown_arch" - - if not isinstance(sys_type, basestring): - raise InvalidSysTypeError(sys_type) - + # Try to create an architecture object using the config file FIRST + functions = [get_sys_type_from_config_file, + get_sys_type_from_uname, + get_sys_type_from_spack_globals, + get_mac_sys_type] + + # TODO: Test for mac OSX system type but I'm sure it will be okay + for func in functions: + sys_type = None + sys_type = func() + if sys_type: + break + return sys_type -- cgit v1.2.3-70-g09d2 From 3ba2842b53752e9a3ec8766f3a1350c6cc7577a2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 20 Oct 2015 13:33:21 -0700 Subject: Got Architecture class working the way i wanted to. Next to write tests --- lib/spack/spack/architecture.py | 70 ++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 8f74fbc2e1..1a9f9de2cb 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -34,47 +34,50 @@ from external import yaml class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__( - "Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__( - "Could not determine sys_type for this machine.") + super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") + class Architecture(object): - def __init__(self, *arch_name): + """ Architecture class that contains a dictionary of architecture name and compiler search strategy methods. + The idea is to create an object that Spack can interact with and know how to search for the compiler + If it is on a Cray architecture it should look in modules. If it is anything else search $PATH. + """ + + def __init__(self, front=None, back=None): - """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) and a strategy for - searching for that architecture's compiler. The target passed to it should be a dictionary of names and strategies. + """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) + and a strategy for searching for that architecture's compiler. + The target passed to it should be a dictionary of names and strategies. """ - self.arch_dict = {} - self.arch_name = arch_name - - def add_arch_strategy(self): - """ Create a dictionary using the tuples of arch_names""" - for n in self.arch_name: - if 'cray' in n.lower(): - self.arch_dict[n] = "MODULES" - elif 'linux' in n.lower() or 'x86_64' in n.lower(): - self.arch_dict[n] = "PATH" - else: - self.arch_dict[n] = "" - - def get_arch_dict(self): - """ Grab the dictionary from the Architecture class, rather than access the internal Architecture attributes """ - return self.arch_dict - - def __eq__(self, other): - if self.arch_dict != {} and other.arch_dict != {}: - return self.arch_dict == other.arch_dict - else: - return self.arch_name == self.arch_name - + names = [] + names.append(front) + names.append(back) + + def add_compiler_strategy(names): + """ Create a dictionary of {'arch-name': 'strategy'} + This will tell Spack whether to look in the $PATH + or $MODULES location for compilers + """ + d = {} + for n in names: + if n: + if 'cray' in n.lower(): + d[n] = "MODULES" + elif 'linux' in n.lower(): + d[n] = "PATH" + else: + d[n] = 'No Strategy' + return d + + self.arch_dict = add_compiler_strategy(names) -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" +def get_sys_type_from_spack_globals(): #TODO: Figure out how this function works + """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): return None elif hasattr(spack.sys_type, "__call__"): @@ -104,7 +107,7 @@ def get_sys_type_from_uname(): """ Returns a sys_type from the uname argument Front-end config """ - return Architecture(os.uname()[0] + " " + os.uname()[-1]) + return Architecture(os.uname()[0]) def get_sys_type_from_config_file(): @@ -149,6 +152,7 @@ def sys_type(): # This function is going to give me issues isn't it?? for func in functions: sys_type = None sys_type = func() + if sys_type: break -- cgit v1.2.3-70-g09d2 From ca3cc5b23e94b8168ba768ad343822b39aac4a25 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 20 Oct 2015 13:37:06 -0700 Subject: Refactored architecture class. Now it will automagically create a dict upon instantiation. --- lib/spack/spack/architecture.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 1a9f9de2cb..760c9cddd1 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -49,7 +49,6 @@ class Architecture(object): """ def __init__(self, front=None, back=None): - """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) and a strategy for searching for that architecture's compiler. The target passed to it should be a dictionary of names and strategies. @@ -63,6 +62,7 @@ class Architecture(object): This will tell Spack whether to look in the $PATH or $MODULES location for compilers """ + #TODO: Look for other strategies d = {} for n in names: if n: @@ -76,21 +76,20 @@ class Architecture(object): self.arch_dict = add_compiler_strategy(names) -def get_sys_type_from_spack_globals(): #TODO: Figure out how this function works +def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): return None elif hasattr(spack.sys_type, "__call__"): - return Architecture(spack.sys_type()) + return Architecture(spack.sys_type()) #If in __init__.py there is a sys_type() then call that else: - return Architecture(spack.sys_type) + return Architecture(spack.sys_type) # Else use the attributed which defaults to None # This is livermore dependent. Hard coded for livermore #def get_sys_type_from_environment(): # """Return $SYS_TYPE or None if it's not defined.""" # return os.environ.get('SYS_TYPE') - def get_mac_sys_type(): """Return a Mac OS SYS_TYPE or None if this isn't a mac. Front-end config -- cgit v1.2.3-70-g09d2 From c31da9bc8f10c3a74b3bf26c2dbac9a13c073eba Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 20 Oct 2015 14:10:54 -0700 Subject: Made sure architecture works with yaml file --- lib/spack/spack/architecture.py | 46 ++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 760c9cddd1..8460b0de1e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -49,18 +49,24 @@ class Architecture(object): """ def __init__(self, front=None, back=None): - """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) - and a strategy for searching for that architecture's compiler. - The target passed to it should be a dictionary of names and strategies. + """ Constructor for the architecture class. It will create a list from the given arguments and iterate + through that list. It will then create a dictionary of {arch_name : strategy} + Takes in two parameters: + + front = None defaults to None. Should be the front-end architecture of the machine + back = None defaults to None. Should be the back-end architecture of the machine + + If no arguments are given it will return an empty dictionary """ - names = [] - names.append(front) - names.append(back) + _names = [] + _names.append(front) + _names.append(back) - def add_compiler_strategy(names): + def _add_compiler_strategy(names): """ Create a dictionary of {'arch-name': 'strategy'} This will tell Spack whether to look in the $PATH or $MODULES location for compilers + Else it will return No Strategy """ #TODO: Look for other strategies d = {} @@ -74,7 +80,7 @@ class Architecture(object): d[n] = 'No Strategy' return d - self.arch_dict = add_compiler_strategy(names) + self.arch_dict = _add_compiler_strategy(_names) def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" @@ -85,20 +91,20 @@ def get_sys_type_from_spack_globals(): else: return Architecture(spack.sys_type) # Else use the attributed which defaults to None + # This is livermore dependent. Hard coded for livermore #def get_sys_type_from_environment(): # """Return $SYS_TYPE or None if it's not defined.""" # return os.environ.get('SYS_TYPE') + def get_mac_sys_type(): """Return a Mac OS SYS_TYPE or None if this isn't a mac. Front-end config """ - mac_ver = py_platform.mac_ver()[0] if not mac_ver: return None - return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) @@ -113,11 +119,9 @@ def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack is installed. Similar to .spackconfig - """ - + """ spack_home_dir = os.environ["HOME"] + "/.spack" yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') - try: config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() arch = config_dict['architecture'] @@ -131,7 +135,7 @@ def get_sys_type_from_config_file(): @memoized -def sys_type(): # This function is going to give me issues isn't it?? +def sys_type(): """Priority of gathering sys-type. 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 2. UNAME @@ -146,14 +150,18 @@ def sys_type(): # This function is going to give me issues isn't it?? get_sys_type_from_uname, get_sys_type_from_spack_globals, get_mac_sys_type] - - # TODO: Test for mac OSX system type but I'm sure it will be okay + + sys_type = None for func in functions: - sys_type = None sys_type = func() - if sys_type: - break + break + if sys_type is None: + return Architecture("unknown_arch") + + if not isinstance(sys_type, Architecture): + raise InvalidSysTypeError(sys_type) + return sys_type -- cgit v1.2.3-70-g09d2 From 9b387e7682c22b74661e3363442adcfb569d8680 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 21 Oct 2015 11:09:05 -0700 Subject: Added strategy method in init --- lib/spack/spack/architecture.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 8460b0de1e..3968b82124 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -57,18 +57,13 @@ class Architecture(object): back = None defaults to None. Should be the back-end architecture of the machine If no arguments are given it will return an empty dictionary + Uses the _add_compiler_strategy(front, back) to create the dictionary """ - _names = [] - _names.append(front) - _names.append(back) - - def _add_compiler_strategy(names): - """ Create a dictionary of {'arch-name': 'strategy'} - This will tell Spack whether to look in the $PATH - or $MODULES location for compilers - Else it will return No Strategy - """ - #TODO: Look for other strategies + + def _add_compiler_strategy(front,back): + names = [] + names.append(front) + names.append(back) d = {} for n in names: if n: @@ -80,7 +75,8 @@ class Architecture(object): d[n] = 'No Strategy' return d - self.arch_dict = _add_compiler_strategy(_names) + self.arch_dict = _add_compiler_strategy(front, back) + def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" -- cgit v1.2.3-70-g09d2 From a89abb435f288c2864d9a21089b20f6eaac5d98b Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 21 Oct 2015 11:32:59 -0700 Subject: Changed structure of class, add compiler strategy is a method and can create a dict --- lib/spack/spack/architecture.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3968b82124..7c1bdfb20f 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -48,6 +48,21 @@ class Architecture(object): If it is on a Cray architecture it should look in modules. If it is anything else search $PATH. """ + def add_compiler_strategy(self, front,back): + names = [] + names.append(front) + names.append(back) + d = {} + for n in names: + if n: + if 'cray' in n.lower(): + d[n] = "MODULES" + elif 'linux' in n.lower(): + d[n] = "PATH" + else: + d[n] = 'No Strategy' + return d + def __init__(self, front=None, back=None): """ Constructor for the architecture class. It will create a list from the given arguments and iterate through that list. It will then create a dictionary of {arch_name : strategy} @@ -59,23 +74,9 @@ class Architecture(object): If no arguments are given it will return an empty dictionary Uses the _add_compiler_strategy(front, back) to create the dictionary """ - - def _add_compiler_strategy(front,back): - names = [] - names.append(front) - names.append(back) - d = {} - for n in names: - if n: - if 'cray' in n.lower(): - d[n] = "MODULES" - elif 'linux' in n.lower(): - d[n] = "PATH" - else: - d[n] = 'No Strategy' - return d - - self.arch_dict = _add_compiler_strategy(front, back) + self.front = front + self.back = back + self.arch_dict = self.add_compiler_strategy(front, back) def get_sys_type_from_spack_globals(): -- cgit v1.2.3-70-g09d2 From 38508c5a3f294f730b31e209dfa90545e1920588 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 22 Oct 2015 12:02:26 -0700 Subject: Created a control flow logic that will loop through the strategies and find compilers using that. TODO: Need to find a way to locate their executables --- lib/spack/spack/cmd/compiler.py | 51 +++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 2a64dc914e..cbe3aba864 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -29,6 +29,8 @@ from llnl.util.tty.color import colorize from llnl.util.tty.colify import colify from llnl.util.lang import index_by +import spack.architecture +import spack.compiler import spack.compilers import spack.spec import spack.config @@ -36,13 +38,12 @@ from spack.util.environment import get_path from spack.spec import CompilerSpec description = "Manage compilers" +ARCHITECTURE = spack.architecture.sys_type() def setup_parser(subparser): - sp = subparser.add_subparsers( - metavar='SUBCOMMAND', dest='compiler_command') + sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='compiler_command') - update_parser = sp.add_parser( - 'add', help='Add compilers to the Spack configuration.') + update_parser = sp.add_parser('add', help='Add compilers to the Spack configuration.') update_parser.add_argument('add_paths', nargs=argparse.REMAINDER) remove_parser = sp.add_parser('remove', help='remove compiler') @@ -55,23 +56,33 @@ def setup_parser(subparser): def compiler_add(args): - """Search either $PATH or a list of paths for compilers and add them + """Search either $PATH or a list of paths OR MODULES for compilers and add them to Spack's configuration.""" - paths = args.add_paths - if not paths: - paths = get_path('PATH') - - compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) - if c.spec not in spack.compilers.all_compilers()] - - if compilers: - spack.compilers.add_compilers_to_config('user', *compilers) - n = len(compilers) - tty.msg("Added %d new compiler%s to %s" % ( - n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) - colify(reversed(sorted(c.spec for c in compilers)), indent=4) - else: - tty.msg("Found no new compilers") + + strategies = ARCHITECTURE.strategy() + + for strategy in strategies: + if strategy == 'PATH': + paths = args.add_paths # This might be a parser method. Parsing method to add_paths + if not paths: + paths = get_path('PATH') + + compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) + if c.spec not in spack.compilers.all_compilers()] + + elif strategy == "MODULES": + from spack.compilers.cray import Cray + compilers = Cray.find_in_modules() + #TODO: Find a way to locate the executables + + if compilers: + spack.compilers.add_compilers_to_config('user', *compilers) + n = len(compilers) + tty.msg("Added %d new compiler%s to %s" % ( + n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) + colify(reversed(sorted(c.spec for c in compilers)), indent=4) + else: + tty.msg("Found no new compilers") def compiler_remove(args): -- cgit v1.2.3-70-g09d2 From 09597fe8dccbd7d49acf1b3198f24ab928874cdb Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 09:44:28 -0700 Subject: updated the executible to return stderr when specified. Added load_module to build_environment.py, loads target --- lib/spack/spack/build_environment.py | 36 +++++++++++++++++++++++++++++------- lib/spack/spack/util/executable.py | 21 ++++++++++++++++----- 2 files changed, 45 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 68388958f5..191e858735 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -86,6 +86,28 @@ class MakeExecutable(Executable): return super(MakeExecutable, self).__call__(*args, **kwargs) +def load_module(mod): + """Takes a module name and removes modules until it is possible to + load that module. It then loads the provided module. Depends on the + modulecmd implementation of modules used in cray and lmod. + """ + #Create an executable of the module command that will output python code + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module and remove any conflicting modules + # We do this without checking that they are already installed + # for ease of programming because unloading a module that is not + # loaded does nothing. + text = modulecmd('show', mod, return_oe=True).split() + for i, word in enumerate(text): + if word == 'conflict': + exec(compile(modulecmd('unload', text[i+1], return_oe=True), '', 'exec')) + # Load the module now that there are no conflicts + load = modulecmd('load', mod, return_oe=True) + exec(compile(load, '', 'exec')) + + def set_compiler_environment_variables(pkg): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -108,11 +130,9 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) - if compiler.PrgEnv: - os.environ['SPACK_CRAYPE'] = compiler.PrgEnv - os.environ['SPACK_COMP_MODULE'] = compiler.module - - + if compiler.modules: + for mod in compiler.modules: + load_module(mod) def set_build_environment_variables(pkg): @@ -163,8 +183,10 @@ def set_build_environment_variables(pkg): pcdir = join_path(p, libdir, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) - path_set("PKG_CONFIG_PATH", pkg_config_dirs) + path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) + if pkg.spec.architecture.compiler_strategy.lower() == 'module': + load_module(pkg.spec.architecture.module_name) def set_module_variables_for_package(pkg): """Populate the module scope of install() with some useful functions. @@ -239,8 +261,8 @@ def get_rpaths(pkg): def setup_package(pkg): """Execute all environment setup routines.""" - set_compiler_environment_variables(pkg) set_build_environment_variables(pkg) + set_compiler_environment_variables(pkg) set_module_variables_for_package(pkg) # Allow dependencies to set up environment as well. diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index d1dfb62ffb..15354089ac 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -56,7 +56,11 @@ class Executable(object): def __call__(self, *args, **kwargs): """Run the executable with subprocess.check_output, return output.""" - return_output = kwargs.get("return_output", False) + # Return oe returns a combined stream, setting both output and error + # without setting return oe returns them concatenated by a double line break + return_oe = kwargs.get("return_oe", False) + return_output = True if return_oe else kwargs.get("return_output", False) + return_error = True if return_oe else kwargs.get("return_error", False) fail_on_error = kwargs.get("fail_on_error", True) ignore_errors = kwargs.get("ignore_errors", ()) @@ -95,8 +99,8 @@ class Executable(object): proc = subprocess.Popen( cmd, stdin=input, - stderr=error, - stdout=subprocess.PIPE if return_output else output) + stdout=subprocess.PIPE if return_output else output, + stderr=subprocess.STDOUT if return_oe else (subprocess.PIPE if return_error else error)) out, err = proc.communicate() self.returncode = proc.returncode @@ -104,8 +108,15 @@ class Executable(object): if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:" % proc.returncode, cmd_line) - if return_output: - return out + # Return out or error if specified. Return combined stream if requested, + # otherwise return them concatenated by double line break if both requested. + if return_output or return_error: + if return_oe or not return_error: + return out + elif return_output: + return out+'\n\n'+err + else: + return err except OSError, e: raise ProcessError( -- cgit v1.2.3-70-g09d2 From 382d8478bf325099582b4814e0d71ff8522696aa Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 10:23:33 -0700 Subject: prototype of new architecture concretization saved as new_concretize_architecture in concretize.py --- lib/spack/spack/concretize.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 66002492cb..c5041d67be 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -122,6 +122,34 @@ class DefaultConcretizer(object): return True # changed + def new_concretize_architecture(self, spec): + """If the spec already has an architecture and it is a an architecture type, + return. Otherwise, if it has an architecture that is a string type, generate an + architecture based on that type. If it has no architecture and the root of the + DAG has an architecture, then use that. Otherwise, take the system's default + architecture. + """ + if spec.architecture is not None: + if isinstance(spec.architecture,spack.architecture.Target): + return False + else: + arch = spack.architecture.sys_type() + spec.architecture = arch.target(spec.architecture) + return True #changed + + if spec.root.architecture: + if isinstance(spec.root.architecture,spack.architecture.Target): + spec.architecture = spec.root.architecture + else: + arch = spack.architecture.sys_type() + spec.architecture = arch.target(spec.root.architecture) + else: + arch = spack.architecture.sys_type() + spec.architecture = arch.target('default') + + return True #changed + + def concretize_variants(self, spec): """If the spec already has variants filled in, return. Otherwise, add the default variants from the package specification. -- cgit v1.2.3-70-g09d2 From 7ab921ff02a7d0a05287e4a49738ec9c51361ef0 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 14:46:26 -0700 Subject: Changed architecture class and added class Target --- lib/spack/spack/architecture.py | 137 ++++++++++++++++++++++++---------------- 1 file changed, 83 insertions(+), 54 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 7c1bdfb20f..ea1f98e06e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -23,11 +23,16 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import imp import platform as py_platform +import inspect -from llnl.util.lang import memoized +from llnl.util.lang import memoized, list_modules +from llnl.util.filesystem import join_path +import llnl.util.tty as tty import spack +from spack.util.naming import mod_to_class import spack.error as serr from spack.version import Version from external import yaml @@ -42,41 +47,55 @@ class NoSysTypeError(serr.SpackError): super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") +class Target(object): + """ This is the processor type e.g. cray-ivybridge """ + # Front end or back end target. Target needs to know which one this is + # Should autodetect from the machine + # features of a target + # - a module name + # -a compiler finding strategy + # -a name + # architecture classes handling the aliasing for front-end, back-end and default + + def __init__(self,name, module_name=None): + self.name = name # case of cray "ivybridge but if it's x86_64 + self.module_name = module_name # craype-ivybridge + + def compiler_strategy(self): + if self.module_name: # If there is a module_name given then use MODULES + return "MODULES" + else: + return "PATH" + class Architecture(object): - """ Architecture class that contains a dictionary of architecture name and compiler search strategy methods. - The idea is to create an object that Spack can interact with and know how to search for the compiler - If it is on a Cray architecture it should look in modules. If it is anything else search $PATH. + """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it + is returned """ - - def add_compiler_strategy(self, front,back): - names = [] - names.append(front) - names.append(back) - d = {} - for n in names: - if n: - if 'cray' in n.lower(): - d[n] = "MODULES" - elif 'linux' in n.lower(): - d[n] = "PATH" - else: - d[n] = 'No Strategy' - return d - - def __init__(self, front=None, back=None): - """ Constructor for the architecture class. It will create a list from the given arguments and iterate - through that list. It will then create a dictionary of {arch_name : strategy} - Takes in two parameters: - front = None defaults to None. Should be the front-end architecture of the machine - back = None defaults to None. Should be the back-end architecture of the machine + priority = None # Subclass needs to set this number. This controls order in which arch is detected. + front = None + back = None + default_front = None # The default front end target. On cray sandybridge + default_back = None # The default back end target. On cray ivybridge + + def __init__(self, name): + self.targets = {} + self.name = name + + def add_target(self, name, target): + self.targets[name] = target - If no arguments are given it will return an empty dictionary - Uses the _add_compiler_strategy(front, back) to create the dictionary + + @classmethod + def detect(self): + """ Subclass is responsible for implementing this method. + Returns True if the architecture detects if it is the current architecture + and False if it's not. """ - self.front = front - self.back = back - self.arch_dict = self.add_compiler_strategy(front, back) + raise NotImplementedError() + + def __str__(self): + return self.name def get_sys_type_from_spack_globals(): @@ -84,9 +103,9 @@ def get_sys_type_from_spack_globals(): if not hasattr(spack, "sys_type"): return None elif hasattr(spack.sys_type, "__call__"): - return Architecture(spack.sys_type()) #If in __init__.py there is a sys_type() then call that + return spack.sys_type() #If in __init__.py there is a sys_type() then call that else: - return Architecture(spack.sys_type) # Else use the attributed which defaults to None + return spack.sys_type # Else use the attributed which defaults to None # This is livermore dependent. Hard coded for livermore @@ -102,15 +121,19 @@ def get_mac_sys_type(): mac_ver = py_platform.mac_ver()[0] if not mac_ver: return None - return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) + return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine()) def get_sys_type_from_uname(): """ Returns a sys_type from the uname argument Front-end config """ - return Architecture(os.uname()[0]) - + try: + arch_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + arch, _ = arch_proc.communicate() + return arch.strip() + except: + return None def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since @@ -131,34 +154,40 @@ def get_sys_type_from_config_file(): return None +@memoized +def all_architectures(): + modules = [] + for name in list_modules(spack.arch_path): + mod_name = 'spack.architectures.' + name + path = join_path(spack.arch_path, name) + ".py" + mod = imp.load_source(mod_name, path) + class_name = mod_to_class(name) + if not hasattr(mod, class_name): + tty.die('No class %s defined in %s' % (class_name, mod_name)) + cls = getattr(mod, class_name) + if not inspect.isclass(cls): + tty.die('%s.%s is not a class' % (mod_name, class_name)) + + modules.append(cls) + + return modules + @memoized def sys_type(): """Priority of gathering sys-type. 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 2. UNAME 3. GLOBALS - 4. MAC OSX + 4. MAC OSX Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. If there is no yaml present then it should move on to the next function and stop immediately once it gets a arch name """ # Try to create an architecture object using the config file FIRST - functions = [get_sys_type_from_config_file, - get_sys_type_from_uname, - get_sys_type_from_spack_globals, - get_mac_sys_type] - - sys_type = None - for func in functions: - sys_type = func() - if sys_type: - break + architecture_list = all_architectures() + architecture_list.sort(key = lambda a: a.priority) - if sys_type is None: - return Architecture("unknown_arch") - - if not isinstance(sys_type, Architecture): - raise InvalidSysTypeError(sys_type) - - return sys_type + for arch in architecture_list: + if arch.detect(): + return arch() -- cgit v1.2.3-70-g09d2 From d177184777591d21cc6a59771ed6f817960394f5 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 15:12:29 -0700 Subject: added target method to architecture.py and minor cleanup --- lib/spack/spack/architecture.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index ea1f98e06e..f04fda2ee9 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -58,9 +58,10 @@ class Target(object): # architecture classes handling the aliasing for front-end, back-end and default def __init__(self,name, module_name=None): - self.name = name # case of cray "ivybridge but if it's x86_64 + self.name = name # case of cray "ivybridge" but if it's x86_64 self.module_name = module_name # craype-ivybridge + @property def compiler_strategy(self): if self.module_name: # If there is a module_name given then use MODULES return "MODULES" @@ -73,10 +74,9 @@ class Architecture(object): """ priority = None # Subclass needs to set this number. This controls order in which arch is detected. - front = None - back = None - default_front = None # The default front end target. On cray sandybridge - default_back = None # The default back end target. On cray ivybridge + front-end = None + back-end = None + default = None # The default back end target. On cray ivybridge def __init__(self, name): self.targets = {} @@ -85,7 +85,19 @@ class Architecture(object): def add_target(self, name, target): self.targets[name] = target - + def target(self, name): + """This is a getter method for the target dictionary that handles defaulting based + on the values provided by default, front-end, and back-end. This can be overwritten + by a subclass for which we want to provide further aliasing options. + """ + if name == 'default': + name = default + elif name == 'front_end': + name = front-end + elif name == 'back_end': + name = back-end + return self.targets[name] + @classmethod def detect(self): """ Subclass is responsible for implementing this method. -- cgit v1.2.3-70-g09d2 From 6e560703862794632176733a1d063550215dc573 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 15:15:36 -0700 Subject: improved aliasing in target method in architecture.py --- lib/spack/spack/architecture.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index f04fda2ee9..3f38540c0e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -92,9 +92,9 @@ class Architecture(object): """ if name == 'default': name = default - elif name == 'front_end': + elif name == 'front_end' or name == 'fe': name = front-end - elif name == 'back_end': + elif name == 'back_end' or name == 'be': name = back-end return self.targets[name] -- cgit v1.2.3-70-g09d2 From db0695e46a6c6fd4d2ac10dba04f7feb18c25a39 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:57:00 -0700 Subject: architectures folder where the Architecture subclasses reside --- lib/spack/spack/architectures/__init__.py | 0 lib/spack/spack/architectures/cray.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 lib/spack/spack/architectures/__init__.py create mode 100644 lib/spack/spack/architectures/cray.py (limited to 'lib') diff --git a/lib/spack/spack/architectures/__init__.py b/lib/spack/spack/architectures/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py new file mode 100644 index 0000000000..e6d6d84922 --- /dev/null +++ b/lib/spack/spack/architectures/cray.py @@ -0,0 +1,17 @@ +import os + +from spack.architecture import Architecture + +class Cray(Architecture): + priority = 20 + front_end = None + back_end = None + default = None + + def __init__(self): + super(Cray, self).__init__('cray') + + @classmethod + def detect(self): + return os.path.exists('/opt/cray/craype') + -- cgit v1.2.3-70-g09d2 From 2d87bb92edb95cf1c9ec7d58f5d8a34e44bd11a4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:58:20 -0700 Subject: Changed front-end and back-end to front_end and back_end to avoid error --- lib/spack/spack/architecture.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3f38540c0e..f488b65cc6 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -74,8 +74,8 @@ class Architecture(object): """ priority = None # Subclass needs to set this number. This controls order in which arch is detected. - front-end = None - back-end = None + front_end = None + back_end = None default = None # The default back end target. On cray ivybridge def __init__(self, name): -- cgit v1.2.3-70-g09d2 From e39586c81da152e8d02fd26bf48c66e00e26519a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:58:49 -0700 Subject: Reverted back all architecture way of finding modules --- lib/spack/spack/cmd/compiler.py | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index cbe3aba864..3e86928977 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -38,7 +38,6 @@ from spack.util.environment import get_path from spack.spec import CompilerSpec description = "Manage compilers" -ARCHITECTURE = spack.architecture.sys_type() def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='compiler_command') @@ -59,30 +58,23 @@ def compiler_add(args): """Search either $PATH or a list of paths OR MODULES for compilers and add them to Spack's configuration.""" - strategies = ARCHITECTURE.strategy() - for strategy in strategies: - if strategy == 'PATH': - paths = args.add_paths # This might be a parser method. Parsing method to add_paths - if not paths: - paths = get_path('PATH') - - compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) - if c.spec not in spack.compilers.all_compilers()] - - elif strategy == "MODULES": - from spack.compilers.cray import Cray - compilers = Cray.find_in_modules() - #TODO: Find a way to locate the executables - - if compilers: - spack.compilers.add_compilers_to_config('user', *compilers) - n = len(compilers) - tty.msg("Added %d new compiler%s to %s" % ( - n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) - colify(reversed(sorted(c.spec for c in compilers)), indent=4) - else: - tty.msg("Found no new compilers") + paths = args.add_paths # This might be a parser method. Parsing method to add_paths + if not paths: + paths = get_path('PATH') + + compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) + if c.spec not in spack.compilers.all_compilers()] + + + if compilers: + spack.compilers.add_compilers_to_config('user', *compilers) + n = len(compilers) + tty.msg("Added %d new compiler%s to %s" % ( + n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) + colify(reversed(sorted(c.spec for c in compilers)), indent=4) + else: + tty.msg("Found no new compilers") def compiler_remove(args): -- cgit v1.2.3-70-g09d2 From 9a91da9ccd8044e147a416fcfa9b16405ef22b3d Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:59:28 -0700 Subject: Added arch_path to list of modules path --- lib/spack/spack/__init__.py | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index caa09eb6e0..0f23f61614 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -37,6 +37,7 @@ etc_path = join_path(prefix, "etc") lib_path = join_path(prefix, "lib", "spack") build_env_path = join_path(lib_path, "env") module_path = join_path(lib_path, "spack") +arch_path = join_path(module_path, 'architectures') compilers_path = join_path(module_path, "compilers") test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") -- cgit v1.2.3-70-g09d2 From eb2cf1698fa8cd606a8b80bcb39de2ee0b724c69 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 2 Nov 2015 11:08:08 -0800 Subject: Fixed some issues with naming --- lib/spack/spack/architecture.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index f488b65cc6..442180242b 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -49,13 +49,6 @@ class NoSysTypeError(serr.SpackError): class Target(object): """ This is the processor type e.g. cray-ivybridge """ - # Front end or back end target. Target needs to know which one this is - # Should autodetect from the machine - # features of a target - # - a module name - # -a compiler finding strategy - # -a name - # architecture classes handling the aliasing for front-end, back-end and default def __init__(self,name, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 @@ -67,7 +60,8 @@ class Target(object): return "MODULES" else: return "PATH" - + + class Architecture(object): """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned @@ -85,17 +79,19 @@ class Architecture(object): def add_target(self, name, target): self.targets[name] = target + def target(self, name): """This is a getter method for the target dictionary that handles defaulting based on the values provided by default, front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ if name == 'default': - name = default + name = self.default elif name == 'front_end' or name == 'fe': - name = front-end + name = self.front_end elif name == 'back_end' or name == 'be': - name = back-end + name = self.back_end + return self.targets[name] @classmethod @@ -148,10 +144,7 @@ def get_sys_type_from_uname(): return None def get_sys_type_from_config_file(): - """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since - The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack - is installed. Similar to .spackconfig - """ + spack_home_dir = os.environ["HOME"] + "/.spack" yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') try: -- cgit v1.2.3-70-g09d2 From 058e72d29c3cd934f91ba626392b25ebaa50e2cc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 2 Nov 2015 11:08:55 -0800 Subject: Added default target and also front end and back end targets --- lib/spack/spack/architectures/cray.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index e6d6d84922..a79c916684 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -4,9 +4,9 @@ from spack.architecture import Architecture class Cray(Architecture): priority = 20 - front_end = None - back_end = None - default = None + front_end = 'sandybridge' + back_end = 'ivybridge' + default = os.environ["CRAY_CPU_TARGET"] def __init__(self): super(Cray, self).__init__('cray') -- cgit v1.2.3-70-g09d2 From 5ac974c9b2072631eab490cce8f2922420eef9e4 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 4 Nov 2015 12:50:22 -0800 Subject: Enforced that the architecture subclass cannot add a target that shares a name with a target alias --- lib/spack/spack/architecture.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 442180242b..890df9b1e5 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -77,9 +77,14 @@ class Architecture(object): self.name = name def add_target(self, name, target): - self.targets[name] = target - - + """Used by the architecture specific subclass to list available targets. Raises an error + if the architecture specifies a name that is reserved by spack as an alias. + """ + if name in ['front_end', 'fe', 'back_end', 'be', 'default']: + raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) + self.targets[name] = target + + def target(self, name): """This is a getter method for the target dictionary that handles defaulting based on the values provided by default, front-end, and back-end. This can be overwritten -- cgit v1.2.3-70-g09d2 From 37260962e545366d0d882b205be807562c36f3b7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 4 Nov 2015 12:57:29 -0800 Subject: changed some potential syntax errors and added a way for target to recognize class --- lib/spack/spack/architecture.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 442180242b..b251b82dcc 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -48,20 +48,29 @@ class NoSysTypeError(serr.SpackError): class Target(object): - """ This is the processor type e.g. cray-ivybridge """ + """ Target is the processor of the host machine. The host machine may have different front-end + and back-end targets, especially if it is a Cray machine. The target will have a name and + also the module_name (e.g craype-compiler). Targets will also recognize which architecture + they came from using the set_architecture method. Targets will have compiler finding strategies + """ + default_strategy = None # Can probably add a compiler path here def __init__(self,name, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 self.module_name = module_name # craype-ivybridge + def set_architecture(self, architecture): # Target should get the architecture class. + self.architecture = architecture + @property def compiler_strategy(self): - if self.module_name: # If there is a module_name given then use MODULES + if default_strategy: + return default_strategy + elif self.module_name: # If there is a module_name given then use MODULES return "MODULES" else: return "PATH" - class Architecture(object): """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned @@ -77,8 +86,8 @@ class Architecture(object): self.name = name def add_target(self, name, target): - self.targets[name] = target - + target.set_architecture(self) + self.targets[name] = target def target(self, name): """This is a getter method for the target dictionary that handles defaulting based @@ -163,7 +172,7 @@ def get_sys_type_from_config_file(): def all_architectures(): modules = [] for name in list_modules(spack.arch_path): - mod_name = 'spack.architectures.' + name + mod_name = 'spack.architectures' + name path = join_path(spack.arch_path, name) + ".py" mod = imp.load_source(mod_name, path) class_name = mod_to_class(name) -- cgit v1.2.3-70-g09d2 From 35532d6b0aaa428e1d8234513e9e887d08c3914c Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 4 Nov 2015 13:00:35 -0800 Subject: Changed cray architecture subclass to add proper targets for front at back end nodes --- lib/spack/spack/architectures/cray.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index a79c916684..420b7c589e 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -10,6 +10,11 @@ class Cray(Architecture): def __init__(self): super(Cray, self).__init__('cray') + # Back End compiler needs the proper target module loaded. + self.add_target('ivybridge','craype-ivybridge') + # Could switch to use modules and fe targets for front end + # Currently using compilers by path for front end. + self.add_target('sandybridge') @classmethod def detect(self): -- cgit v1.2.3-70-g09d2 From 9bf8e8573cff42581074ed9e492ab0e0bc6c7253 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 4 Nov 2015 13:08:48 -0800 Subject: generic linux architecture subclass --- lib/spack/spack/architectures/linux.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 lib/spack/spack/architectures/linux.py (limited to 'lib') diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py new file mode 100644 index 0000000000..7238575660 --- /dev/null +++ b/lib/spack/spack/architectures/linux.py @@ -0,0 +1,17 @@ +import subprocess +from spack.architecture import Architecture + +class Linux(Architecture): + priority = 60 + front_end = "x86_64" + back_end = "x86_64" + default = "x86_64" + + def __init__(self): + super(Linux, self).__init__('linux') + + @classmethod + def detect(self): + arch = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + arch, _ = arch.communicate() + return 'x86_64' in arch.strip() -- cgit v1.2.3-70-g09d2 From 3a73ae1683c4e83791f273d2034a99824e81aeca Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 4 Nov 2015 13:12:11 -0800 Subject: Fixed the previous commit --- lib/spack/spack/architectures/cray.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index 420b7c589e..640a3933e0 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -1,20 +1,20 @@ import os -from spack.architecture import Architecture +from spack.architecture import Architecture, Target class Cray(Architecture): priority = 20 front_end = 'sandybridge' back_end = 'ivybridge' - default = os.environ["CRAY_CPU_TARGET"] + default = 'ivybridge' def __init__(self): super(Cray, self).__init__('cray') # Back End compiler needs the proper target module loaded. - self.add_target('ivybridge','craype-ivybridge') + self.add_target('ivybridge', Target('ivybridge','craype-ivybridge')) # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target('sandybridge') + self.add_target('sandybridge', Target('sandybridge')) @classmethod def detect(self): -- cgit v1.2.3-70-g09d2 From b61d554dc818c2514384b017c74dfa61c04a8c3a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 4 Nov 2015 13:28:12 -0800 Subject: Fixed architecture.py file so it doesn't have the weird merge changes i.e <<>>>>>> 8b3f2ec1d117e1a8b206927f51db8684396c231b def target(self, name): """This is a getter method for the target dictionary that handles defaulting based on the values provided by default, front-end, and back-end. This can be overwritten -- cgit v1.2.3-70-g09d2 From 08729315c617b42e6fd1eaa1af4919aa52f96a0f Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 5 Nov 2015 09:21:56 -0800 Subject: Added bgq architecture subclass: may require additional functionality to support --- lib/spack/spack/architectures/bgq.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 lib/spack/spack/architectures/bgq.py (limited to 'lib') diff --git a/lib/spack/spack/architectures/bgq.py b/lib/spack/spack/architectures/bgq.py new file mode 100644 index 0000000000..d3d4446e09 --- /dev/null +++ b/lib/spack/spack/architectures/bgq.py @@ -0,0 +1,19 @@ +import os + +from spack.architecture import Architecture, Target + +class Bgq(Architecture): + priority = 30 + front_end = 'power7' + back_end = 'powerpc' + default = 'powerpc' + + def __init__(self): + super(Bgq, self).__init__('cray') + self.add_target('power7', Target('power7')) + self.add_target('powerpc', Target('powerpc')) + + @classmethod + def detect(self): + return os.path.exists('/bgsys') + -- cgit v1.2.3-70-g09d2