From 58cb2cc2afbe742eb7a453e263437c5d503862c6 Mon Sep 17 00:00:00 2001 From: Sergey Kosukhin Date: Thu, 8 Dec 2016 13:35:11 +0100 Subject: Updated python: reformat code. --- var/spack/repos/builtin/packages/python/package.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'var') diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index bf34b83c99..55c73e581c 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -37,7 +37,7 @@ class Python(Package): """The Python programming language.""" homepage = "http://www.python.org" - url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz" + url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz" version('3.5.2', '3fe8434643a78630c61c6464fe2e7e72') version('3.5.1', 'be78e48cdfc1a7ad90efff146dce6cfe') @@ -54,7 +54,7 @@ class Python(Package): extendable = True - variant('tk', default=False, description='Provide support for Tkinter') + variant('tk', default=False, description='Provide support for Tkinter') variant('ucs4', default=False, description='Enable UCS4 (wide) unicode strings') # From https://docs.python.org/2/c-api/unicode.html: Python's default @@ -71,7 +71,7 @@ class Python(Package): depends_on("ncurses") depends_on("sqlite") depends_on("zlib") - depends_on("tk", when="+tk") + depends_on("tk", when="+tk") depends_on("tcl", when="+tk") patch('ncurses.patch') @@ -197,7 +197,7 @@ class Python(Package): ] for filename in files: - filter_file(env['CC'], self.compiler.cc, + filter_file(env['CC'], self.compiler.cc, join_path(dirname, filename), **kwargs) filter_file(env['CXX'], self.compiler.cxx, join_path(dirname, filename), **kwargs) @@ -273,12 +273,12 @@ class Python(Package): module.setup_py = Executable(python_path + ' setup.py --no-user-cfg') # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. - module.python_lib_dir = join_path(ext_spec.prefix, - self.python_lib_dir) + module.python_lib_dir = join_path(ext_spec.prefix, + self.python_lib_dir) module.python_include_dir = join_path(ext_spec.prefix, self.python_include_dir) - module.site_packages_dir = join_path(ext_spec.prefix, - self.site_packages_dir) + module.site_packages_dir = join_path(ext_spec.prefix, + self.site_packages_dir) # Make the site packages directory for extensions if ext_spec.package.is_extension: @@ -330,8 +330,8 @@ class Python(Package): continue if re.search(r'^(import|#)', line): continue - if ((ext.name != 'py-setuptools' and - re.search(r'setuptools.*egg$', line))): + if (ext.name != 'py-setuptools' and + re.search(r'setuptools.*egg$', line)): continue paths.append(line) -- cgit v1.2.3-70-g09d2 From ec855df071acaa758d40a18074a955e5effc9321 Mon Sep 17 00:00:00 2001 From: Sergey Kosukhin Date: Thu, 8 Dec 2016 13:36:18 +0100 Subject: Updated python: account for lib64 when filtering compilers. --- var/spack/repos/builtin/packages/python/package.py | 36 ++++++++++------------ 1 file changed, 16 insertions(+), 20 deletions(-) (limited to 'var') diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 55c73e581c..2eab9fa558 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -132,19 +132,17 @@ class Python(Package): make() make('install') - self.filter_compilers(spec, prefix) + self.filter_compilers(prefix) # TODO: # On OpenSuse 13, python uses /lib64/python2.7/lib-dynload/*.so # instead of /lib/python2.7/lib-dynload/*.so. Oddly enough the # result is that Python can not find modules like cPickle. A workaround # for now is to symlink to `lib`: - src = os.path.join(prefix, - 'lib64', + src = os.path.join(prefix.lib64, 'python{0}'.format(self.version.up_to(2)), 'lib-dynload') - dst = os.path.join(prefix, - 'lib', + dst = os.path.join(prefix.lib, 'python{0}'.format(self.version.up_to(2)), 'lib-dynload') if os.path.isdir(src) and not os.path.isdir(dst): @@ -174,7 +172,7 @@ class Python(Package): # >>> import Tkinter # >>> Tkinter._test() - def filter_compilers(self, spec, prefix): + def filter_compilers(self, prefix): """Run after install to tell the configuration files and Makefiles to use the compilers that Spack built the package with. @@ -184,23 +182,21 @@ class Python(Package): kwargs = {'ignore_absent': True, 'backup': False, 'string': True} - dirname = join_path(prefix.lib, - 'python{0}'.format(self.version.up_to(2))) + lib_dirnames = [ + join_path(lib_dir, 'python{0}'.format(self.version.up_to(2))) for + lib_dir in [prefix.lib, prefix.lib64]] - config = 'config' - if spec.satisfies('@3:'): - config = 'config-{0}m'.format(self.version.up_to(2)) + config_dirname = 'config-{0}m'.format( + self.version.up_to(2)) if self.spec.satisfies('@3:') else 'config' - files = [ - '_sysconfigdata.py', - join_path(config, 'Makefile') - ] + rel_filenames = ['_sysconfigdata.py', + join_path(config_dirname, 'Makefile')] + + abs_filenames = [join_path(dirname, filename) for dirname in + lib_dirnames for filename in rel_filenames] - for filename in files: - filter_file(env['CC'], self.compiler.cc, - join_path(dirname, filename), **kwargs) - filter_file(env['CXX'], self.compiler.cxx, - join_path(dirname, filename), **kwargs) + filter_file(env['CC'], self.compiler.cc, *abs_filenames, **kwargs) + filter_file(env['CXX'], self.compiler.cxx, *abs_filenames, **kwargs) # ======================================================================== # Set up environment to make install easy for python extensions. -- cgit v1.2.3-70-g09d2 From 3ba88a750970a995fd295e92dbf231dcb6a13d6c Mon Sep 17 00:00:00 2001 From: Sergey Kosukhin Date: Thu, 8 Dec 2016 13:39:10 +0100 Subject: Updated python: pass LDSHARED to dependants' setup scripts. --- var/spack/repos/builtin/packages/python/package.py | 88 ++++++++++++++++++++++ 1 file changed, 88 insertions(+) (limited to 'var') diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 2eab9fa558..b5c11ee527 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -22,12 +22,14 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import ast import os import re from contextlib import closing import spack import llnl.util.tty as tty +import yaml from llnl.util.lang import match_predicate from spack import * from spack.util.environment import * @@ -132,6 +134,8 @@ class Python(Package): make() make('install') + self.save_distutils_data(prefix) + self.filter_compilers(prefix) # TODO: @@ -172,6 +176,84 @@ class Python(Package): # >>> import Tkinter # >>> Tkinter._test() + def save_distutils_data(self, prefix): + """ + Run before changing automatically generated contents of the + _sysconfigdata.py, which is used by distutils to figure out what + executables to use while compiling and linking extensions. If we build + extensions with spack those executables should be spack's wrappers. + Spack partially covers this by setting environment variables that + are also accounted for by distutils. Currently there is one more known + variable that must be set, which is LDSHARED, so the method saves its + autogenerated value to pass it to the dependant package's setup script. + """ + + input_filename = None + for filename in [join_path(lib_dir, + 'python{0}'.format(self.version.up_to(2)), + '_sysconfigdata.py') + for lib_dir in [prefix.lib, prefix.lib64]]: + if os.path.isfile(filename): + input_filename = filename + break + + if not input_filename: + return + + input_dict = None + + try: + with open(input_filename) as input_file: + match = re.search(r'build_time_vars\s*=\s*(?P{.*})', + input_file.read(), + flags=re.DOTALL) + + if match: + input_dict = ast.literal_eval(match.group('dict')) + except (IOError, SyntaxError): + pass + + if not input_dict: + return + + vars_to_save = ['LDSHARED'] + saved_vars = {} + + for var_name in vars_to_save: + if var_name in input_dict: + saved_vars[var_name] = input_dict[var_name] + + if len(saved_vars) > 0: + try: + output_filename = join_path( + spack.store.layout.metadata_path(self.spec), + 'sysconfig.yaml') + with open(output_filename, 'w') as output_file: + yaml.dump(saved_vars, stream=output_file, + default_flow_style=False) + except (yaml.YAMLError, IOError): + pass + + setattr(self, '_distutils_data_cache', saved_vars) + + def load_distutils_data(self): + if not hasattr(self, '_distutils_data_cache'): + input_filename = join_path( + spack.store.layout.metadata_path(self.spec), + 'sysconfig.yaml') + if os.path.isfile(input_filename): + try: + with open(input_filename) as input_file: + setattr(self, '_distutils_data_cache', + yaml.load(input_file)) + except (yaml.YAMLError, IOError): + pass + + if not hasattr(self, '_distutils_data_cache'): + setattr(self, '_distutils_data_cache', None) + + return self._distutils_data_cache + def filter_compilers(self, prefix): """Run after install to tell the configuration files and Makefiles to use the compilers that Spack built the package with. @@ -268,6 +350,12 @@ class Python(Package): module.python = Executable(python_path) module.setup_py = Executable(python_path + ' setup.py --no-user-cfg') + distutils_data = self.load_distutils_data() + + if distutils_data: + for key, value in distutils_data.iteritems(): + module.setup_py.add_default_env(key, value) + # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. module.python_lib_dir = join_path(ext_spec.prefix, self.python_lib_dir) -- cgit v1.2.3-70-g09d2 From d8c44189528207dd341766284dd08d0a78bc879f Mon Sep 17 00:00:00 2001 From: Sergey Kosukhin Date: Fri, 9 Dec 2016 10:18:10 +0100 Subject: Updated python: refactoring and warning messages. --- var/spack/repos/builtin/packages/python/package.py | 93 +++++++++++++--------- 1 file changed, 57 insertions(+), 36 deletions(-) (limited to 'var') diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index b5c11ee527..010632dbe7 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -29,10 +29,11 @@ from contextlib import closing import spack import llnl.util.tty as tty -import yaml from llnl.util.lang import match_predicate +from llnl.util.filesystem import force_remove from spack import * from spack.util.environment import * +import spack.util.spack_json as sjson class Python(Package): @@ -78,6 +79,10 @@ class Python(Package): patch('ncurses.patch') + _DISTUTIL_VARS_TO_SAVE = ['LDSHARED'] + _DISTUTIL_CACHE_FILENAME = 'sysconfig.json' + _distutil_vars = None + @when('@2.7,3.4:') def patch(self): # NOTE: Python's default installation procedure makes it possible for a @@ -134,7 +139,7 @@ class Python(Package): make() make('install') - self.save_distutils_data(prefix) + self._save_distutil_vars(prefix) self.filter_compilers(prefix) @@ -176,7 +181,7 @@ class Python(Package): # >>> import Tkinter # >>> Tkinter._test() - def save_distutils_data(self, prefix): + def _save_distutil_vars(self, prefix): """ Run before changing automatically generated contents of the _sysconfigdata.py, which is used by distutils to figure out what @@ -188,6 +193,8 @@ class Python(Package): autogenerated value to pass it to the dependant package's setup script. """ + self._distutil_vars = {} + input_filename = None for filename in [join_path(lib_dir, 'python{0}'.format(self.version.up_to(2)), @@ -201,7 +208,6 @@ class Python(Package): return input_dict = None - try: with open(input_filename) as input_file: match = re.search(r'build_time_vars\s*=\s*(?P{.*})', @@ -214,45 +220,60 @@ class Python(Package): pass if not input_dict: + tty.warn('Failed to find \'build_time_vars\' dictionary in file ' + '\'%s\'. This might cause the extensions that are ' + 'installed with distutils to call compilers directly ' + 'avoiding Spack\'s wrappers.' % input_filename) return - vars_to_save = ['LDSHARED'] - saved_vars = {} - - for var_name in vars_to_save: + for var_name in Python._DISTUTIL_VARS_TO_SAVE: if var_name in input_dict: - saved_vars[var_name] = input_dict[var_name] - - if len(saved_vars) > 0: + self._distutil_vars[var_name] = input_dict[var_name] + else: + tty.warn('Failed to find key \'%s\' in \'build_time_vars\' ' + 'dictionary in file \'%s\'. This might cause the ' + 'extensions that are installed with distutils to ' + 'call compilers directly avoiding Spack\'s wrappers.' + % (var_name, input_filename)) + + if len(self._distutil_vars) > 0: + output_filename = None try: output_filename = join_path( spack.store.layout.metadata_path(self.spec), - 'sysconfig.yaml') + Python._DISTUTIL_CACHE_FILENAME) with open(output_filename, 'w') as output_file: - yaml.dump(saved_vars, stream=output_file, - default_flow_style=False) - except (yaml.YAMLError, IOError): + sjson.dump(self._distutil_vars, output_file) + except: + tty.warn('Failed to save metadata for distutils. This might ' + 'cause the extensions that are installed with ' + 'distutils to call compilers directly avoiding ' + 'Spack\'s wrappers.') + # We make the cache empty if we failed to save it to file + # to provide the same behaviour as in the case when the cache + # is initialized by the method load_distutils_data(). + self._distutil_vars = {} + if output_filename: + force_remove(output_filename) + + def _load_distutil_vars(self): + # We update and keep the cache unchanged only if the package is + # installed. + if not self._distutil_vars and self.installed: + try: + input_filename = join_path( + spack.store.layout.metadata_path(self.spec), + Python._DISTUTIL_CACHE_FILENAME) + if os.path.isfile(input_filename): + with open(input_filename) as input_file: + self._distutil_vars = sjson.load(input_file) + except: pass - setattr(self, '_distutils_data_cache', saved_vars) - - def load_distutils_data(self): - if not hasattr(self, '_distutils_data_cache'): - input_filename = join_path( - spack.store.layout.metadata_path(self.spec), - 'sysconfig.yaml') - if os.path.isfile(input_filename): - try: - with open(input_filename) as input_file: - setattr(self, '_distutils_data_cache', - yaml.load(input_file)) - except (yaml.YAMLError, IOError): - pass - - if not hasattr(self, '_distutils_data_cache'): - setattr(self, '_distutils_data_cache', None) + if not self._distutil_vars: + self._distutil_vars = {} - return self._distutils_data_cache + return self._distutil_vars def filter_compilers(self, prefix): """Run after install to tell the configuration files and Makefiles @@ -350,10 +371,10 @@ class Python(Package): module.python = Executable(python_path) module.setup_py = Executable(python_path + ' setup.py --no-user-cfg') - distutils_data = self.load_distutils_data() + distutil_vars = self._load_distutil_vars() - if distutils_data: - for key, value in distutils_data.iteritems(): + if distutil_vars: + for key, value in distutil_vars.iteritems(): module.setup_py.add_default_env(key, value) # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. -- cgit v1.2.3-70-g09d2