From 58cb2cc2afbe742eb7a453e263437c5d503862c6 Mon Sep 17 00:00:00 2001
From: Sergey Kosukhin <skosukhin@gmail.com>
Date: Thu, 8 Dec 2016 13:35:11 +0100
Subject: Updated python: reformat code.
---
var/spack/repos/builtin/packages/python/package.py | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
index bf34b83c99..55c73e581c 100644
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -37,7 +37,7 @@ class Python(Package):
"""The Python programming language."""
homepage = "http://www.python.org"
- url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz"
+ url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz"
version('3.5.2', '3fe8434643a78630c61c6464fe2e7e72')
version('3.5.1', 'be78e48cdfc1a7ad90efff146dce6cfe')
@@ -54,7 +54,7 @@ class Python(Package):
extendable = True
- variant('tk', default=False, description='Provide support for Tkinter')
+ variant('tk', default=False, description='Provide support for Tkinter')
variant('ucs4', default=False,
description='Enable UCS4 (wide) unicode strings')
# From https://docs.python.org/2/c-api/unicode.html: Python's default
@@ -71,7 +71,7 @@ class Python(Package):
depends_on("ncurses")
depends_on("sqlite")
depends_on("zlib")
- depends_on("tk", when="+tk")
+ depends_on("tk", when="+tk")
depends_on("tcl", when="+tk")
patch('ncurses.patch')
@@ -197,7 +197,7 @@ class Python(Package):
]
for filename in files:
- filter_file(env['CC'], self.compiler.cc,
+ filter_file(env['CC'], self.compiler.cc,
join_path(dirname, filename), **kwargs)
filter_file(env['CXX'], self.compiler.cxx,
join_path(dirname, filename), **kwargs)
@@ -273,12 +273,12 @@ class Python(Package):
module.setup_py = Executable(python_path + ' setup.py --no-user-cfg')
# Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs.
- module.python_lib_dir = join_path(ext_spec.prefix,
- self.python_lib_dir)
+ module.python_lib_dir = join_path(ext_spec.prefix,
+ self.python_lib_dir)
module.python_include_dir = join_path(ext_spec.prefix,
self.python_include_dir)
- module.site_packages_dir = join_path(ext_spec.prefix,
- self.site_packages_dir)
+ module.site_packages_dir = join_path(ext_spec.prefix,
+ self.site_packages_dir)
# Make the site packages directory for extensions
if ext_spec.package.is_extension:
@@ -330,8 +330,8 @@ class Python(Package):
continue
if re.search(r'^(import|#)', line):
continue
- if ((ext.name != 'py-setuptools' and
- re.search(r'setuptools.*egg$', line))):
+ if (ext.name != 'py-setuptools' and
+ re.search(r'setuptools.*egg$', line)):
continue
paths.append(line)
--
cgit v1.2.3-70-g09d2
From ec855df071acaa758d40a18074a955e5effc9321 Mon Sep 17 00:00:00 2001
From: Sergey Kosukhin <skosukhin@gmail.com>
Date: Thu, 8 Dec 2016 13:36:18 +0100
Subject: Updated python: account for lib64 when filtering compilers.
---
var/spack/repos/builtin/packages/python/package.py | 36 ++++++++++------------
1 file changed, 16 insertions(+), 20 deletions(-)
diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
index 55c73e581c..2eab9fa558 100644
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -132,19 +132,17 @@ class Python(Package):
make()
make('install')
- self.filter_compilers(spec, prefix)
+ self.filter_compilers(prefix)
# TODO:
# On OpenSuse 13, python uses <prefix>/lib64/python2.7/lib-dynload/*.so
# instead of <prefix>/lib/python2.7/lib-dynload/*.so. Oddly enough the
# result is that Python can not find modules like cPickle. A workaround
# for now is to symlink to `lib`:
- src = os.path.join(prefix,
- 'lib64',
+ src = os.path.join(prefix.lib64,
'python{0}'.format(self.version.up_to(2)),
'lib-dynload')
- dst = os.path.join(prefix,
- 'lib',
+ dst = os.path.join(prefix.lib,
'python{0}'.format(self.version.up_to(2)),
'lib-dynload')
if os.path.isdir(src) and not os.path.isdir(dst):
@@ -174,7 +172,7 @@ class Python(Package):
# >>> import Tkinter
# >>> Tkinter._test()
- def filter_compilers(self, spec, prefix):
+ def filter_compilers(self, prefix):
"""Run after install to tell the configuration files and Makefiles
to use the compilers that Spack built the package with.
@@ -184,23 +182,21 @@ class Python(Package):
kwargs = {'ignore_absent': True, 'backup': False, 'string': True}
- dirname = join_path(prefix.lib,
- 'python{0}'.format(self.version.up_to(2)))
+ lib_dirnames = [
+ join_path(lib_dir, 'python{0}'.format(self.version.up_to(2))) for
+ lib_dir in [prefix.lib, prefix.lib64]]
- config = 'config'
- if spec.satisfies('@3:'):
- config = 'config-{0}m'.format(self.version.up_to(2))
+ config_dirname = 'config-{0}m'.format(
+ self.version.up_to(2)) if self.spec.satisfies('@3:') else 'config'
- files = [
- '_sysconfigdata.py',
- join_path(config, 'Makefile')
- ]
+ rel_filenames = ['_sysconfigdata.py',
+ join_path(config_dirname, 'Makefile')]
+
+ abs_filenames = [join_path(dirname, filename) for dirname in
+ lib_dirnames for filename in rel_filenames]
- for filename in files:
- filter_file(env['CC'], self.compiler.cc,
- join_path(dirname, filename), **kwargs)
- filter_file(env['CXX'], self.compiler.cxx,
- join_path(dirname, filename), **kwargs)
+ filter_file(env['CC'], self.compiler.cc, *abs_filenames, **kwargs)
+ filter_file(env['CXX'], self.compiler.cxx, *abs_filenames, **kwargs)
# ========================================================================
# Set up environment to make install easy for python extensions.
--
cgit v1.2.3-70-g09d2
From 3ba88a750970a995fd295e92dbf231dcb6a13d6c Mon Sep 17 00:00:00 2001
From: Sergey Kosukhin <skosukhin@gmail.com>
Date: Thu, 8 Dec 2016 13:39:10 +0100
Subject: Updated python: pass LDSHARED to dependants' setup scripts.
---
var/spack/repos/builtin/packages/python/package.py | 88 ++++++++++++++++++++++
1 file changed, 88 insertions(+)
diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
index 2eab9fa558..b5c11ee527 100644
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -22,12 +22,14 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
+import ast
import os
import re
from contextlib import closing
import spack
import llnl.util.tty as tty
+import yaml
from llnl.util.lang import match_predicate
from spack import *
from spack.util.environment import *
@@ -132,6 +134,8 @@ class Python(Package):
make()
make('install')
+ self.save_distutils_data(prefix)
+
self.filter_compilers(prefix)
# TODO:
@@ -172,6 +176,84 @@ class Python(Package):
# >>> import Tkinter
# >>> Tkinter._test()
+ def save_distutils_data(self, prefix):
+ """
+ Run before changing automatically generated contents of the
+ _sysconfigdata.py, which is used by distutils to figure out what
+ executables to use while compiling and linking extensions. If we build
+ extensions with spack those executables should be spack's wrappers.
+ Spack partially covers this by setting environment variables that
+ are also accounted for by distutils. Currently there is one more known
+ variable that must be set, which is LDSHARED, so the method saves its
+ autogenerated value to pass it to the dependant package's setup script.
+ """
+
+ input_filename = None
+ for filename in [join_path(lib_dir,
+ 'python{0}'.format(self.version.up_to(2)),
+ '_sysconfigdata.py')
+ for lib_dir in [prefix.lib, prefix.lib64]]:
+ if os.path.isfile(filename):
+ input_filename = filename
+ break
+
+ if not input_filename:
+ return
+
+ input_dict = None
+
+ try:
+ with open(input_filename) as input_file:
+ match = re.search(r'build_time_vars\s*=\s*(?P<dict>{.*})',
+ input_file.read(),
+ flags=re.DOTALL)
+
+ if match:
+ input_dict = ast.literal_eval(match.group('dict'))
+ except (IOError, SyntaxError):
+ pass
+
+ if not input_dict:
+ return
+
+ vars_to_save = ['LDSHARED']
+ saved_vars = {}
+
+ for var_name in vars_to_save:
+ if var_name in input_dict:
+ saved_vars[var_name] = input_dict[var_name]
+
+ if len(saved_vars) > 0:
+ try:
+ output_filename = join_path(
+ spack.store.layout.metadata_path(self.spec),
+ 'sysconfig.yaml')
+ with open(output_filename, 'w') as output_file:
+ yaml.dump(saved_vars, stream=output_file,
+ default_flow_style=False)
+ except (yaml.YAMLError, IOError):
+ pass
+
+ setattr(self, '_distutils_data_cache', saved_vars)
+
+ def load_distutils_data(self):
+ if not hasattr(self, '_distutils_data_cache'):
+ input_filename = join_path(
+ spack.store.layout.metadata_path(self.spec),
+ 'sysconfig.yaml')
+ if os.path.isfile(input_filename):
+ try:
+ with open(input_filename) as input_file:
+ setattr(self, '_distutils_data_cache',
+ yaml.load(input_file))
+ except (yaml.YAMLError, IOError):
+ pass
+
+ if not hasattr(self, '_distutils_data_cache'):
+ setattr(self, '_distutils_data_cache', None)
+
+ return self._distutils_data_cache
+
def filter_compilers(self, prefix):
"""Run after install to tell the configuration files and Makefiles
to use the compilers that Spack built the package with.
@@ -268,6 +350,12 @@ class Python(Package):
module.python = Executable(python_path)
module.setup_py = Executable(python_path + ' setup.py --no-user-cfg')
+ distutils_data = self.load_distutils_data()
+
+ if distutils_data:
+ for key, value in distutils_data.iteritems():
+ module.setup_py.add_default_env(key, value)
+
# Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs.
module.python_lib_dir = join_path(ext_spec.prefix,
self.python_lib_dir)
--
cgit v1.2.3-70-g09d2
From d8c44189528207dd341766284dd08d0a78bc879f Mon Sep 17 00:00:00 2001
From: Sergey Kosukhin <skosukhin@gmail.com>
Date: Fri, 9 Dec 2016 10:18:10 +0100
Subject: Updated python: refactoring and warning messages.
---
var/spack/repos/builtin/packages/python/package.py | 93 +++++++++++++---------
1 file changed, 57 insertions(+), 36 deletions(-)
diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
index b5c11ee527..010632dbe7 100644
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -29,10 +29,11 @@ from contextlib import closing
import spack
import llnl.util.tty as tty
-import yaml
from llnl.util.lang import match_predicate
+from llnl.util.filesystem import force_remove
from spack import *
from spack.util.environment import *
+import spack.util.spack_json as sjson
class Python(Package):
@@ -78,6 +79,10 @@ class Python(Package):
patch('ncurses.patch')
+ _DISTUTIL_VARS_TO_SAVE = ['LDSHARED']
+ _DISTUTIL_CACHE_FILENAME = 'sysconfig.json'
+ _distutil_vars = None
+
@when('@2.7,3.4:')
def patch(self):
# NOTE: Python's default installation procedure makes it possible for a
@@ -134,7 +139,7 @@ class Python(Package):
make()
make('install')
- self.save_distutils_data(prefix)
+ self._save_distutil_vars(prefix)
self.filter_compilers(prefix)
@@ -176,7 +181,7 @@ class Python(Package):
# >>> import Tkinter
# >>> Tkinter._test()
- def save_distutils_data(self, prefix):
+ def _save_distutil_vars(self, prefix):
"""
Run before changing automatically generated contents of the
_sysconfigdata.py, which is used by distutils to figure out what
@@ -188,6 +193,8 @@ class Python(Package):
autogenerated value to pass it to the dependant package's setup script.
"""
+ self._distutil_vars = {}
+
input_filename = None
for filename in [join_path(lib_dir,
'python{0}'.format(self.version.up_to(2)),
@@ -201,7 +208,6 @@ class Python(Package):
return
input_dict = None
-
try:
with open(input_filename) as input_file:
match = re.search(r'build_time_vars\s*=\s*(?P<dict>{.*})',
@@ -214,45 +220,60 @@ class Python(Package):
pass
if not input_dict:
+ tty.warn('Failed to find \'build_time_vars\' dictionary in file '
+ '\'%s\'. This might cause the extensions that are '
+ 'installed with distutils to call compilers directly '
+ 'avoiding Spack\'s wrappers.' % input_filename)
return
- vars_to_save = ['LDSHARED']
- saved_vars = {}
-
- for var_name in vars_to_save:
+ for var_name in Python._DISTUTIL_VARS_TO_SAVE:
if var_name in input_dict:
- saved_vars[var_name] = input_dict[var_name]
-
- if len(saved_vars) > 0:
+ self._distutil_vars[var_name] = input_dict[var_name]
+ else:
+ tty.warn('Failed to find key \'%s\' in \'build_time_vars\' '
+ 'dictionary in file \'%s\'. This might cause the '
+ 'extensions that are installed with distutils to '
+ 'call compilers directly avoiding Spack\'s wrappers.'
+ % (var_name, input_filename))
+
+ if len(self._distutil_vars) > 0:
+ output_filename = None
try:
output_filename = join_path(
spack.store.layout.metadata_path(self.spec),
- 'sysconfig.yaml')
+ Python._DISTUTIL_CACHE_FILENAME)
with open(output_filename, 'w') as output_file:
- yaml.dump(saved_vars, stream=output_file,
- default_flow_style=False)
- except (yaml.YAMLError, IOError):
+ sjson.dump(self._distutil_vars, output_file)
+ except:
+ tty.warn('Failed to save metadata for distutils. This might '
+ 'cause the extensions that are installed with '
+ 'distutils to call compilers directly avoiding '
+ 'Spack\'s wrappers.')
+ # We make the cache empty if we failed to save it to file
+ # to provide the same behaviour as in the case when the cache
+ # is initialized by the method load_distutils_data().
+ self._distutil_vars = {}
+ if output_filename:
+ force_remove(output_filename)
+
+ def _load_distutil_vars(self):
+ # We update and keep the cache unchanged only if the package is
+ # installed.
+ if not self._distutil_vars and self.installed:
+ try:
+ input_filename = join_path(
+ spack.store.layout.metadata_path(self.spec),
+ Python._DISTUTIL_CACHE_FILENAME)
+ if os.path.isfile(input_filename):
+ with open(input_filename) as input_file:
+ self._distutil_vars = sjson.load(input_file)
+ except:
pass
- setattr(self, '_distutils_data_cache', saved_vars)
-
- def load_distutils_data(self):
- if not hasattr(self, '_distutils_data_cache'):
- input_filename = join_path(
- spack.store.layout.metadata_path(self.spec),
- 'sysconfig.yaml')
- if os.path.isfile(input_filename):
- try:
- with open(input_filename) as input_file:
- setattr(self, '_distutils_data_cache',
- yaml.load(input_file))
- except (yaml.YAMLError, IOError):
- pass
-
- if not hasattr(self, '_distutils_data_cache'):
- setattr(self, '_distutils_data_cache', None)
+ if not self._distutil_vars:
+ self._distutil_vars = {}
- return self._distutils_data_cache
+ return self._distutil_vars
def filter_compilers(self, prefix):
"""Run after install to tell the configuration files and Makefiles
@@ -350,10 +371,10 @@ class Python(Package):
module.python = Executable(python_path)
module.setup_py = Executable(python_path + ' setup.py --no-user-cfg')
- distutils_data = self.load_distutils_data()
+ distutil_vars = self._load_distutil_vars()
- if distutils_data:
- for key, value in distutils_data.iteritems():
+ if distutil_vars:
+ for key, value in distutil_vars.iteritems():
module.setup_py.add_default_env(key, value)
# Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs.
--
cgit v1.2.3-70-g09d2