Fix Python 3.8 build on macOS (#13338)

1 files changed, 166 insertions, 150 deletions

diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
index b05bd6dd9b..2813121dda 100644
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -40,21 +40,21 @@ class Python(AutotoolsPackage):
     version('3.6.8',  sha256='7f5b1f08b3b0a595387ef6c64c85b1b13b38abef0dd871835ee923262e4f32f0')
     version('3.6.7',  sha256='b7c36f7ed8f7143b2c46153b7332db2227669f583ea0cce753facf549d1a4239')
     version('3.6.6',  sha256='7d56dadf6c7d92a238702389e80cfe66fbfae73e584189ed6f89c75bbf3eda58')
-    version('3.6.5', sha256='53a3e17d77cd15c5230192b6a8c1e031c07cd9f34a2f089a731c6f6bd343d5c6')
-    version('3.6.4', sha256='7dc453e1a93c083388eb1a23a256862407f8234a96dc4fae0fc7682020227486')
-    version('3.6.3', sha256='ab6193af1921b30f587b302fe385268510e80187ca83ca82d2bfe7ab544c6f91')
-    version('3.6.2', sha256='7919489310a5f17f7acbab64d731e46dca0702874840dadce8bd4b2b3b8e7a82')
-    version('3.6.1', sha256='aa50b0143df7c89ce91be020fe41382613a817354b33acdc6641b44f8ced3828')
-    version('3.6.0', sha256='aa472515800d25a3739833f76ca3735d9f4b2fe77c3cb21f69275e0cce30cb2b')
-    version('3.5.7', sha256='542d94920a2a06a471a73b51614805ad65366af98145b0369bc374cf248b521b')
-    version('3.5.2', sha256='1524b840e42cf3b909e8f8df67c1724012c7dc7f9d076d4feef2d3eff031e8a0')
-    version('3.5.1', sha256='687e067d9f391da645423c7eda8205bae9d35edc0c76ef5218dcbe4cc770d0d7')
-    version('3.5.0', sha256='584e3d5a02692ca52fce505e68ecd77248a6f2c99adf9db144a39087336b0fe0')
+    version('3.6.5',  sha256='53a3e17d77cd15c5230192b6a8c1e031c07cd9f34a2f089a731c6f6bd343d5c6')
+    version('3.6.4',  sha256='7dc453e1a93c083388eb1a23a256862407f8234a96dc4fae0fc7682020227486')
+    version('3.6.3',  sha256='ab6193af1921b30f587b302fe385268510e80187ca83ca82d2bfe7ab544c6f91')
+    version('3.6.2',  sha256='7919489310a5f17f7acbab64d731e46dca0702874840dadce8bd4b2b3b8e7a82')
+    version('3.6.1',  sha256='aa50b0143df7c89ce91be020fe41382613a817354b33acdc6641b44f8ced3828')
+    version('3.6.0',  sha256='aa472515800d25a3739833f76ca3735d9f4b2fe77c3cb21f69275e0cce30cb2b')
+    version('3.5.7',  sha256='542d94920a2a06a471a73b51614805ad65366af98145b0369bc374cf248b521b')
+    version('3.5.2',  sha256='1524b840e42cf3b909e8f8df67c1724012c7dc7f9d076d4feef2d3eff031e8a0')
+    version('3.5.1',  sha256='687e067d9f391da645423c7eda8205bae9d35edc0c76ef5218dcbe4cc770d0d7')
+    version('3.5.0',  sha256='584e3d5a02692ca52fce505e68ecd77248a6f2c99adf9db144a39087336b0fe0')
     version('3.4.10', sha256='217757699249ab432571b381386d441e12b433100ab5f908051fcb7cced2539d')
-    version('3.4.3', sha256='8b743f56e9e50bf0923b9e9c45dd927c071d7aa56cd46569d8818add8cf01147')
-    version('3.3.6', sha256='0a58ad1f1def4ecc90b18b0c410a3a0e1a48cf7692c75d1f83d0af080e5d2034')
-    version('3.2.6', sha256='fc1e41296e29d476f696303acae293ae7a2310f0f9d0d637905e722a3f16163e')
-    version('3.1.5', sha256='d12dae6d06f52ef6bf1271db4d5b4d14b5dd39813e324314e72b648ef1bc0103')
+    version('3.4.3',  sha256='8b743f56e9e50bf0923b9e9c45dd927c071d7aa56cd46569d8818add8cf01147')
+    version('3.3.6',  sha256='0a58ad1f1def4ecc90b18b0c410a3a0e1a48cf7692c75d1f83d0af080e5d2034')
+    version('3.2.6',  sha256='fc1e41296e29d476f696303acae293ae7a2310f0f9d0d637905e722a3f16163e')
+    version('3.1.5',  sha256='d12dae6d06f52ef6bf1271db4d5b4d14b5dd39813e324314e72b648ef1bc0103')
     version('2.7.16', sha256='01da813a3600876f03f46db11cc5c408175e99f03af2ba942ef324389a83bad5', preferred=True)
     version('2.7.15', sha256='18617d1f15a380a919d517630a9cd85ce17ea602f9bbdc58ddc672df4b0239db')
     version('2.7.14', sha256='304c9b202ea6fbd0a4a8e0ad3733715fbd4749f2204a9173a58ec53c32ea73e8')
@@ -62,8 +62,8 @@ class Python(AutotoolsPackage):
     version('2.7.12', sha256='3cb522d17463dfa69a155ab18cffa399b358c966c0363d6c8b5b3bf1384da4b6')
     version('2.7.11', sha256='82929b96fd6afc8da838b149107078c02fa1744b7e60999a8babbc0d3fa86fc6')
     version('2.7.10', sha256='eda8ce6eec03e74991abb5384170e7c65fcd7522e409b8e83d7e6372add0f12a')
-    version('2.7.9', sha256='c8bba33e66ac3201dabdc556f0ea7cfe6ac11946ec32d357c4c6f9b018c12c5b')
-    version('2.7.8', sha256='74d70b914da4487aa1d97222b29e9554d042f825f26cb2b93abd20fdda56b557')
+    version('2.7.9',  sha256='c8bba33e66ac3201dabdc556f0ea7cfe6ac11946ec32d357c4c6f9b018c12c5b')
+    version('2.7.8',  sha256='74d70b914da4487aa1d97222b29e9554d042f825f26cb2b93abd20fdda56b557')
 
     extendable = True
 
@@ -135,7 +135,7 @@ class Python(AutotoolsPackage):
         # a Mac.
         depends_on('libuuid', when='+uuid')
 
-    patch('tkinter.patch', when='@:2.8,3.3: platform=darwin')
+    patch('tkinter.patch', when='@:2.8,3.3:3.7 platform=darwin')
 
     # Ensure that distutils chooses correct compiler option for RPATH on cray:
     patch('cray-rpath-2.3.patch', when='@2.3:3.0.1 platform=cray')
@@ -190,7 +190,7 @@ class Python(AutotoolsPackage):
         # Python v2.7 and v3.4+ (see https://bugs.python.org/issue1180) and
         # adding support for ignoring user configuration will require
         # significant changes to this package for other Python versions.
-        if not spec.satisfies('@2.7,3.4:'):
+        if not spec.satisfies('@2.7:2.8,3.4:'):
             tty.warn(('Python v{0} may not install properly if Python '
                       'user configurations are present.').format(self.version))
 
@@ -290,20 +290,111 @@ class Python(AutotoolsPackage):
         return config_args
 
     @run_after('install')
-    def post_install(self):
-        spec = self.spec
-        prefix = self.prefix
+    def _save_distutil_vars(self):
+        """
+        Run before changing automatically generated contents of the
+        _sysconfigdata.py, which is used by distutils to figure out what
+        executables to use while compiling and linking extensions. If we build
+        extensions with spack those executables should be spack's wrappers.
+        Spack partially covers this by setting environment variables that
+        are also accounted for by distutils. Currently there is one more known
+        variable that must be set, which is LDSHARED, so the method saves its
+        autogenerated value to pass it to the dependent package's setup script.
+        """
 
-        self.sysconfigfilename = '_sysconfigdata.py'
-        if spec.satisfies('@3.6:'):
-            # Python 3.6.0 renamed the sys config file
-            sc = 'import sysconfig; print(sysconfig._get_sysconfigdata_name())'
-            cf = self.command('-c', sc, output=str).strip()
-            self.sysconfigfilename = '{0}.py'.format(cf)
+        self._distutil_vars = {}
+
+        input_filename = self.get_sysconfigdata_name()
+        input_dict = None
+        try:
+            with open(input_filename) as input_file:
+                match = re.search(r'build_time_vars\s*=\s*(?P<dict>{.*})',
+                                  input_file.read(),
+                                  flags=re.DOTALL)
+
+                if match:
+                    input_dict = ast.literal_eval(match.group('dict'))
+        except (IOError, SyntaxError):
+            pass
+
+        if not input_dict:
+            tty.warn("Failed to find 'build_time_vars' dictionary in file "
+                     "'%s'. This might cause the extensions that are "
+                     "installed with distutils to call compilers directly "
+                     "avoiding Spack's wrappers." % input_filename)
+            return
+
+        for var_name in Python._DISTUTIL_VARS_TO_SAVE:
+            if var_name in input_dict:
+                self._distutil_vars[var_name] = input_dict[var_name]
+            else:
+                tty.warn("Failed to find key '%s' in 'build_time_vars' "
+                         "dictionary in file '%s'. This might cause the "
+                         "extensions that are installed with distutils to "
+                         "call compilers directly avoiding Spack's wrappers."
+                         % (var_name, input_filename))
+
+        if len(self._distutil_vars) > 0:
+            output_filename = None
+            try:
+                output_filename = join_path(
+                    spack.store.layout.metadata_path(self.spec),
+                    Python._DISTUTIL_CACHE_FILENAME)
+                with open(output_filename, 'w') as output_file:
+                    sjson.dump(self._distutil_vars, output_file)
+            except Exception:
+                tty.warn("Failed to save metadata for distutils. This might "
+                         "cause the extensions that are installed with "
+                         "distutils to call compilers directly avoiding "
+                         "Spack's wrappers.")
+                # We make the cache empty if we failed to save it to file
+                # to provide the same behaviour as in the case when the cache
+                # is initialized by the method load_distutils_data().
+                self._distutil_vars = {}
+                if output_filename:
+                    force_remove(output_filename)
 
-        self._save_distutil_vars(prefix)
+    def _load_distutil_vars(self):
+        # We update and keep the cache unchanged only if the package is
+        # installed.
+        if not self._distutil_vars and self.installed:
+            try:
+                input_filename = join_path(
+                    spack.store.layout.metadata_path(self.spec),
+                    Python._DISTUTIL_CACHE_FILENAME)
+                if os.path.isfile(input_filename):
+                    with open(input_filename) as input_file:
+                        self._distutil_vars = sjson.load(input_file)
+            except Exception:
+                pass
+
+            if not self._distutil_vars:
+                self._distutil_vars = {}
 
-        self.filter_compilers(prefix)
+        return self._distutil_vars
+
+    @run_after('install')
+    def filter_compilers(self):
+        """Run after install to tell the configuration files and Makefiles
+        to use the compilers that Spack built the package with.
+
+        If this isn't done, they'll have CC and CXX set to Spack's generic
+        cc and c++. We want them to be bound to whatever compiler
+        they were built with."""
+
+        kwargs = {'ignore_absent': True, 'backup': False, 'string': True}
+
+        filenames = [
+            self.get_sysconfigdata_name(), self.get_makefile_filename()
+        ]
+
+        filter_file(spack_cc,  self.compiler.cc,  *filenames, **kwargs)
+        filter_file(spack_cxx, self.compiler.cxx, *filenames, **kwargs)
+
+    @run_after('install')
+    def symlink(self):
+        spec = self.spec
+        prefix = self.prefix
 
         # TODO:
         # On OpenSuse 13, python uses <prefix>/lib64/python2.7/lib-dynload/*.so
@@ -398,125 +489,6 @@ class Python(AutotoolsPackage):
             if '+uuid' in spec:
                 self.command('-c', 'import uuid')
 
-    def _save_distutil_vars(self, prefix):
-        """
-        Run before changing automatically generated contents of the
-        _sysconfigdata.py, which is used by distutils to figure out what
-        executables to use while compiling and linking extensions. If we build
-        extensions with spack those executables should be spack's wrappers.
-        Spack partially covers this by setting environment variables that
-        are also accounted for by distutils. Currently there is one more known
-        variable that must be set, which is LDSHARED, so the method saves its
-        autogenerated value to pass it to the dependent package's setup script.
-        """
-
-        self._distutil_vars = {}
-
-        input_filename = None
-        for filename in [join_path(lib_dir,
-                                   'python{0}'.format(self.version.up_to(2)),
-                                   self.sysconfigfilename)
-                         for lib_dir in [prefix.lib, prefix.lib64]]:
-            if os.path.isfile(filename):
-                input_filename = filename
-                break
-        if not input_filename:
-            return
-
-        input_dict = None
-        try:
-            with open(input_filename) as input_file:
-                match = re.search(r'build_time_vars\s*=\s*(?P<dict>{.*})',
-                                  input_file.read(),
-                                  flags=re.DOTALL)
-
-                if match:
-                    input_dict = ast.literal_eval(match.group('dict'))
-        except (IOError, SyntaxError):
-            pass
-
-        if not input_dict:
-            tty.warn("Failed to find 'build_time_vars' dictionary in file "
-                     "'%s'. This might cause the extensions that are "
-                     "installed with distutils to call compilers directly "
-                     "avoiding Spack's wrappers." % input_filename)
-            return
-
-        for var_name in Python._DISTUTIL_VARS_TO_SAVE:
-            if var_name in input_dict:
-                self._distutil_vars[var_name] = input_dict[var_name]
-            else:
-                tty.warn("Failed to find key '%s' in 'build_time_vars' "
-                         "dictionary in file '%s'. This might cause the "
-                         "extensions that are installed with distutils to "
-                         "call compilers directly avoiding Spack's wrappers."
-                         % (var_name, input_filename))
-
-        if len(self._distutil_vars) > 0:
-            output_filename = None
-            try:
-                output_filename = join_path(
-                    spack.store.layout.metadata_path(self.spec),
-                    Python._DISTUTIL_CACHE_FILENAME)
-                with open(output_filename, 'w') as output_file:
-                    sjson.dump(self._distutil_vars, output_file)
-            except Exception:
-                tty.warn("Failed to save metadata for distutils. This might "
-                         "cause the extensions that are installed with "
-                         "distutils to call compilers directly avoiding "
-                         "Spack's wrappers.")
-                # We make the cache empty if we failed to save it to file
-                # to provide the same behaviour as in the case when the cache
-                # is initialized by the method load_distutils_data().
-                self._distutil_vars = {}
-                if output_filename:
-                    force_remove(output_filename)
-
-    def _load_distutil_vars(self):
-        # We update and keep the cache unchanged only if the package is
-        # installed.
-        if not self._distutil_vars and self.installed:
-            try:
-                input_filename = join_path(
-                    spack.store.layout.metadata_path(self.spec),
-                    Python._DISTUTIL_CACHE_FILENAME)
-                if os.path.isfile(input_filename):
-                    with open(input_filename) as input_file:
-                        self._distutil_vars = sjson.load(input_file)
-            except Exception:
-                pass
-
-            if not self._distutil_vars:
-                self._distutil_vars = {}
-
-        return self._distutil_vars
-
-    def filter_compilers(self, prefix):
-        """Run after install to tell the configuration files and Makefiles
-        to use the compilers that Spack built the package with.
-
-        If this isn't done, they'll have CC and CXX set to Spack's generic
-        cc and c++. We want them to be bound to whatever compiler
-        they were built with."""
-
-        kwargs = {'ignore_absent': True, 'backup': False, 'string': True}
-
-        lib_dirnames = [
-            join_path(lib_dir, 'python{0}'.format(self.version.up_to(2))) for
-            lib_dir in [prefix.lib, prefix.lib64]]
-
-        config_dirname = 'config-{0}m'.format(
-            self.version.up_to(2)) if self.spec.satisfies('@3:') else 'config'
-
-        rel_filenames = [self.sysconfigfilename,
-                         join_path(config_dirname, 'Makefile')]
-
-        abs_filenames = [join_path(dirname, filename) for dirname in
-                         lib_dirnames for filename in rel_filenames]
-
-        filter_file(env['CC'], self.compiler.cc, *abs_filenames, **kwargs)
-        filter_file(env['CXX'], self.compiler.cxx, *abs_filenames, **kwargs)
-
     # ========================================================================
     # Set up environment to make install easy for python extensions.
     # ========================================================================
@@ -579,7 +551,7 @@ class Python(AutotoolsPackage):
             return 'print({0})'.format(string)
 
     def get_config_var(self, key):
-        """Returns the value of a single variable. Wrapper around
+        """Return the value of a single variable. Wrapper around
         ``distutils.sysconfig.get_config_var()``."""
 
         cmd = 'from distutils.sysconfig import get_config_var; '
@@ -588,7 +560,7 @@ class Python(AutotoolsPackage):
         return self.command('-c', cmd, output=str).strip()
 
     def get_config_h_filename(self):
-        """Returns the full path name of the configuration header.
+        """Return the full path name of the configuration header.
         Wrapper around ``distutils.sysconfig.get_config_h_filename()``."""
 
         cmd = 'from distutils.sysconfig import get_config_h_filename; '
@@ -596,6 +568,50 @@ class Python(AutotoolsPackage):
 
         return self.command('-c', cmd, output=str).strip()
 
+    def get_makefile_filename(self):
+        """Return the full path name of ``Makefile`` used to build Python.
+        Wrapper around ``distutils.sysconfig.get_makefile_filename()``."""
+
+        cmd = 'from distutils.sysconfig import get_makefile_filename; '
+        cmd += self.print_string('get_makefile_filename()')
+
+        return self.command('-c', cmd, output=str).strip()
+
+    def get_python_inc(self):
+        """Return the directory for either the general or platform-dependent C
+        include files. Wrapper around ``distutils.sysconfig.get_python_inc()``.
+        """
+
+        cmd = 'from distutils.sysconfig import get_python_inc; '
+        cmd += self.print_string('get_python_inc()')
+
+        return self.command('-c', cmd, output=str).strip()
+
+    def get_python_lib(self):
+        """Return the directory for either the general or platform-dependent
+        library installation. Wrapper around
+        ``distutils.sysconfig.get_python_lib()``."""
+
+        cmd = 'from distutils.sysconfig import get_python_lib; '
+        cmd += self.print_string('get_python_lib()')
+
+        return self.command('-c', cmd, output=str).strip()
+
+    def get_sysconfigdata_name(self):
+        """Return the full path name of the sysconfigdata file."""
+
+        libdest = self.get_config_var('LIBDEST')
+
+        filename = '_sysconfigdata.py'
+        if self.spec.satisfies('@3.6:'):
+            # Python 3.6.0 renamed the sys config file
+            cmd = 'from sysconfig import _get_sysconfigdata_name; '
+            cmd += self.print_string('_get_sysconfigdata_name()')
+            filename = self.command('-c', cmd, output=str).strip()
+            filename += '.py'
+
+        return join_path(libdest, filename)
+
     @property
     def home(self):
         """Most of the time, ``PYTHONHOME`` is simply