Python flake8 overhaul

1 files changed, 114 insertions, 76 deletions

diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
index d6d5fac4b1..b6a62bf742 100644
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -22,9 +22,6 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import functools
-import glob
-import inspect
 import os
 import re
 from contextlib import closing
@@ -37,11 +34,10 @@ from spack.util.environment import *
 
 class Python(Package):
     """The Python programming language."""
+
     homepage = "http://www.python.org"
     url      = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz"
 
-    extendable = True
-
     version('3.5.1', 'be78e48cdfc1a7ad90efff146dce6cfe')
     version('3.5.0', 'a56c0c0b45d75a0ec9c6dee933c41c36')
     version('2.7.11', '6b6076ec9e93f05dd63e47eb9c15728b', preferred=True)
@@ -49,6 +45,8 @@ class Python(Package):
     version('2.7.9', '5eebcaa0030dc4061156d3429657fb83')
     version('2.7.8', 'd4bca0159acb0b44a781292b5231936f')
 
+    extendable = True
+
     depends_on("openssl")
     depends_on("bzip2")
     depends_on("readline")
@@ -64,39 +62,63 @@ class Python(Package):
         # Rest of install is pretty standard except setup.py needs to
         # be able to read the CPPFLAGS and LDFLAGS as it scans for the
         # library and headers to build
-        configure_args= [
-                  "--prefix=%s" % prefix,
-                  "--with-threads",
-                  "--enable-shared",
-                  "CPPFLAGS=-I%s/include -I%s/include -I%s/include -I%s/include -I%s/include -I%s/include" % (
-                       spec['openssl'].prefix, spec['bzip2'].prefix,
-                       spec['readline'].prefix, spec['ncurses'].prefix,
-                       spec['sqlite'].prefix, spec['zlib'].prefix),
-                  "LDFLAGS=-L%s/lib -L%s/lib -L%s/lib -L%s/lib -L%s/lib -L%s/lib" % (
-                       spec['openssl'].prefix, spec['bzip2'].prefix,
-                       spec['readline'].prefix, spec['ncurses'].prefix,
-                       spec['sqlite'].prefix, spec['zlib'].prefix)
-                  ]
+        cppflags = ' -I'.join([
+            spec['openssl'].prefix.include,  spec['bzip2'].prefix.include,
+            spec['readline'].prefix.include, spec['ncurses'].prefix.include,
+            spec['sqlite'].prefix.include,   spec['zlib'].prefix.include
+        ])
+
+        ldflags = ' -L'.join([
+            spec['openssl'].prefix.lib,  spec['bzip2'].prefix.lib,
+            spec['readline'].prefix.lib, spec['ncurses'].prefix.lib,
+            spec['sqlite'].prefix.lib,   spec['zlib'].prefix.lib
+        ])
+
+        config_args = [
+            "--prefix={0}".format(prefix),
+            "--with-threads",
+            "--enable-shared",
+            "CPPFLAGS=-I{0}".format(cppflags),
+            "LDFLAGS=-L{0}".format(ldflags)
+        ]
+
         if spec.satisfies('@3:'):
-            configure_args.append('--without-ensurepip')
-        configure(*configure_args)
+            config_args.append('--without-ensurepip')
+
+        configure(*config_args)
+
         make()
         make("install")
 
-        # Modify compiler paths in configuration files. This is necessary for
-        # building site packages outside of spack
-        filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)',
-                    (r'\4\5'),
-                    join_path(prefix.lib, 'python%d.%d' % self.version[:2], '_sysconfigdata.py'))
+        self.filter_compilers(spec, prefix)
+
+    def filter_compilers(self, spec, prefix):
+        """Run after install to tell the configuration files and Makefiles
+        to use the compilers that Spack built the package with.
+
+        If this isn't done, they'll have CC and CXX set to Spack's generic
+        cc and c++. We want them to be bound to whatever compiler
+        they were built with."""
+
+        kwargs = {'ignore_absent': True, 'backup': False, 'string': True}
 
-        python3_version = ''
+        dirname = join_path(prefix.lib,
+                            'python{0}'.format(self.version.up_to(2)))
+
+        config = 'config'
         if spec.satisfies('@3:'):
-            python3_version = '-%d.%dm' % self.version[:2]
-        makefile_filepath = join_path(prefix.lib, 'python%d.%d' % self.version[:2], 'config%s' % python3_version, 'Makefile')
-        filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)',
-                    (r'\4\5'),
-                    makefile_filepath)
+            config = 'config-{0}m'.format(self.version.up_to(2))
+
+        files = [
+            '_sysconfigdata.py',
+            join_path(config, 'Makefile')
+        ]
 
+        for filename in files:
+            filter_file(env['CC'],  self.compiler.cc,
+                        join_path(dirname, filename), **kwargs)
+            filter_file(env['CXX'], self.compiler.cxx,
+                        join_path(dirname, filename), **kwargs)
 
     # ========================================================================
     # Set up environment to make install easy for python extensions.
@@ -104,57 +126,59 @@ class Python(Package):
 
     @property
     def python_lib_dir(self):
-        return os.path.join('lib', 'python%d.%d' % self.version[:2])
-
+        return join_path('lib', 'python{0}'.format(self.version.up_to(2)))
 
     @property
     def python_include_dir(self):
-        return os.path.join('include', 'python%d.%d' % self.version[:2])
-
+        return join_path('include', 'python{0}'.format(self.version.up_to(2)))
 
     @property
     def site_packages_dir(self):
-        return os.path.join(self.python_lib_dir, 'site-packages')
-
+        return join_path(self.python_lib_dir, 'site-packages')
 
     def setup_dependent_environment(self, spack_env, run_env, extension_spec):
-        # TODO: do this only for actual extensions.
+        """Set PYTHONPATH to include site-packages dir for the
+        extension and any other python extensions it depends on."""
 
-        # Set PYTHONPATH to include site-packages dir for the
-        # extension and any other python extensions it depends on.
         python_paths = []
         for d in extension_spec.traverse():
             if d.package.extends(self.spec):
-                python_paths.append(os.path.join(d.prefix, self.site_packages_dir))
+                python_paths.append(join_path(d.prefix,
+                                              self.site_packages_dir))
 
         pythonpath = ':'.join(python_paths)
         spack_env.set('PYTHONPATH', pythonpath)
 
-        # For run time environment set only the path for extension_spec and prepend it to PYTHONPATH
+        # For run time environment set only the path for
+        # extension_spec and prepend it to PYTHONPATH
         if extension_spec.package.extends(self.spec):
-            run_env.prepend_path('PYTHONPATH', os.path.join(extension_spec.prefix, self.site_packages_dir))
-
+            run_env.prepend_path('PYTHONPATH', join_path(
+                extension_spec.prefix, self.site_packages_dir))
 
     def setup_dependent_package(self, module, ext_spec):
-        """
-        Called before python modules' install() methods.
+        """Called before python modules' install() methods.
 
         In most cases, extensions will only need to have one line::
 
-        python('setup.py', 'install', '--prefix=%s' % prefix)
-        """
+        python('setup.py', 'install', '--prefix={0}'.format(prefix))"""
+
         # Python extension builds can have a global python executable function
-        if self.version >= Version("3.0.0") and self.version < Version("4.0.0"):
-            module.python = Executable(join_path(self.spec.prefix.bin, 'python3'))
+        if Version("3.0.0") <= self.version < Version("4.0.0"):
+            module.python = Executable(join_path(self.spec.prefix.bin,
+                                                 'python3'))
         else:
-            module.python = Executable(join_path(self.spec.prefix.bin, 'python'))
+            module.python = Executable(join_path(self.spec.prefix.bin,
+                                                 'python'))
 
         # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs.
-        module.python_lib_dir     = os.path.join(ext_spec.prefix, self.python_lib_dir)
-        module.python_include_dir = os.path.join(ext_spec.prefix, self.python_include_dir)
-        module.site_packages_dir  = os.path.join(ext_spec.prefix, self.site_packages_dir)
-
-        # Make the site packages directory for extensions, if it does not exist already.
+        module.python_lib_dir     = join_path(ext_spec.prefix,
+                                              self.python_lib_dir)
+        module.python_include_dir = join_path(ext_spec.prefix,
+                                              self.python_include_dir)
+        module.site_packages_dir  = join_path(ext_spec.prefix,
+                                              self.site_packages_dir)
+
+        # Make the site packages directory for extensions
         if ext_spec.package.is_extension:
             mkdirp(module.site_packages_dir)
 
@@ -170,21 +194,25 @@ class Python(Package):
         patterns = [r'site-packages/easy-install\.pth$']
 
         # Ignore pieces of setuptools installed by other packages.
+        # Must include directory name or it will remove all site*.py files.
         if ext_pkg.name != 'py-setuptools':
-            patterns.append(r'bin/easy_install[^/]*$')
-            patterns.append(r'site-packages/setuptools\.pth$')
-            patterns.append(r'site-packages/site[^/]*\.pyc?$')
+            patterns.extend([
+                r'bin/easy_install[^/]*$',
+                r'site-packages/setuptools[^/]*\.egg$',
+                r'site-packages/setuptools\.pth$',
+                r'site-packages/site[^/]*\.pyc?$',
+                r'site-packages/__pycache__/site[^/]*\.pyc?$'
+            ])
         if ext_pkg.name != 'py-numpy':
             patterns.append(r'bin/f2py$')
 
         return match_predicate(ignore_arg, patterns)
 
-
     def write_easy_install_pth(self, exts):
         paths = []
         for ext in sorted(exts.values()):
-            ext_site_packages = os.path.join(ext.prefix, self.site_packages_dir)
-            easy_pth = "%s/easy-install.pth" % ext_site_packages
+            ext_site_packages = join_path(ext.prefix, self.site_packages_dir)
+            easy_pth = join_path(ext_site_packages, "easy-install.pth")
 
             if not os.path.isfile(easy_pth):
                 continue
@@ -194,15 +222,18 @@ class Python(Package):
                     line = line.rstrip()
 
                     # Skip lines matching these criteria
-                    if not line: continue
-                    if re.search(r'^(import|#)', line): continue
-                    if (ext.name != 'py-setuptools' and
-                        re.search(r'setuptools.*egg$', line)): continue
+                    if not line:
+                        continue
+                    if re.search(r'^(import|#)', line):
+                        continue
+                    if ((ext.name != 'py-setuptools' and
+                         re.search(r'setuptools.*egg$', line))):
+                        continue
 
                     paths.append(line)
 
-        site_packages = os.path.join(self.prefix, self.site_packages_dir)
-        main_pth = "%s/easy-install.pth" % site_packages
+        site_packages = join_path(self.prefix, self.site_packages_dir)
+        main_pth = join_path(site_packages, "easy-install.pth")
 
         if not paths:
             if os.path.isfile(main_pth):
@@ -210,15 +241,22 @@ class Python(Package):
 
         else:
             with closing(open(main_pth, 'w')) as f:
-                f.write("import sys; sys.__plen = len(sys.path)\n")
+                f.write("""
+import sys
+sys.__plen = len(sys.path)
+""")
                 for path in paths:
-                    f.write("%s\n" % path)
-                f.write("import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; "
-                        "p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new)\n")
-
+                    f.write("{0}\n".format(path))
+                f.write("""
+new = sys.path[sys.__plen:]
+del sys.path[sys.__plen:]
+p = getattr(sys, '__egginsert', 0)
+sys.path[p:p] = new
+sys.__egginsert = p + len(new)
+""")
 
     def activate(self, ext_pkg, **args):
-        ignore=self.python_ignore(ext_pkg, args)
+        ignore = self.python_ignore(ext_pkg, args)
         args.update(ignore=ignore)
 
         super(Python, self).activate(ext_pkg, **args)
@@ -227,12 +265,12 @@ class Python(Package):
         exts[ext_pkg.name] = ext_pkg.spec
         self.write_easy_install_pth(exts)
 
-
     def deactivate(self, ext_pkg, **args):
         args.update(ignore=self.python_ignore(ext_pkg, args))
         super(Python, self).deactivate(ext_pkg, **args)
 
         exts = spack.install_layout.extension_map(self.spec)
-        if ext_pkg.name in exts:        # Make deactivate idempotent.
+        # Make deactivate idempotent
+        if ext_pkg.name in exts:
             del exts[ext_pkg.name]
             self.write_easy_install_pth(exts)