Use python extend_path as pyqt sip fix (#15297)

* try extend path to solve PyQt5.sip not found issue

* disable private sip installation in sippackage class

* undo manual PyQt5 dir creation in py-sip site-packages dir

* fix typo

* fix typo

* also apply fix to PyQt4

* tidy up

* flake8 and tidy up

* tidy and undo hardcoding of python_include_dir

* replace hardcoded python inc dir

* fix minor issues

* rethink include dir variable name

* improve style

* add new versions

* implement new sip setup to qsci installation

* set sip-incdir correctly for the new setup

* setup extend_path thing before qsci python bindings

* take care of conflict

* flake8

* also extend for PyQt4

* improve style

* improve style

* SipPackage build sys should depend on py-sip

* consolidate extend_path fixes into SipPackage

* fix typo

* fix bugs

* flake8

* revert sip doc to pre-resource setup

* import os module

* flake8

Co-authored-by: Sinan81 <sbulut@3vgeomatics.com>

5 files changed, 53 insertions, 41 deletions

diff --git a/lib/spack/docs/build_systems/sippackage.rst b/lib/spack/docs/build_systems/sippackage.rst
index b8c08ec513..ddf9a26ab9 100644
--- a/lib/spack/docs/build_systems/sippackage.rst
+++ b/lib/spack/docs/build_systems/sippackage.rst
@@ -51,10 +51,8 @@ Build system dependencies
 ``SIPPackage`` requires several dependencies. Python is needed to run
 the ``configure.py`` build script, and to run the resulting Python
 libraries. Qt is needed to provide the ``qmake`` command. SIP is also
-needed to build the package. SIP is an unusual dependency in that it
-must be installed in the same installation directory as the package,
-so instead of a ``depends_on``, we use a ``resource``. All of these
-dependencies are automatically added via the base class
+needed to build the package. All of these dependencies are automatically
+added via the base class
 
 .. code-block:: python
 
@@ -62,11 +60,7 @@ dependencies are automatically added via the base class
 
    depends_on('qt', type='build')
 
-   resource(name='sip',
-            url='https://www.riverbankcomputing.com/static/Downloads/sip/4.19.18/sip-4.19.18.tar.gz',
-            sha256='c0bd863800ed9b15dcad477c4017cdb73fa805c25908b0240564add74d697e1e',
-            destination='.')
-
+   depends_on('py-sip', type='build')
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Passing arguments to ``configure.py``
diff --git a/lib/spack/spack/build_systems/sip.py b/lib/spack/spack/build_systems/sip.py
index 314f91d5d2..f814ef1837 100644
--- a/lib/spack/spack/build_systems/sip.py
+++ b/lib/spack/spack/build_systems/sip.py
@@ -5,9 +5,10 @@
 
 import inspect
 
-from llnl.util.filesystem import working_dir
-from spack.directives import depends_on, extends, resource
-from spack.package import PackageBase, run_before, run_after
+from llnl.util.filesystem import working_dir, join_path
+from spack.directives import depends_on, extends
+from spack.package import PackageBase, run_after
+import os
 
 
 class SIPPackage(PackageBase):
@@ -40,33 +41,12 @@ class SIPPackage(PackageBase):
     extends('python')
 
     depends_on('qt')
-
-    resource(name='sip',
-             url='https://www.riverbankcomputing.com/static/Downloads/sip/4.19.18/sip-4.19.18.tar.gz',
-             sha256='c0bd863800ed9b15dcad477c4017cdb73fa805c25908b0240564add74d697e1e',
-             destination='.')
+    depends_on('py-sip')
 
     def python(self, *args, **kwargs):
         """The python ``Executable``."""
         inspect.getmodule(self).python(*args, **kwargs)
 
-    @run_before('configure')
-    def install_sip(self):
-        args = [
-            '--sip-module={0}'.format(self.sip_module),
-            '--bindir={0}'.format(self.prefix.bin),
-            '--destdir={0}'.format(inspect.getmodule(self).site_packages_dir),
-            '--incdir={0}'.format(inspect.getmodule(self).python_include_dir),
-            '--sipdir={0}'.format(self.prefix.share.sip),
-            '--stubsdir={0}'.format(inspect.getmodule(self).site_packages_dir),
-        ]
-
-        with working_dir('sip-4.19.18'):
-            self.python('configure.py', *args)
-
-            inspect.getmodule(self).make()
-            inspect.getmodule(self).make('install')
-
     def configure_file(self):
         """Returns the name of the configure file to use."""
         return 'configure.py'
@@ -77,12 +57,15 @@ class SIPPackage(PackageBase):
 
         args = self.configure_args()
 
+        python_include_dir = 'python' + str(spec['python'].version.up_to(2))
+
         args.extend([
             '--verbose',
             '--confirm-license',
             '--qmake', spec['qt'].prefix.bin.qmake,
-            '--sip', prefix.bin.sip,
-            '--sip-incdir', inspect.getmodule(self).python_include_dir,
+            '--sip', spec['py-sip'].prefix.bin.sip,
+            '--sip-incdir', join_path(spec['py-sip'].prefix.include,
+                                      python_include_dir),
             '--bindir', prefix.bin,
             '--destdir', inspect.getmodule(self).site_packages_dir,
         ])
@@ -131,3 +114,14 @@ class SIPPackage(PackageBase):
 
     # Check that self.prefix is there after installation
     run_after('install')(PackageBase.sanity_check_prefix)
+
+    @run_after('install')
+    def extend_path_setup(self):
+        # See github issue #14121 and PR #15297
+        module = self.spec['py-sip'].variants['module'].value
+        if module != 'sip':
+            module = module.split('.')[0]
+            with working_dir(inspect.getmodule(self).site_packages_dir):
+                with open(os.path.join(module, '__init__.py'), 'a') as f:
+                    f.write('from pkgutil import extend_path\n')
+                    f.write('__path__ = extend_path(__path__, __name__)\n')
diff --git a/var/spack/repos/builtin/packages/py-pyqt4/package.py b/var/spack/repos/builtin/packages/py-pyqt4/package.py
index c0e3ae1085..d1a90042d9 100644
--- a/var/spack/repos/builtin/packages/py-pyqt4/package.py
+++ b/var/spack/repos/builtin/packages/py-pyqt4/package.py
@@ -34,6 +34,7 @@ class PyPyqt4(SIPPackage):
     # Supposedly can also be built with Qt 5 compatibility layer
     depends_on('qt@:4')
     depends_on('qscintilla', when='+qsci')
+    depends_on('py-sip module=PyQt4.sip')
 
     # For building Qscintilla python bindings
     resource(name='qscintilla',
@@ -68,7 +69,7 @@ class PyPyqt4(SIPPackage):
                 pydir = join_path(site_packages_dir, 'PyQt4')
                 python = self.spec['python'].command
                 python('configure.py',
-                       '--sip=' + self.prefix.bin.sip,
+                       '--sip=' + self.spec['py-sip'].prefix.bin.sip,
                        '--qsci-incdir=' +
                        self.spec['qscintilla'].prefix.include,
                        '--qsci-libdir=' + self.spec['qscintilla'].prefix.lib,
@@ -76,7 +77,10 @@ class PyPyqt4(SIPPackage):
                        '--apidir=' + self.prefix.share.qsci,
                        '--destdir=' + pydir,
                        '--pyqt-sipdir=' + self.prefix.share.sip.PyQt4,
-                       '--sip-incdir=' + python_include_dir,
+                       '--sip-incdir=' +
+                       join_path(self.spec['py-sip'].prefix.include,
+                                 'python' +
+                                 str(self.spec['python'].version.up_to(2))),
                        '--stubsdir=' + pydir)
 
                 # Fix build errors
diff --git a/var/spack/repos/builtin/packages/py-pyqt5/package.py b/var/spack/repos/builtin/packages/py-pyqt5/package.py
index b91833389b..bc37d4dcab 100644
--- a/var/spack/repos/builtin/packages/py-pyqt5/package.py
+++ b/var/spack/repos/builtin/packages/py-pyqt5/package.py
@@ -26,6 +26,7 @@ class PyPyqt5(SIPPackage):
         'PyQt5.QtXmlPatterns'
     ]
 
+    version('5.13.1', sha256='54b7f456341b89eeb3930e786837762ea67f235e886512496c4152ebe106d4af')
     version('5.13.0', sha256='0cdbffe5135926527b61cc3692dd301cd0328dd87eeaf1313e610787c46faff9')
     version('5.12.3', sha256='0db0fa37debab147450f9e052286f7a530404e2aaddc438e97a7dcdf56292110')
 
@@ -36,7 +37,8 @@ class PyPyqt5(SIPPackage):
     depends_on('qt@5:+opengl')
     depends_on('python@2.6:', type=('build', 'run'))
     depends_on('py-enum34', type=('build', 'run'), when='^python@:3.3')
-
+    depends_on('py-sip module=PyQt5.sip', type=('build', 'run'))
+    depends_on('py-sip@:4.19.18 module=PyQt5.sip', type=('build', 'run'), when='@:5.13.0')
     depends_on('qscintilla', when='+qsci')
 
     # For building Qscintilla python bindings
@@ -75,7 +77,7 @@ class PyPyqt5(SIPPackage):
                     'PyQt5')
                 python = self.spec['python'].command
                 python('configure.py', '--pyqt=PyQt5',
-                       '--sip=' + self.prefix.bin.sip,
+                       '--sip=' + self.spec['py-sip'].prefix.bin.sip,
                        '--qsci-incdir=' +
                        self.spec['qscintilla'].prefix.include,
                        '--qsci-libdir=' + self.spec['qscintilla'].prefix.lib,
@@ -83,7 +85,10 @@ class PyPyqt5(SIPPackage):
                        '--apidir=' + self.prefix.share.qsci,
                        '--destdir=' + pydir,
                        '--pyqt-sipdir=' + self.prefix.share.sip.PyQt5,
-                       '--sip-incdir=' + python_include_dir,
+                       '--sip-incdir=' +
+                       join_path(self.spec['py-sip'].prefix.include,
+                                 'python' +
+                                 str(self.spec['python'].version.up_to(2))),
                        '--stubsdir=' + pydir)
 
                 # Fix build errors
diff --git a/var/spack/repos/builtin/packages/py-sip/package.py b/var/spack/repos/builtin/packages/py-sip/package.py
index 09c6b743b1..d454f05740 100644
--- a/var/spack/repos/builtin/packages/py-sip/package.py
+++ b/var/spack/repos/builtin/packages/py-sip/package.py
@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 from spack import *
+import os
 
 
 class PySip(Package):
@@ -16,6 +17,9 @@ class PySip(Package):
     hg       = "https://www.riverbankcomputing.com/hg/sip"
 
     version('develop', hg=hg)  # wasn't actually able to clone this
+    version('4.19.21', sha256='6af9979ab41590e8311b8cc94356718429ef96ba0e3592bdd630da01211200ae')
+    version('4.19.20', sha256='04cc2f87ac97e8718d8e1ef036e3ec26050ab44c21f9277618d5b67432fcbfd6')
+    version('4.19.19', sha256='5436b61a78f48c7e8078e93a6b59453ad33780f80c644e5f3af39f94be1ede44')
     version('4.19.18', sha256='c0bd863800ed9b15dcad477c4017cdb73fa805c25908b0240564add74d697e1e')
     version('4.19.15', sha256='2b5c0b2c0266b467b365c21376d50dde61a3236722ab87ff1e8dacec283eb610')
     version('4.19.13', sha256='e353a7056599bf5fbd5d3ff9842a6ab2ea3cf4e0304a0f925ec5862907c0d15e')
@@ -53,3 +57,14 @@ class PySip(Package):
 
     def install(self, spec, prefix):
         make('install')
+
+    @run_after('install')
+    def extend_path_setup(self):
+        # See github issue #14121 and PR #15297
+        module = self.spec.variants['module'].value
+        if module != 'sip':
+            module = module.split('.')[0]
+            with working_dir(site_packages_dir):
+                with open(os.path.join(module, '__init__.py'), 'w') as f:
+                    f.write('from pkgutil import extend_path\n')
+                    f.write('__path__ = extend_path(__path__, __name__)\n')