Python: query distutils to find site-packages directory (#24095)

Third-party Python libraries may be installed in one of several directories:

1. `lib/pythonX.Y/site-packages` for Spack-installed Python
2. `lib64/pythonX.Y/site-packages` for system Python on RHEL/CentOS/Fedora
3. `lib/pythonX/dist-packages` for system Python on Debian/Ubuntu

Previously, Spack packages were hard-coded to use the (1). Now, we query the Python installation itself and ask it which to use. Ever since #21446 this is how we've been determining where to install Python libraries anyway.

Note: there are still many packages that are hard-coded to use (1). I can change them in this PR, but I don't have the bandwidth to test all of them.

* Python: handle dist-packages and site-packages
* Query Python to find site-packages directory
* Add try-except statements for when distutils isn't installed
* Catch more errors
* Fix root directory used in import tests
* Rely on site_packages_dir property

3 files changed, 32 insertions, 43 deletions

diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py
index 96f11b631d..896c1cad4e 100644
--- a/lib/spack/spack/bootstrap.py
+++ b/lib/spack/spack/bootstrap.py
@@ -95,10 +95,8 @@ def make_module_available(module, spec=None, install=False):
         # TODO: make sure run-environment is appropriate
         module_path = os.path.join(ispec.prefix,
                                    ispec['python'].package.site_packages_dir)
-        module_path_64 = module_path.replace('/lib/', '/lib64/')
         try:
             sys.path.append(module_path)
-            sys.path.append(module_path_64)
             __import__(module)
             return
         except ImportError:
@@ -122,10 +120,8 @@ def make_module_available(module, spec=None, install=False):
 
     module_path = os.path.join(spec.prefix,
                                spec['python'].package.site_packages_dir)
-    module_path_64 = module_path.replace('/lib/', '/lib64/')
     try:
         sys.path.append(module_path)
-        sys.path.append(module_path_64)
         __import__(module)
         return
     except ImportError:
diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py
index b4f55ab707..27b1d4a10c 100644
--- a/lib/spack/spack/build_systems/python.py
+++ b/lib/spack/spack/build_systems/python.py
@@ -127,24 +127,22 @@ class PythonPackage(PackageBase):
             list: list of strings of module names
         """
         modules = []
+        root = self.spec['python'].package.get_python_lib(prefix=self.prefix)
 
-        # Python libraries may be installed in lib or lib64
-        # See issues #18520 and #17126
-        for lib in ['lib', 'lib64']:
-            root = os.path.join(self.prefix, lib, 'python{0}'.format(
-                self.spec['python'].version.up_to(2)), 'site-packages')
-            # Some Python libraries are packages: collections of modules
-            # distributed in directories containing __init__.py files
-            for path in find(root, '__init__.py', recursive=True):
-                modules.append(path.replace(root + os.sep, '', 1).replace(
-                    os.sep + '__init__.py', '').replace('/', '.'))
-            # Some Python libraries are modules: individual *.py files
-            # found in the site-packages directory
-            for path in find(root, '*.py', recursive=False):
-                modules.append(path.replace(root + os.sep, '', 1).replace(
-                    '.py', '').replace('/', '.'))
+        # Some Python libraries are packages: collections of modules
+        # distributed in directories containing __init__.py files
+        for path in find(root, '__init__.py', recursive=True):
+            modules.append(path.replace(root + os.sep, '', 1).replace(
+                os.sep + '__init__.py', '').replace('/', '.'))
+
+        # Some Python libraries are modules: individual *.py files
+        # found in the site-packages directory
+        for path in find(root, '*.py', recursive=False):
+            modules.append(path.replace(root + os.sep, '', 1).replace(
+                '.py', '').replace('/', '.'))
 
         tty.debug('Detected the following modules: {0}'.format(modules))
+
         return modules
 
     def setup_file(self):
@@ -254,15 +252,12 @@ class PythonPackage(PackageBase):
         # Get all relative paths since we set the root to `prefix`
         # We query the python with which these will be used for the lib and inc
         # directories. This ensures we use `lib`/`lib64` as expected by python.
-        python = spec['python'].package.command
-        command_start = 'print(distutils.sysconfig.'
-        commands = ';'.join([
-            'import distutils.sysconfig',
-            command_start + 'get_python_lib(plat_specific=False, prefix=""))',
-            command_start + 'get_python_lib(plat_specific=True, prefix=""))',
-            command_start + 'get_python_inc(plat_specific=True, prefix=""))'])
-        pure_site_packages_dir, plat_site_packages_dir, inc_dir = python(
-            '-c', commands, output=str, error=str).strip().split('\n')
+        pure_site_packages_dir = spec['python'].package.get_python_lib(
+            plat_specific=False, prefix='')
+        plat_site_packages_dir = spec['python'].package.get_python_lib(
+            plat_specific=True, prefix='')
+        inc_dir = spec['python'].package.get_python_inc(
+            plat_specific=True, prefix='')
 
         args += ['--root=%s' % prefix,
                  '--install-purelib=%s' % pure_site_packages_dir,
diff --git a/lib/spack/spack/build_systems/sip.py b/lib/spack/spack/build_systems/sip.py
index c32c46ec46..744989e2d6 100644
--- a/lib/spack/spack/build_systems/sip.py
+++ b/lib/spack/spack/build_systems/sip.py
@@ -64,24 +64,22 @@ class SIPPackage(PackageBase):
             list: list of strings of module names
         """
         modules = []
+        root = self.spec['python'].package.get_python_lib(prefix=self.prefix)
 
-        # Python libraries may be installed in lib or lib64
-        # See issues #18520 and #17126
-        for lib in ['lib', 'lib64']:
-            root = os.path.join(self.prefix, lib, 'python{0}'.format(
-                self.spec['python'].version.up_to(2)), 'site-packages')
-            # Some Python libraries are packages: collections of modules
-            # distributed in directories containing __init__.py files
-            for path in find(root, '__init__.py', recursive=True):
-                modules.append(path.replace(root + os.sep, '', 1).replace(
-                    os.sep + '__init__.py', '').replace('/', '.'))
-            # Some Python libraries are modules: individual *.py files
-            # found in the site-packages directory
-            for path in find(root, '*.py', recursive=False):
-                modules.append(path.replace(root + os.sep, '', 1).replace(
-                    '.py', '').replace('/', '.'))
+        # Some Python libraries are packages: collections of modules
+        # distributed in directories containing __init__.py files
+        for path in find(root, '__init__.py', recursive=True):
+            modules.append(path.replace(root + os.sep, '', 1).replace(
+                os.sep + '__init__.py', '').replace('/', '.'))
+
+        # Some Python libraries are modules: individual *.py files
+        # found in the site-packages directory
+        for path in find(root, '*.py', recursive=False):
+            modules.append(path.replace(root + os.sep, '', 1).replace(
+                '.py', '').replace('/', '.'))
 
         tty.debug('Detected the following modules: {0}'.format(modules))
+
         return modules
 
     def python(self, *args, **kwargs):