Python: improve site_packages_dir handling (#28346)

* Python: improve site_packages_dir handling

* Replace all site_packages_dir with purelib/platlib

43 files changed, 159 insertions, 192 deletions

diff --git a/var/spack/repos/builtin/packages/ascent/package.py b/var/spack/repos/builtin/packages/ascent/package.py
index c57f0e3d33..8c974d8cc5 100644
--- a/var/spack/repos/builtin/packages/ascent/package.py
+++ b/var/spack/repos/builtin/packages/ascent/package.py
@@ -392,7 +392,7 @@ class Ascent(CMakePackage, CudaPackage):
             try:
                 cfg.write("# python module install dir\n")
                 cfg.write(cmake_cache_entry("PYTHON_MODULE_INSTALL_PREFIX",
-                          site_packages_dir))
+                          python_platlib))
             except NameError:
                 # spack's  won't exist in a subclass
                 pass
diff --git a/var/spack/repos/builtin/packages/bart/package.py b/var/spack/repos/builtin/packages/bart/package.py
index 897572e2c7..51e66a1213 100644
--- a/var/spack/repos/builtin/packages/bart/package.py
+++ b/var/spack/repos/builtin/packages/bart/package.py
@@ -64,18 +64,15 @@ class Bart(MakefilePackage, CudaPackage):
             env['GPUARCH_FLAGS'] = ' '.join(self.cuda_flags(cuda_arch))
 
     def install(self, spec, prefix):
-        python_dir = join_path(prefix,
-                               spec['python'].package.site_packages_dir)
-
         make('install')
 
         install_tree('scripts', prefix.scripts)
         install_tree('matlab', prefix.matlab)
         install('startup.m', prefix)
 
-        install('python/bart.py', python_dir)
-        install('python/cfl.py', python_dir)
-        install('python/wslsupport.py', python_dir)
+        install('python/bart.py', python_platlib)
+        install('python/cfl.py', python_platlib)
+        install('python/wslsupport.py', python_platlib)
 
         if '^python@3:' in spec:
             install('python/bartview3.py', join_path(prefix.bin, 'bartview'))
diff --git a/var/spack/repos/builtin/packages/bohrium/package.py b/var/spack/repos/builtin/packages/bohrium/package.py
index 3e5895fb2d..d1c2e41bbc 100644
--- a/var/spack/repos/builtin/packages/bohrium/package.py
+++ b/var/spack/repos/builtin/packages/bohrium/package.py
@@ -234,8 +234,7 @@ class Bohrium(CMakePackage, CudaPackage):
 
         # Add the PYTHONPATH to bohrium to the PYTHONPATH environment
         pythonpaths = [p for p in os.environ["PYTHONPATH"].split(":")]
-        pythonpaths.append(join_path(self.prefix,
-                                     spec['python'].package.site_packages_dir))
+        pythonpaths.append(python_platlib)
         test_env["PYTHONPATH"] = ":".join(pythonpaths)
 
         # Collect the stacks which should be available:
diff --git a/var/spack/repos/builtin/packages/clingo/package.py b/var/spack/repos/builtin/packages/clingo/package.py
index e4eb15c5e0..bc6f97515f 100644
--- a/var/spack/repos/builtin/packages/clingo/package.py
+++ b/var/spack/repos/builtin/packages/clingo/package.py
@@ -68,12 +68,10 @@ class Clingo(CMakePackage):
         """Return standard CMake defines to ensure that the
         current spec is the one found by CMake find_package(Python, ...)
         """
-        python_spec = self.spec['python']
-        include_dir = join_path(
-            python_spec.prefix, python_spec.package.config_vars['python_inc']['false'])
+        python = self.spec['python']
         return [
-            self.define('Python_EXECUTABLE', str(python_spec.command)),
-            self.define('Python_INCLUDE_DIR', include_dir)
+            self.define('Python_EXECUTABLE', str(python.command)),
+            self.define('Python_INCLUDE_DIR', python.package.config_vars['include'])
         ]
 
     @property
diff --git a/var/spack/repos/builtin/packages/conduit/package.py b/var/spack/repos/builtin/packages/conduit/package.py
index b7b293f0b9..fbb6a33100 100644
--- a/var/spack/repos/builtin/packages/conduit/package.py
+++ b/var/spack/repos/builtin/packages/conduit/package.py
@@ -444,7 +444,7 @@ class Conduit(CMakePackage):
             try:
                 cfg.write("# python module install dir\n")
                 cfg.write(cmake_cache_entry("PYTHON_MODULE_INSTALL_PREFIX",
-                          site_packages_dir))
+                          python_platlib))
             except NameError:
                 # spack's  won't exist in a subclass
                 pass
diff --git a/var/spack/repos/builtin/packages/elemental/package.py b/var/spack/repos/builtin/packages/elemental/package.py
index be006f93c1..2336c799ea 100644
--- a/var/spack/repos/builtin/packages/elemental/package.py
+++ b/var/spack/repos/builtin/packages/elemental/package.py
@@ -167,6 +167,6 @@ class Elemental(CMakePackage):
 
         if '+python' in spec:
             args.extend([
-                '-DPYTHON_SITE_PACKAGES:STRING={0}'.format(site_packages_dir)])
+                '-DPYTHON_SITE_PACKAGES:STRING={0}'.format(python_platlib)])
 
         return args
diff --git a/var/spack/repos/builtin/packages/faiss/package.py b/var/spack/repos/builtin/packages/faiss/package.py
index 2e7162901e..662fe1677f 100644
--- a/var/spack/repos/builtin/packages/faiss/package.py
+++ b/var/spack/repos/builtin/packages/faiss/package.py
@@ -125,4 +125,4 @@ class Faiss(AutotoolsPackage, CudaPackage):
 
     def setup_run_environment(self, env):
         if '+python' in self.spec:
-            env.prepend_path('PYTHONPATH', site_packages_dir)
+            env.prepend_path('PYTHONPATH', python_platlib)
diff --git a/var/spack/repos/builtin/packages/flann/package.py b/var/spack/repos/builtin/packages/flann/package.py
index d186f2f8c4..725edb5207 100644
--- a/var/spack/repos/builtin/packages/flann/package.py
+++ b/var/spack/repos/builtin/packages/flann/package.py
@@ -89,7 +89,7 @@ class Flann(CMakePackage):
         # Fix the install location so that spack activate works
         if '+python' in self.spec:
             filter_file("share/flann/python",
-                        site_packages_dir,
+                        python_platlib,
                         "src/python/CMakeLists.txt")
         # Hack. Don't install setup.py
         filter_file("install( FILES",
diff --git a/var/spack/repos/builtin/packages/gdl/package.py b/var/spack/repos/builtin/packages/gdl/package.py
index 02c10dbdd8..8f169b30f5 100644
--- a/var/spack/repos/builtin/packages/gdl/package.py
+++ b/var/spack/repos/builtin/packages/gdl/package.py
@@ -144,7 +144,7 @@ class Gdl(CMakePackage):
             src = os.path.join(
                 self.spec.prefix.lib,
                 'site-python')
-            dst = site_packages_dir
+            dst = python_platlib
             if os.path.isdir(src):
                 if not os.path.isdir(dst):
                     mkdirp(dst)
diff --git a/var/spack/repos/builtin/packages/hepmc3/package.py b/var/spack/repos/builtin/packages/hepmc3/package.py
index 34cfc7be33..875372f71d 100644
--- a/var/spack/repos/builtin/packages/hepmc3/package.py
+++ b/var/spack/repos/builtin/packages/hepmc3/package.py
@@ -51,11 +51,10 @@ class Hepmc3(CMakePackage):
 
         if self.spec.satisfies('+python'):
             py_ver = spec['python'].version.up_to(2)
-            py_sitepkg = join_path(self.prefix, site_packages_dir)
             args.extend([
                 '-DHEPMC3_PYTHON_VERSIONS={0}'.format(py_ver),
                 '-DHEPMC3_Python_SITEARCH{0}={1}'.format(
-                    py_ver.joined, py_sitepkg)
+                    py_ver.joined, python_platlib)
             ])
 
         if self.spec.satisfies('+rootio'):
diff --git a/var/spack/repos/builtin/packages/hoomd-blue/package.py b/var/spack/repos/builtin/packages/hoomd-blue/package.py
index 5a44ea6647..b0153ad817 100644
--- a/var/spack/repos/builtin/packages/hoomd-blue/package.py
+++ b/var/spack/repos/builtin/packages/hoomd-blue/package.py
@@ -60,12 +60,10 @@ class HoomdBlue(CMakePackage):
 
     def cmake_args(self):
         spec = self.spec
-        install_dir = spec['python'].package.site_packages_dir
-        install_path = os.path.join(spec.prefix, install_dir)
 
         cmake_args = [
             '-DPYTHON_EXECUTABLE={0}'.format(spec['python'].command.path),
-            '-DCMAKE_INSTALL_PREFIX={0}'.format(install_path)
+            '-DCMAKE_INSTALL_PREFIX={0}'.format(python_platlib)
         ]
 
         # MPI support
diff --git a/var/spack/repos/builtin/packages/libxml2/package.py b/var/spack/repos/builtin/packages/libxml2/package.py
index 994f2e3db5..58614fda62 100644
--- a/var/spack/repos/builtin/packages/libxml2/package.py
+++ b/var/spack/repos/builtin/packages/libxml2/package.py
@@ -62,7 +62,7 @@ class Libxml2(AutotoolsPackage):
         if '+python' in spec:
             args.extend([
                 '--with-python={0}'.format(spec['python'].home),
-                '--with-python-install-dir={0}'.format(site_packages_dir)
+                '--with-python-install-dir={0}'.format(python_platlib)
             ])
         else:
             args.append('--without-python')
diff --git a/var/spack/repos/builtin/packages/llvm-doe/package.py b/var/spack/repos/builtin/packages/llvm-doe/package.py
index e2d1284116..07ba3bb42c 100644
--- a/var/spack/repos/builtin/packages/llvm-doe/package.py
+++ b/var/spack/repos/builtin/packages/llvm-doe/package.py
@@ -639,10 +639,10 @@ class LlvmDoe(CMakePackage, CudaPackage):
                 ninja()
                 ninja("install")
         if "+python" in self.spec:
-            install_tree("llvm/bindings/python", site_packages_dir)
+            install_tree("llvm/bindings/python", python_platlib)
 
             if "+clang" in self.spec:
-                install_tree("clang/bindings/python", site_packages_dir)
+                install_tree("clang/bindings/python", python_platlib)
 
         with working_dir(self.build_directory):
             install_tree("bin", join_path(self.prefix, "libexec", "llvm"))
diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py
index 1c2255b677..4e86a8871a 100644
--- a/var/spack/repos/builtin/packages/llvm/package.py
+++ b/var/spack/repos/builtin/packages/llvm/package.py
@@ -702,10 +702,10 @@ class Llvm(CMakePackage, CudaPackage):
                 ninja()
                 ninja("install")
         if "+python" in self.spec:
-            install_tree("llvm/bindings/python", site_packages_dir)
+            install_tree("llvm/bindings/python", python_platlib)
 
             if "+clang" in self.spec:
-                install_tree("clang/bindings/python", site_packages_dir)
+                install_tree("clang/bindings/python", python_platlib)
 
         with working_dir(self.build_directory):
             install_tree("bin", join_path(self.prefix, "libexec", "llvm"))
diff --git a/var/spack/repos/builtin/packages/lvarray/package.py b/var/spack/repos/builtin/packages/lvarray/package.py
index 9b8b3188d3..0168132c6b 100644
--- a/var/spack/repos/builtin/packages/lvarray/package.py
+++ b/var/spack/repos/builtin/packages/lvarray/package.py
@@ -131,8 +131,8 @@ class Lvarray(CMakePackage, CudaPackage):
           subclasses this package provide a specific site packages
           dir when calling this function. `py_site_pkgs_dir` should
           be an absolute path or `None`.
-          This is necessary because the spack `site_packages_dir`
-          var will not exist in the base class. For more details
+          This is necessary because the spack `python_purelib` and `python_platlib`
+          vars will not exist in the base class. For more details
           on this issue see: https://github.com/spack/spack/issues/6261
         """
 
diff --git a/var/spack/repos/builtin/packages/migraphx/package.py b/var/spack/repos/builtin/packages/migraphx/package.py
index 5bdc8dd3ed..1a7ddecd75 100644
--- a/var/spack/repos/builtin/packages/migraphx/package.py
+++ b/var/spack/repos/builtin/packages/migraphx/package.py
@@ -64,11 +64,9 @@ class Migraphx(CMakePackage):
         """Include the python include path to the
         CMake based on current spec
         """
-        python_spec = self.spec['python']
-        include_dir = join_path(
-            python_spec.prefix, python_spec.package.config_vars['python_inc']['false'])
+        python = self.spec['python']
         return [
-            self.define('Python_INCLUDE_DIR', include_dir)
+            self.define('Python_INCLUDE_DIR', python.package.config_vars['include'])
         ]
 
     def cmake_args(self):
diff --git a/var/spack/repos/builtin/packages/openturns/package.py b/var/spack/repos/builtin/packages/openturns/package.py
index 7ab5405052..428a7033aa 100644
--- a/var/spack/repos/builtin/packages/openturns/package.py
+++ b/var/spack/repos/builtin/packages/openturns/package.py
@@ -53,7 +53,7 @@ class Openturns(CMakePackage):
                 # By default picks up the system python not the Spack build
                 '-DPYTHON_EXECUTABLE={0}'.format(spec['python'].command.path),
                 # By default installs to the python prefix
-                '-DPYTHON_SITE_PACKAGES={0}'.format(site_packages_dir),
+                '-DPYTHON_SITE_PACKAGES={0}'.format(python_platlib),
             ])
 
         return args
diff --git a/var/spack/repos/builtin/packages/openvdb/package.py b/var/spack/repos/builtin/packages/openvdb/package.py
index f42efd4ed6..5a5cce2722 100644
--- a/var/spack/repos/builtin/packages/openvdb/package.py
+++ b/var/spack/repos/builtin/packages/openvdb/package.py
@@ -70,7 +70,7 @@ class Openvdb(CMakePackage):
 
     # The python extension is being put in the wrong directory
     # by OpenVDB's cmake, instead it needs to be in
-    # site_packages_dir. For RHEL systems we seem to get the
+    # python_platlib. For RHEL systems we seem to get the
     # dso in lib64/ instead of lib/
     @run_after('install')
     def post_install(self):
@@ -87,5 +87,4 @@ class Openvdb(CMakePackage):
             src = prefix.lib.join(pyver).join(pyso)
             if not os.path.isfile(src):
                 src = prefix.lib64.join(pyver).join(pyso)
-            mkdirp(site_packages_dir)
-            os.rename(src, os.path.join(site_packages_dir, pyso))
+            os.rename(src, os.path.join(python_platlib, pyso))
diff --git a/var/spack/repos/builtin/packages/pagmo/package.py b/var/spack/repos/builtin/packages/pagmo/package.py
index 6ca785d7c5..1ed0a6ed9f 100644
--- a/var/spack/repos/builtin/packages/pagmo/package.py
+++ b/var/spack/repos/builtin/packages/pagmo/package.py
@@ -93,7 +93,7 @@ class Pagmo(CMakePackage):
                 # By default picks up the system python not the Spack build
                 '-DPYTHON_EXECUTABLE={0}'.format(spec['python'].command.path),
                 # By default installs to the python prefix not the pagmo prefix
-                '-DPYTHON_MODULES_DIR={0}'.format(site_packages_dir),
+                '-DPYTHON_MODULES_DIR={0}'.format(python_platlib),
             ])
 
         return args
diff --git a/var/spack/repos/builtin/packages/precice/package.py b/var/spack/repos/builtin/packages/precice/package.py
index 5f73c57531..bffe570c1e 100644
--- a/var/spack/repos/builtin/packages/precice/package.py
+++ b/var/spack/repos/builtin/packages/precice/package.py
@@ -143,7 +143,7 @@ class Precice(CMakePackage):
             python_include = spec['python'].headers.directories[0]
             numpy_include = join_path(
                 spec['py-numpy'].prefix,
-                spec['python'].package.site_packages_dir,
+                spec['python'].package.platlib,
                 'numpy', 'core', 'include')
             if xsdk_mode:
                 cmake_args.append('-DTPL_ENABLE_PYTHON:BOOL=ON')
diff --git a/var/spack/repos/builtin/packages/py-flit-core/package.py b/var/spack/repos/builtin/packages/py-flit-core/package.py
index bd4c94e3a1..a4463b95c1 100644
--- a/var/spack/repos/builtin/packages/py-flit-core/package.py
+++ b/var/spack/repos/builtin/packages/py-flit-core/package.py
@@ -29,4 +29,4 @@ class PyFlitCore(PythonPackage):
     def install(self, spec, prefix):
         wheel = glob.glob(os.path.join('flit_core', 'dist', '*.whl'))[0]
         with zipfile.ZipFile(wheel) as f:
-            f.extractall(site_packages_dir)
+            f.extractall(python_purelib)
diff --git a/var/spack/repos/builtin/packages/py-fury/package.py b/var/spack/repos/builtin/packages/py-fury/package.py
index 693da3e01c..5ec637c1f7 100644
--- a/var/spack/repos/builtin/packages/py-fury/package.py
+++ b/var/spack/repos/builtin/packages/py-fury/package.py
@@ -31,7 +31,7 @@ class PyFury(PythonPackage):
     def install_test(self):
         with working_dir('spack-test', create=True):
             pytest = which('pytest')
-            pytest(join_path(self.prefix, site_packages_dir, 'fury'),
+            pytest(join_path(python_purelib, 'fury'),
                    # 'Some warning' is not propagated to __warningregistry__ so
                    # that the test fails, disable it for now
                    # running all tests manually after the package is installed
diff --git a/var/spack/repos/builtin/packages/py-gpaw/package.py b/var/spack/repos/builtin/packages/py-gpaw/package.py
index a631d97d3d..3b0eff0433 100644
--- a/var/spack/repos/builtin/packages/py-gpaw/package.py
+++ b/var/spack/repos/builtin/packages/py-gpaw/package.py
@@ -58,7 +58,7 @@ class PyGpaw(PythonPackage):
         python_include = spec['python'].headers.directories[0]
         numpy_include = join_path(
             spec['py-numpy'].prefix,
-            spec['python'].package.site_packages_dir,
+            spec['python'].package.platlib,
             'numpy', 'core', 'include')
 
         libs = blas.libs + lapack.libs + libxc.libs
diff --git a/var/spack/repos/builtin/packages/py-ninja/package.py b/var/spack/repos/builtin/packages/py-ninja/package.py
index 944163676f..a1b6125f56 100644
--- a/var/spack/repos/builtin/packages/py-ninja/package.py
+++ b/var/spack/repos/builtin/packages/py-ninja/package.py
@@ -27,7 +27,7 @@ class PyNinja(PythonPackage):
                                    'ninja_syntax.py')
         bin_file = os.path.join(self.spec['ninja'].prefix.bin,
                                 'ninja')
-        dst = os.path.join(site_packages_dir,
+        dst = os.path.join(python_platlib,
                            'ninja')
         dstbin = os.path.join(dst, 'data', 'bin')
         mkdirp(dstbin)
diff --git a/var/spack/repos/builtin/packages/py-protobuf/package.py b/var/spack/repos/builtin/packages/py-protobuf/package.py
index bacad6f018..83ba3fd69c 100644
--- a/var/spack/repos/builtin/packages/py-protobuf/package.py
+++ b/var/spack/repos/builtin/packages/py-protobuf/package.py
@@ -86,6 +86,4 @@ class PyProtobuf(PythonPackage):
     def fix_import_error(self):
         if str(self.spec['python'].version.up_to(1)) == '2':
             touch = which('touch')
-            touch(self.prefix + '/' +
-                  self.spec['python'].package.site_packages_dir +
-                  '/google/__init__.py')
+            touch(join_path(python_platlib, 'google', '__init__.py'))
diff --git a/var/spack/repos/builtin/packages/py-pymol/package.py b/var/spack/repos/builtin/packages/py-pymol/package.py
index 66f28e8e5c..3e3ab85a26 100644
--- a/var/spack/repos/builtin/packages/py-pymol/package.py
+++ b/var/spack/repos/builtin/packages/py-pymol/package.py
@@ -48,10 +48,7 @@ class PyPymol(PythonPackage):
         binpath = self.prefix.bin
         mkdirp(self.prefix.bin)
         fname = join_path(binpath, 'pymol')
-        script = join_path(self.prefix,
-                           self.spec['python'].package.site_packages_dir,
-                           'pymol',
-                           '__init__.py')
+        script = join_path(python_platlib, 'pymol', '__init__.py')
 
         shebang = '#!/bin/sh\n'
         fdata = 'exec {0} {1} \"$@\"'.format(self.spec['python'].command,
diff --git a/var/spack/repos/builtin/packages/py-pyqt4/package.py b/var/spack/repos/builtin/packages/py-pyqt4/package.py
index 73a72efcb8..e9b0455c4c 100644
--- a/var/spack/repos/builtin/packages/py-pyqt4/package.py
+++ b/var/spack/repos/builtin/packages/py-pyqt4/package.py
@@ -36,7 +36,7 @@ class PyPyqt4(SIPPackage):
         args = [
             '--pyuic4-interpreter', self.spec['python'].command.path,
             '--sipdir', self.prefix.share.sip.PyQt4,
-            '--stubsdir', join_path(site_packages_dir, 'PyQt4')
+            '--stubsdir', join_path(python_platlib, 'PyQt4')
         ]
         if '+qsci_api' in self.spec:
             args.extend(['--qsci-api',
diff --git a/var/spack/repos/builtin/packages/py-pyqt5/package.py b/var/spack/repos/builtin/packages/py-pyqt5/package.py
index 5556e76576..f550dcb2cc 100644
--- a/var/spack/repos/builtin/packages/py-pyqt5/package.py
+++ b/var/spack/repos/builtin/packages/py-pyqt5/package.py
@@ -39,10 +39,7 @@ class PyPyqt5(SIPPackage):
             '--sipdir', self.prefix.share.sip.PyQt5,
             '--designer-plugindir', self.prefix.plugins.designer,
             '--qml-plugindir', self.prefix.plugins.PyQt5,
-            '--stubsdir', join_path(
-                self.prefix,
-                self.spec['python'].package.site_packages_dir,
-                'PyQt5'),
+            '--stubsdir', join_path(python_platlib, 'PyQt5'),
         ]
         if '+qsci_api' in self.spec:
             args.extend(['--qsci-api',
diff --git a/var/spack/repos/builtin/packages/py-pyside/package.py b/var/spack/repos/builtin/packages/py-pyside/package.py
index 9e14889cf5..681f85e296 100644
--- a/var/spack/repos/builtin/packages/py-pyside/package.py
+++ b/var/spack/repos/builtin/packages/py-pyside/package.py
@@ -38,10 +38,8 @@ class PyPyside(PythonPackage):
     def patch(self):
         """Undo PySide RPATH handling and add Spack RPATH."""
         # Figure out the special RPATH
-        pypkg = self.spec['python'].package
         rpath = self.rpath
-        rpath.append(os.path.join(
-            self.prefix, pypkg.site_packages_dir, 'PySide'))
+        rpath.append(os.path.join(python_platlib, 'PySide'))
 
         # Fix subprocess.mswindows check for Python 3.5
         # https://github.com/pyside/pyside-setup/pull/55
diff --git a/var/spack/repos/builtin/packages/py-python-meep/package.py b/var/spack/repos/builtin/packages/py-python-meep/package.py
index 3f3edb8616..fcc0d23c5b 100644
--- a/var/spack/repos/builtin/packages/py-python-meep/package.py
+++ b/var/spack/repos/builtin/packages/py-python-meep/package.py
@@ -41,7 +41,7 @@ class PyPythonMeep(PythonPackage):
             spec['meep'].prefix.include,
             os.path.join(
                 spec['py-numpy'].prefix,
-                spec['python'].package.python_include_dir
+                spec['python'].package.include
             )
         ]
 
diff --git a/var/spack/repos/builtin/packages/py-ruamel-yaml/package.py b/var/spack/repos/builtin/packages/py-ruamel-yaml/package.py
index 640f749ff9..0a657f276e 100644
--- a/var/spack/repos/builtin/packages/py-ruamel-yaml/package.py
+++ b/var/spack/repos/builtin/packages/py-ruamel-yaml/package.py
@@ -31,6 +31,4 @@ class PyRuamelYaml(PythonPackage):
     def fix_import_error(self):
         if str(self.spec['python'].version.up_to(1)) == '2':
             touch = which('touch')
-            touch(self.prefix + '/' +
-                  self.spec['python'].package.site_packages_dir +
-                  '/ruamel/__init__.py')
+            touch(join_path(python_purelib, 'ruamel', '__init__.py'))
diff --git a/var/spack/repos/builtin/packages/py-scikit-learn/package.py b/var/spack/repos/builtin/packages/py-scikit-learn/package.py
index d5ff6ee3db..7defbc7df8 100644
--- a/var/spack/repos/builtin/packages/py-scikit-learn/package.py
+++ b/var/spack/repos/builtin/packages/py-scikit-learn/package.py
@@ -100,4 +100,4 @@ class PyScikitLearn(PythonPackage):
         # https://scikit-learn.org/stable/developers/advanced_installation.html#testing
         with working_dir('spack-test', create=True):
             pytest = which('pytest')
-            pytest(join_path(self.prefix, site_packages_dir, 'sklearn'))
+            pytest(join_path(self.prefix, python_purelib, 'sklearn'))
diff --git a/var/spack/repos/builtin/packages/py-shiboken/package.py b/var/spack/repos/builtin/packages/py-shiboken/package.py
index 5395682533..a9636321d3 100644
--- a/var/spack/repos/builtin/packages/py-shiboken/package.py
+++ b/var/spack/repos/builtin/packages/py-shiboken/package.py
@@ -26,10 +26,8 @@ class PyShiboken(PythonPackage):
         """Undo Shiboken RPATH handling and add Spack RPATH."""
         # Add Spack's standard CMake args to the sub-builds.
         # They're called BY setup.py so we have to patch it.
-        pypkg = self.spec['python'].package
         rpath = self.rpath
-        rpath.append(os.path.join(
-            self.prefix, pypkg.site_packages_dir, 'Shiboken'))
+        rpath.append(os.path.join(python_platlib, 'Shiboken'))
 
         filter_file(
             r'OPTION_CMAKE,',
diff --git a/var/spack/repos/builtin/packages/py-sip/package.py b/var/spack/repos/builtin/packages/py-sip/package.py
index 768f2fb989..981959150f 100644
--- a/var/spack/repos/builtin/packages/py-sip/package.py
+++ b/var/spack/repos/builtin/packages/py-sip/package.py
@@ -52,10 +52,10 @@ class PySip(PythonPackage):
             args = [
                 '--sip-module={0}'.format(spec.variants['module'].value),
                 '--bindir={0}'.format(prefix.bin),
-                '--destdir={0}'.format(site_packages_dir),
-                '--incdir={0}'.format(python_include_dir),
+                '--destdir={0}'.format(python_platlib),
+                '--incdir={0}'.format(spec['python'].package.include),
                 '--sipdir={0}'.format(prefix.share.sip),
-                '--stubsdir={0}'.format(site_packages_dir),
+                '--stubsdir={0}'.format(python_platlib),
             ]
 
             python('configure.py', *args)
@@ -75,7 +75,7 @@ class PySip(PythonPackage):
             module = self.spec.variants['module'].value
             if module != 'sip':
                 module = module.split('.')[0]
-                with working_dir(site_packages_dir):
+                with working_dir(python_platlib):
                     with open(os.path.join(module, '__init__.py'), 'w') as f:
                         f.write('from pkgutil import extend_path\n')
                         f.write('__path__ = extend_path(__path__, __name__)\n')
diff --git a/var/spack/repos/builtin/packages/py-tensorflow/package.py b/var/spack/repos/builtin/packages/py-tensorflow/package.py
index 251db91b9a..0544eddc85 100644
--- a/var/spack/repos/builtin/packages/py-tensorflow/package.py
+++ b/var/spack/repos/builtin/packages/py-tensorflow/package.py
@@ -310,7 +310,7 @@ class PyTensorflow(Package, CudaPackage):
         env.set('PYTHON_BIN_PATH', spec['python'].command.path)
 
         # Please input the desired Python library path to use
-        env.set('PYTHON_LIB_PATH', site_packages_dir)
+        env.set('PYTHON_LIB_PATH', python_platlib)
 
         # Ensure swig is in PATH or set SWIG_PATH
         env.set('SWIG_PATH', spec['swig'].prefix.bin.swig)
diff --git a/var/spack/repos/builtin/packages/py-torch/package.py b/var/spack/repos/builtin/packages/py-torch/package.py
index 3b667deafb..1563310bb2 100644
--- a/var/spack/repos/builtin/packages/py-torch/package.py
+++ b/var/spack/repos/builtin/packages/py-torch/package.py
@@ -207,14 +207,12 @@ class PyTorch(PythonPackage, CudaPackage):
 
     @property
     def libs(self):
-        root = join_path(self.prefix, self.spec['python'].package.site_packages_dir,
-                         'torch', 'lib')
+        root = join_path(python_platlib, 'torch', 'lib')
         return find_libraries('libtorch', root)
 
     @property
     def headers(self):
-        root = join_path(self.prefix, self.spec['python'].package.site_packages_dir,
-                         'torch', 'include')
+        root = join_path(python_platlib, 'torch', 'include')
         headers = find_all_headers(root)
         headers.directories = [root]
         return headers
diff --git a/var/spack/repos/builtin/packages/py-xdot/package.py b/var/spack/repos/builtin/packages/py-xdot/package.py
index 12d448eeea..cf26c9f386 100644
--- a/var/spack/repos/builtin/packages/py-xdot/package.py
+++ b/var/spack/repos/builtin/packages/py-xdot/package.py
@@ -40,8 +40,7 @@ class PyXdot(PythonPackage):
             join_path(spec['atk'].prefix.lib, 'girepository-1.0'),
             join_path(spec['gdk-pixbuf'].prefix.lib, 'girepository-1.0'),
             join_path(spec['gtkplus'].prefix.lib, 'girepository-1.0'))
-        dst = join_path(spec.prefix, spec['python'].package.site_packages_dir,
-                        'xdot/__init__.py')
+        dst = join_path(python_platlib, 'xdot', '__init__.py')
         filter_file("import sys",
                     "import sys\nimport os\nos.environ['GI_TYPELIB_PATH']" +
                     " = '%s'" % repo_paths, dst)
diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
index ee4d00df5f..c8146e48a4 100644
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -15,7 +15,6 @@ from llnl.util.lang import match_predicate
 
 from spack import *
 from spack.build_environment import dso_suffix
-from spack.compilers import NoCompilerForSpecError
 from spack.util.environment import is_system_path
 from spack.util.prefix import Prefix
 
@@ -716,40 +715,29 @@ class Python(AutotoolsPackage):
     def config_vars(self):
         """Return a set of variable definitions associated with a Python installation.
 
-        Wrapper around various ``distutils.sysconfig`` functions.
+        Wrapper around various ``sysconfig`` functions. To see these variables on the
+        command line, run:
+
+        .. code-block:: console
+
+           $ python -m sysconfig
 
         Returns:
             dict: variable definitions
         """
-        # TODO: distutils is deprecated in Python 3.10 and will be removed in
-        # Python 3.12, find a different way to access this information.
-        # Also, calling the python executable disallows us from cross-compiling,
-        # so we want to try to avoid that if possible.
         cmd = """
 import json
-from distutils.sysconfig import (
+from sysconfig import (
     get_config_vars,
     get_config_h_filename,
     get_makefile_filename,
-    get_python_inc,
-    get_python_lib,
+    get_paths,
 )
 
 config = get_config_vars()
 config['config_h_filename'] = get_config_h_filename()
 config['makefile_filename'] = get_makefile_filename()
-config['python_inc'] = {}
-config['python_lib'] = {}
-
-for plat_specific in [True, False]:
-    plat_key = str(plat_specific).lower()
-    config['python_inc'][plat_key] = get_python_inc(plat_specific, prefix='')
-    config['python_lib'][plat_key] = {}
-    for standard_lib in [True, False]:
-        lib_key = str(standard_lib).lower()
-        config['python_lib'][plat_key][lib_key] = get_python_lib(
-            plat_specific, standard_lib, prefix=''
-        )
+config.update(get_paths())
 
 %s
 """ % self.print_string("json.dumps(config)")
@@ -759,16 +747,10 @@ for plat_specific in [True, False]:
         if dag_hash not in self._config_vars:
             # Default config vars
             version = self.version.up_to(2)
-            try:
-                cc = self.compiler.cc
-                cxx = self.compiler.cxx
-            except (TypeError, NoCompilerForSpecError):
-                cc = 'cc'
-                cxx = 'c++'
-
             config = {
-                'CC': cc,
-                'CXX': cxx,
+                # get_config_vars
+                'CC': 'cc',
+                'CXX': 'c++',
                 'INCLUDEPY': self.prefix.include.join('python{}').format(version),
                 'LIBDEST': self.prefix.lib.join('python{}').format(version),
                 'LIBDIR': self.prefix.lib,
@@ -776,34 +758,31 @@ for plat_specific in [True, False]:
                     'config-{0}-{1}').format(version, sys.platform),
                 'LDLIBRARY': 'libpython{}.{}'.format(version, dso_suffix),
                 'LIBRARY': 'libpython{}.a'.format(version),
-                'LDSHARED': cc,
-                'LDCXXSHARED': cxx,
+                'LDSHARED': 'cc',
+                'LDCXXSHARED': 'c++',
                 'PYTHONFRAMEWORKPREFIX': '/System/Library/Frameworks',
+                'base': self.prefix,
+                'installed_base': self.prefix,
+                'installed_platbase': self.prefix,
+                'platbase': self.prefix,
                 'prefix': self.prefix,
+                # get_config_h_filename
                 'config_h_filename': self.prefix.include.join('python{}').join(
                     'pyconfig.h').format(version),
+                # get_makefile_filename
                 'makefile_filename': self.prefix.lib.join('python{0}').join(
                     'config-{0}-{1}').Makefile.format(version, sys.platform),
-                'python_inc': {
-                    # plat_specific
-                    'true': os.path.join('include64', 'python{}'.format(version)),
-                    'false': os.path.join('include', 'python{}'.format(version)),
-                },
-                'python_lib': {
-                    # plat_specific
-                    'true': {
-                        # standard_lib
-                        'true': os.path.join('lib64', 'python{}'.format(version)),
-                        'false': os.path.join(
-                            'lib64', 'python{}'.format(version), 'site-packages'),
-                    },
-                    'false': {
-                        # standard_lib
-                        'true': os.path.join('lib', 'python{}'.format(version)),
-                        'false': os.path.join(
-                            'lib', 'python{}'.format(version), 'site-packages'),
-                    },
-                },
+                # get_paths
+                'data': self.prefix,
+                'include': self.prefix.include.join('python{}'.format(version)),
+                'platinclude': self.prefix.include64.join('python{}'.format(version)),
+                'platlib': self.prefix.lib64.join(
+                    'python{}'.format(version)).join('site-packages'),
+                'platstdlib': self.prefix.lib64.join('python{}'.format(version)),
+                'purelib': self.prefix.lib.join(
+                    'python{}'.format(version)).join('site-packages'),
+                'scripts': self.prefix.bin,
+                'stdlib': self.prefix.lib.join('python{}'.format(version)),
             }
 
             try:
@@ -906,66 +885,68 @@ for plat_specific in [True, False]:
         headers.directories = [os.path.dirname(config_h)]
         return headers
 
-    @property
-    def python_include_dir(self):
-        """Directory for the include files.
+    # https://docs.python.org/3/library/sysconfig.html#installation-paths
 
-        On most systems, and for Spack-installed Python, this will look like:
+    @property
+    def platlib(self):
+        """Directory for site-specific, platform-specific files.
 
-            ``include/pythonX.Y``
+        Exact directory depends on platform/OS/Python version. Examples include:
 
-        However, some systems append a ``m`` to the end of this path.
+        * ``lib/pythonX.Y/site-packages`` on most POSIX systems
+        * ``lib64/pythonX.Y/site-packages`` on RHEL/CentOS/Fedora with system Python
+        * ``lib/pythonX/dist-packages`` on Debian/Ubuntu with system Python
+        * ``lib/python/site-packages`` on macOS with framework Python
+        * ``Lib/site-packages`` on Windows
 
         Returns:
-            str: include files directory
+            str: platform-specific site-packages directory
         """
-        return self.config_vars['python_inc']['false']
+        return self.config_vars['platlib'].replace(
+            self.config_vars['platbase'] + os.sep, ''
+        )
 
     @property
-    def python_lib_dir(self):
-        """Directory for the standard library.
-
-        On most systems, and for Spack-installed Python, this will look like:
-
-            ``lib/pythonX.Y``
+    def purelib(self):
+        """Directory for site-specific, non-platform-specific files.
 
-        On RHEL/CentOS/Fedora, when using the system Python, this will look like:
+        Exact directory depends on platform/OS/Python version. Examples include:
 
-            ``lib64/pythonX.Y``
-
-        On Debian/Ubuntu, when using the system Python, this will look like:
-
-            ``lib/pythonX``
+        * ``lib/pythonX.Y/site-packages`` on most POSIX systems
+        * ``lib/pythonX/dist-packages`` on Debian/Ubuntu with system Python
+        * ``lib/python/site-packages`` on macOS with framework Python
+        * ``Lib/site-packages`` on Windows
 
         Returns:
-            str: standard library directory
+            str: platform-independent site-packages directory
         """
-        return self.config_vars['python_lib']['false']['true']
+        return self.config_vars['purelib'].replace(
+            self.config_vars['base'] + os.sep, ''
+        )
 
     @property
-    def site_packages_dir(self):
-        """Directory where third-party extensions should be installed.
-
-        On most systems, and for Spack-installed Python, this will look like:
-
-            ``lib/pythonX.Y/site-packages``
+    def include(self):
+        """Directory for non-platform-specific header files.
 
-        On RHEL/CentOS/Fedora, when using the system Python, this will look like:
+        Exact directory depends on platform/Python version/ABI flags. Examples include:
 
-            ``lib64/pythonX.Y/site-packages``
-
-        On Debian/Ubuntu, when using the system Python, this will look like:
-
-            ``lib/pythonX/dist-packages``
+        * ``include/pythonX.Y`` on most POSIX systems
+        * ``include/pythonX.Yd`` for debug builds
+        * ``include/pythonX.Ym`` for malloc builds
+        * ``include/pythonX.Yu`` for wide unicode builds
+        * ``include`` on macOS with framework Python
+        * ``Include`` on Windows
 
         Returns:
-            str: site-packages directory
+            str: platform-independent header file directory
         """
-        return self.config_vars['python_lib']['false']['false']
+        return self.config_vars['include'].replace(
+            self.config_vars['installed_base'] + os.sep, ''
+        )
 
     @property
     def easy_install_file(self):
-        return join_path(self.site_packages_dir, "easy-install.pth")
+        return join_path(self.purelib, "easy-install.pth")
 
     def setup_run_environment(self, env):
         env.prepend_path('CPATH', os.pathsep.join(
@@ -985,10 +966,28 @@ for plat_specific in [True, False]:
         if not is_system_path(path):
             env.prepend_path('PATH', path)
 
-        for d in dependent_spec.traverse(deptype=('build', 'run', 'test'), root=True):
-            if d.package.extends(self.spec):
-                env.prepend_path('PYTHONPATH', join_path(
-                    d.prefix, self.site_packages_dir))
+        # Add installation prefix to PYTHONPATH, needed to run import tests
+        prefixes = set()
+        if dependent_spec.package.extends(self.spec):
+            prefixes.add(dependent_spec.prefix)
+
+        # Add direct build/run/test dependencies to PYTHONPATH,
+        # needed to build the package and to run import tests
+        for direct_dep in dependent_spec.dependencies(deptype=('build', 'run', 'test')):
+            if direct_dep.package.extends(self.spec):
+                prefixes.add(direct_dep.prefix)
+
+                # Add recursive run dependencies of all direct dependencies,
+                # needed by direct dependencies at run-time
+                for indirect_dep in direct_dep.traverse(deptype='run'):
+                    if indirect_dep.package.extends(self.spec):
+                        prefixes.add(indirect_dep.prefix)
+
+        for prefix in prefixes:
+            # Packages may be installed in platform-specific or platform-independent
+            # site-packages directories
+            for directory in {self.platlib, self.purelib}:
+                env.prepend_path('PYTHONPATH', os.path.join(prefix, directory))
 
         # We need to make sure that the extensions are compiled and linked with
         # the Spack wrapper. Paths to the executables that are used for these
@@ -1051,8 +1050,12 @@ for plat_specific in [True, False]:
         """
         for d in dependent_spec.traverse(deptype=('run'), root=True):
             if d.package.extends(self.spec):
-                env.prepend_path('PYTHONPATH', join_path(
-                    d.prefix, self.site_packages_dir))
+                # Packages may be installed in platform-specific or platform-independent
+                # site-packages directories
+                for directory in {self.platlib, self.purelib}:
+                    env.prepend_path(
+                        'PYTHONPATH', os.path.join(d.prefix, directory)
+                    )
 
     def setup_dependent_package(self, module, dependent_spec):
         """Called before python modules' install() methods."""
@@ -1061,19 +1064,15 @@ for plat_specific in [True, False]:
         module.setup_py = Executable(
             self.command.path + ' setup.py --no-user-cfg')
 
-        # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs.
-        module.python_lib_dir = join_path(dependent_spec.prefix,
-                                          self.python_lib_dir)
-        module.python_include_dir = join_path(dependent_spec.prefix,
-                                              self.python_include_dir)
-        module.site_packages_dir = join_path(dependent_spec.prefix,
-                                             self.site_packages_dir)
+        module.python_platlib = join_path(dependent_spec.prefix, self.platlib)
+        module.python_purelib = join_path(dependent_spec.prefix, self.purelib)
 
         self.spec.home = self.home
 
         # Make the site packages directory for extensions
         if dependent_spec.package.is_extension:
-            mkdirp(module.site_packages_dir)
+            mkdirp(module.python_platlib)
+            mkdirp(module.python_purelib)
 
     # ========================================================================
     # Handle specifics of activating and deactivating python modules.
diff --git a/var/spack/repos/builtin/packages/qscintilla/package.py b/var/spack/repos/builtin/packages/qscintilla/package.py
index 5c1cd35766..57e954b168 100644
--- a/var/spack/repos/builtin/packages/qscintilla/package.py
+++ b/var/spack/repos/builtin/packages/qscintilla/package.py
@@ -87,10 +87,7 @@ class Qscintilla(QMakePackage):
                 pyqtx = 'PyQt5'
 
             with working_dir(join_path(self.stage.source_path, 'Python')):
-                pydir = join_path(
-                    self.prefix,
-                    self.spec['python'].package.site_packages_dir,
-                    pyqtx)
+                pydir = join_path(python_platlib, pyqtx)
                 mkdirp(os.path.join(self.prefix.share.sip, pyqtx))
                 python = self.spec['python'].command
                 python('configure.py', '--pyqt=' + pyqtx,
@@ -144,7 +141,7 @@ class Qscintilla(QMakePackage):
         module = self.spec['py-sip'].variants['module'].value
         if module != 'sip':
             module = module.split('.')[0]
-            with working_dir(site_packages_dir):
+            with working_dir(python_platlib):
                 with open(os.path.join(module, '__init__.py'), 'w') as f:
                     f.write('from pkgutil import extend_path\n')
                     f.write('__path__ = extend_path(__path__, __name__)\n')
diff --git a/var/spack/repos/builtin/packages/sollve/package.py b/var/spack/repos/builtin/packages/sollve/package.py
index 8acfa7fb9a..073c3ebb0d 100644
--- a/var/spack/repos/builtin/packages/sollve/package.py
+++ b/var/spack/repos/builtin/packages/sollve/package.py
@@ -306,7 +306,7 @@ class Sollve(CMakePackage):
         if '+clang' in self.spec and '+python' in self.spec:
             install_tree(
                 'tools/clang/bindings/python/clang',
-                join_path(site_packages_dir, 'clang'))
+                join_path(python_platlib, 'clang'))
 
         with working_dir(self.build_directory):
             install_tree('bin', self.prefix.libexec.llvm)
diff --git a/var/spack/repos/builtin/packages/sz/package.py b/var/spack/repos/builtin/packages/sz/package.py
index ca1ced87ee..06fd1ebacd 100644
--- a/var/spack/repos/builtin/packages/sz/package.py
+++ b/var/spack/repos/builtin/packages/sz/package.py
@@ -93,7 +93,7 @@ class Sz(CMakePackage):
 
         if "+python" in self.spec:
             args.append("-DBUILD_PYTHON_WRAPPER=ON")
-            args.append("-DSZ_PYTHON_SITELIB={0}".format(site_packages_dir))
+            args.append("-DSZ_PYTHON_SITELIB={0}".format(python_platlib))
         else:
             args.append("-DBUILD_PYTHON_WRAPPER=OFF")
 
diff --git a/var/spack/repos/builtin/packages/vapor/package.py b/var/spack/repos/builtin/packages/vapor/package.py
index 08e3937d4d..2feee38c67 100644
--- a/var/spack/repos/builtin/packages/vapor/package.py
+++ b/var/spack/repos/builtin/packages/vapor/package.py
@@ -42,12 +42,12 @@ class Vapor(CMakePackage):
             python = self.spec['python']
             f.write('set (PYTHONVERSION {0})\n'.format(python.version.up_to(2)))
             f.write('set (PYTHONDIR {0})\n'.format(python.home))
-            f.write('set (PYTHONPATH {0})\n'.format(python.package.site_packages_dir))
+            f.write('set (PYTHONPATH {0})\n'.format(python.package.platlib))
             # install expects the share/images directory to install below this path
             f.write('set (THIRD_PARTY_DIR {0})\n'.format(self.stage.source_path))
             numpy_include = join_path(
                 self.spec['py-numpy'].prefix,
-                self.spec['python'].package.site_packages_dir,
+                self.spec['python'].package.platlib,
                 'numpy', 'core', 'include')
             f.write('set (THIRD_PARTY_INC_DIR "{0}")\n'.format(numpy_include))
 
diff --git a/var/spack/repos/builtin/packages/vigra/package.py b/var/spack/repos/builtin/packages/vigra/package.py
index 94b5a21e5a..fa606f1bca 100644
--- a/var/spack/repos/builtin/packages/vigra/package.py
+++ b/var/spack/repos/builtin/packages/vigra/package.py
@@ -71,7 +71,7 @@ class Vigra(CMakePackage):
                 '-DBoost_DIR={0}'.format(spec['boost'].prefix),
                 '-DBoost_INCLUDE_DIR={0}'.format(spec['boost'].prefix.include),
                 '-DBoost_PYTHON_LIBRARY={0}'.format(boost_python_lib),
-                '-DVIGRANUMPY_INSTALL_DIR={0}'.format(site_packages_dir)
+                '-DVIGRANUMPY_INSTALL_DIR={0}'.format(python_platlib)
             ])
         if '+fftw' in spec:
             args.extend([