python: add 3.12.0 (but keep 3.11 preferred) (#40282)

9 files changed, 318 insertions, 6 deletions

diff --git a/lib/spack/spack/build_systems/meson.py b/lib/spack/spack/build_systems/meson.py
index 1a5e69e081..38939dc7ad 100644
--- a/lib/spack/spack/build_systems/meson.py
+++ b/lib/spack/spack/build_systems/meson.py
@@ -10,7 +10,7 @@ import llnl.util.filesystem as fs
 
 import spack.builder
 import spack.package_base
-from spack.directives import build_system, depends_on, variant
+from spack.directives import build_system, conflicts, depends_on, variant
 from spack.multimethod import when
 
 from ._checks import BaseBuilder, execute_build_time_tests
@@ -47,6 +47,13 @@ class MesonPackage(spack.package_base.PackageBase):
         variant("strip", default=False, description="Strip targets on install")
         depends_on("meson", type="build")
         depends_on("ninja", type="build")
+        # Python detection in meson requires distutils to be importable, but distutils no longer
+        # exists in Python 3.12. In Spack, we can't use setuptools as distutils replacement,
+        # because the distutils-precedence.pth startup file that setuptools ships with is not run
+        # when setuptools is in PYTHONPATH; it has to be in system site-packages. In a future meson
+        # release, the distutils requirement will be dropped, so this conflict can be relaxed.
+        # We have patches to make it work with meson 1.1 and above.
+        conflicts("^python@3.12:", when="^meson@:1.0")
 
     def flags_to_build_system_args(self, flags):
         """Produces a list of all command line arguments to pass the specified
diff --git a/var/spack/repos/builtin/packages/flux-core/package.py b/var/spack/repos/builtin/packages/flux-core/package.py
index 2e5126fc54..70d8cc9873 100644
--- a/var/spack/repos/builtin/packages/flux-core/package.py
+++ b/var/spack/repos/builtin/packages/flux-core/package.py
@@ -141,6 +141,8 @@ class FluxCore(AutotoolsPackage):
     # `link` dependency on python due to Flux's `pymod` module
     depends_on("python@3.6:", when="@0.17:", type=("build", "link", "run"))
     depends_on("python@2.7:", type=("build", "link", "run"))
+    # Use of distutils in configure script dropped in v0.55
+    depends_on("python@:3.11", when="@:0.54", type=("build", "link", "run"))
     depends_on("py-cffi@1.1:", type=("build", "run"))
     depends_on("py-six@1.9:", when="@:0.24", type=("build", "run"))
     depends_on("py-pyyaml@3.10:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/glib/package.py b/var/spack/repos/builtin/packages/glib/package.py
index e1744d1d23..4f8d1415f1 100644
--- a/var/spack/repos/builtin/packages/glib/package.py
+++ b/var/spack/repos/builtin/packages/glib/package.py
@@ -139,7 +139,8 @@ class Glib(MesonPackage, AutotoolsPackage):
     depends_on("zlib-api")
     depends_on("gettext")
     depends_on("perl", type=("build", "run"))
-    depends_on("python", type=("build", "run"), when="@2.53.4:")
+    # Uses distutils in gio/gdbus-2.0/codegen/utils.py
+    depends_on("python@:3.11", type=("build", "run"), when="@2.53.4:")
     depends_on("pcre2", when="@2.73.2:")
     depends_on("pcre2@10.34:", when="@2.74:")
     depends_on("pcre+utf", when="@2.48:2.73.1")
diff --git a/var/spack/repos/builtin/packages/mesa/package.py b/var/spack/repos/builtin/packages/mesa/package.py
index 60fb95beb6..e1e666a6bc 100644
--- a/var/spack/repos/builtin/packages/mesa/package.py
+++ b/var/spack/repos/builtin/packages/mesa/package.py
@@ -52,7 +52,8 @@ class Mesa(MesonPackage):
     depends_on("cmake", type="build")
     depends_on("flex", type="build")
     depends_on("gettext", type="build")
-    depends_on("python@3:", type="build")
+    # Upperbound on 3.11 because distutils is used for checking py-mako
+    depends_on("python@3:3.11", type="build")
     depends_on("py-mako@0.8.0:", type="build")
     depends_on("unwind")
     depends_on("expat")
diff --git a/var/spack/repos/builtin/packages/meson/package.py b/var/spack/repos/builtin/packages/meson/package.py
index 9cc6a63723..cabafbe001 100644
--- a/var/spack/repos/builtin/packages/meson/package.py
+++ b/var/spack/repos/builtin/packages/meson/package.py
@@ -18,6 +18,7 @@ class Meson(PythonPackage):
 
     maintainers("eli-schwartz", "michaelkuhn")
 
+    version("1.2.2", sha256="1caa0ef6082e311bdca9836e7907f548b8c3f041a42ed41f0ff916b83ac7dddd")
     version("1.2.1", sha256="e1f3b32b636cc86496261bd89e63f00f206754697c7069788b62beed5e042713")
     version("1.2.0", sha256="603489f0aaa6305f806c6cc4a4455a965f22290fc74f65871f589b002110c790")
     version("1.1.1", sha256="1c3b9e1a3a36b51adb5de498d582fd5cbf6763fadbcf151de9f2a762e02bd2e6")
@@ -85,6 +86,9 @@ class Meson(PythonPackage):
     # https://github.com/mesonbuild/meson/pull/9850
     patch("oneapi.patch", when="@0.62:0.63 %oneapi")
 
+    # Python 3.12 detection support
+    patch("python-3.12-support.patch", when="@1.1:1.2.2")
+
     executables = ["^meson$"]
 
     @classmethod
diff --git a/var/spack/repos/builtin/packages/meson/python-3.12-support.patch b/var/spack/repos/builtin/packages/meson/python-3.12-support.patch
new file mode 100644
index 0000000000..727976ff26
--- /dev/null
+++ b/var/spack/repos/builtin/packages/meson/python-3.12-support.patch
@@ -0,0 +1,283 @@
+From 5f96e35b873d6230970fd63ba2e706bbd3f4e26f Mon Sep 17 00:00:00 2001
+From: Eli Schwartz <eschwartz93@gmail.com>
+Date: Fri, 8 Sep 2023 16:54:48 -0400
+Subject: [PATCH 1/6] python dependency: ensure that setuptools doesn't inject
+ itself into distutils
+
+We do not use setuptools for anything, and only lightly use distutils.
+Unpredictable issues can occur due to setuptools monkey-patching, which
+interferes with our intended use. Tell setuptools to simply never get
+involved.
+
+Note: while it's otherwise possible to check if the probe is run using
+sys.executable and avoid forking, setuptools unconditionally injects
+itself at startup in a way that requires subprocess isolation to
+disable.
+
+(cherry picked from commit 9f610ad5b72ea91de2d7aeb6f3266d0a7477062e)
+---
+ mesonbuild/dependencies/python.py | 4 +++-
+ 1 file changed, 3 insertions(+), 1 deletion(-)
+
+diff --git a/mesonbuild/dependencies/python.py b/mesonbuild/dependencies/python.py
+index 160772888..f04494674 100644
+--- a/mesonbuild/dependencies/python.py
++++ b/mesonbuild/dependencies/python.py
+@@ -113,7 +113,9 @@ class BasicPythonExternalProgram(ExternalProgram):
+ 
+         with importlib.resources.path('mesonbuild.scripts', 'python_info.py') as f:
+             cmd = self.get_command() + [str(f)]
+-            p, stdout, stderr = mesonlib.Popen_safe(cmd)
++            env = os.environ.copy()
++            env['SETUPTOOLS_USE_DISTUTILS'] = 'stdlib'
++            p, stdout, stderr = mesonlib.Popen_safe(cmd, env=env)
+ 
+         try:
+             info = json.loads(stdout)
+-- 
+2.39.2
+
+
+From cb4e62a8c55118988babac8b8254e0af1dc9698b Mon Sep 17 00:00:00 2001
+From: Eli Schwartz <eschwartz@archlinux.org>
+Date: Mon, 21 Nov 2022 20:47:14 -0500
+Subject: [PATCH 2/6] python module: stop using distutils schemes on
+ sufficiently new Debian
+
+Since 3.10.3, Debian finally started patching sysconfig with custom
+paths, instead of just distutils. This means we can now go use that
+instead. It reduces our reliance on the deprecated distutils module.
+
+Partial fix for #7702
+
+(cherry picked from commit 40f897fa92f7d3cc43788d3000733310ce77cf0c)
+---
+ mesonbuild/scripts/python_info.py | 32 +++++++++++++++++++++++--------
+ 1 file changed, 24 insertions(+), 8 deletions(-)
+
+diff --git a/mesonbuild/scripts/python_info.py b/mesonbuild/scripts/python_info.py
+index 9c3a0791a..65597b121 100755
+--- a/mesonbuild/scripts/python_info.py
++++ b/mesonbuild/scripts/python_info.py
+@@ -13,7 +13,6 @@ if sys.path[0].endswith('scripts'):
+     del sys.path[0]
+ 
+ import json, os, sysconfig
+-import distutils.command.install
+ 
+ def get_distutils_paths(scheme=None, prefix=None):
+     import distutils.dist
+@@ -37,15 +36,32 @@ def get_distutils_paths(scheme=None, prefix=None):
+ # default scheme to a custom one pointing to /usr/local and replacing
+ # site-packages with dist-packages.
+ # See https://github.com/mesonbuild/meson/issues/8739.
+-# XXX: We should be using sysconfig, but Debian only patches distutils.
++#
++# We should be using sysconfig, but before 3.10.3, Debian only patches distutils.
++# So we may end up falling back.
+ 
+-if 'deb_system' in distutils.command.install.INSTALL_SCHEMES:
+-    paths = get_distutils_paths(scheme='deb_system')
+-    install_paths = get_distutils_paths(scheme='deb_system', prefix='')
+-else:
+-    paths = sysconfig.get_paths()
++def get_install_paths():
++    if sys.version_info >= (3, 10):
++        scheme = sysconfig.get_default_scheme()
++    else:
++        scheme = sysconfig._get_default_scheme()
++
++    if sys.version_info >= (3, 10, 3):
++        if 'deb_system' in sysconfig.get_scheme_names():
++            scheme = 'deb_system'
++    else:
++        import distutils.command.install
++        if 'deb_system' in distutils.command.install.INSTALL_SCHEMES:
++            paths = get_distutils_paths(scheme='deb_system')
++            install_paths = get_distutils_paths(scheme='deb_system', prefix='')
++            return paths, install_paths
++
++    paths = sysconfig.get_paths(scheme=scheme)
+     empty_vars = {'base': '', 'platbase': '', 'installed_base': ''}
+-    install_paths = sysconfig.get_paths(vars=empty_vars)
++    install_paths = sysconfig.get_paths(scheme=scheme, vars=empty_vars)
++    return paths, install_paths
++
++paths, install_paths = get_install_paths()
+ 
+ def links_against_libpython():
+     from distutils.core import Distribution, Extension
+-- 
+2.39.2
+
+
+From c179c18765514d5c37737dec996b4c91cb31477f Mon Sep 17 00:00:00 2001
+From: Eli Schwartz <eschwartz93@gmail.com>
+Date: Mon, 2 Oct 2023 16:40:15 -0400
+Subject: [PATCH 3/6] python module: refactor pypy detection into a consistent
+ variable
+
+(cherry picked from commit 3d3a10ef022284c8377bd9f8e1b1adec73c50d95)
+---
+ mesonbuild/scripts/python_info.py | 4 +++-
+ 1 file changed, 3 insertions(+), 1 deletion(-)
+
+diff --git a/mesonbuild/scripts/python_info.py b/mesonbuild/scripts/python_info.py
+index 65597b121..d17b3a376 100755
+--- a/mesonbuild/scripts/python_info.py
++++ b/mesonbuild/scripts/python_info.py
+@@ -72,6 +72,8 @@ def links_against_libpython():
+ variables = sysconfig.get_config_vars()
+ variables.update({'base_prefix': getattr(sys, 'base_prefix', sys.prefix)})
+ 
++is_pypy = '__pypy__' in sys.builtin_module_names
++
+ if sys.version_info < (3, 0):
+     suffix = variables.get('SO')
+ elif sys.version_info < (3, 8, 7):
+@@ -88,7 +90,7 @@ print(json.dumps({
+   'install_paths': install_paths,
+   'version': sysconfig.get_python_version(),
+   'platform': sysconfig.get_platform(),
+-  'is_pypy': '__pypy__' in sys.builtin_module_names,
++  'is_pypy': is_pypy,
+   'is_venv': sys.prefix != variables['base_prefix'],
+   'link_libpython': links_against_libpython(),
+   'suffix': suffix,
+-- 
+2.39.2
+
+
+From 3c493dae4bd8410bfb09e8f654605f65e15d8e66 Mon Sep 17 00:00:00 2001
+From: Eli Schwartz <eschwartz@archlinux.org>
+Date: Tue, 22 Nov 2022 22:56:10 -0500
+Subject: [PATCH 4/6] python module: stop using distutils "link to libpython"
+ probe on recent python
+
+On python >=3.8, this information is expected to be encoded in the
+sysconfig vars.
+
+In distutils, it is always necessary to link to libpython on Windows;
+for posix platforms, it depends on the value of LIBPYTHON (which is the
+library to link to, possibly the empty string) as generated by
+configure.ac and embedded into python.pc and python-config.sh, and then
+coded a second time in the distutils python sources.
+
+There are a couple of caveats which have ramifications for Cygwin and
+Android:
+
+- python.pc and python-config.sh disagree with distutils when python is
+  not built shared. In that case, the former act the same as a shared
+  build, while the latter *never* links to libpython
+
+- python.pc disagrees with python-config.sh and distutils when python is
+  built shared. The former never links to libpython, while the latter do
+
+The disagreement is resolved in favor of distutils' behavior in all
+cases, and python.pc is correct for our purposes on python 3.12; see:
+https://github.com/python/cpython/pull/100356
+https://github.com/python/cpython/pull/100967
+
+Although it was not backported to older releases, Cygwin at least has
+always patched in a fix for python.pc, which behavior is now declared
+canonical. We can reliably assume it is always correct.
+
+This is the other half of the fix for #7702
+
+(cherry picked from commit 2d6c10908b3771216e7ce086af1ee4dc77e698c2)
+---
+ mesonbuild/scripts/python_info.py | 17 +++++++++++++----
+ 1 file changed, 13 insertions(+), 4 deletions(-)
+
+diff --git a/mesonbuild/scripts/python_info.py b/mesonbuild/scripts/python_info.py
+index d17b3a376..a3f3d3535 100755
+--- a/mesonbuild/scripts/python_info.py
++++ b/mesonbuild/scripts/python_info.py
+@@ -64,10 +64,19 @@ def get_install_paths():
+ paths, install_paths = get_install_paths()
+ 
+ def links_against_libpython():
+-    from distutils.core import Distribution, Extension
+-    cmd = Distribution().get_command_obj('build_ext')
+-    cmd.ensure_finalized()
+-    return bool(cmd.get_libraries(Extension('dummy', [])))
++    # on versions supporting python-embed.pc, this is the non-embed lib
++    #
++    # PyPy is not yet up to 3.12 and work is still pending to export the
++    # relevant information (it doesn't automatically provide arbitrary
++    # Makefile vars)
++    if sys.version_info >= (3, 8) and not is_pypy:
++        variables = sysconfig.get_config_vars()
++        return bool(variables.get('LIBPYTHON', 'yes'))
++    else:
++        from distutils.core import Distribution, Extension
++        cmd = Distribution().get_command_obj('build_ext')
++        cmd.ensure_finalized()
++        return bool(cmd.get_libraries(Extension('dummy', [])))
+ 
+ variables = sysconfig.get_config_vars()
+ variables.update({'base_prefix': getattr(sys, 'base_prefix', sys.prefix)})
+-- 
+2.39.2
+
+
+From ae44d9a379faca6274db819be44ffca3e0159f56 Mon Sep 17 00:00:00 2001
+From: Eli Schwartz <eschwartz93@gmail.com>
+Date: Mon, 2 Oct 2023 23:51:57 -0400
+Subject: [PATCH 5/6] tests: fix test case to not import distutils on python
+ 3.12
+
+Testing the correctness of the `modules: ` kwarg can be done with other
+guaranteed stdlib modules that are even more guaranteed since they
+didn't get deprecated for removal.
+
+(cherry picked from commit ecf261330c498783760cbde00b613b7469f8d3c0)
+---
+ test cases/python/5 modules kwarg/meson.build | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/test cases/python/5 modules kwarg/meson.build b/test cases/python/5 modules kwarg/meson.build
+index 9751adaab..41a9a4fae 100644
+--- a/test cases/python/5 modules kwarg/meson.build	
++++ b/test cases/python/5 modules kwarg/meson.build	
+@@ -1,7 +1,7 @@
+ project('python kwarg')
+ 
+ py = import('python')
+-prog_python = py.find_installation('python3', modules : ['distutils'])
++prog_python = py.find_installation('python3', modules : ['os', 'sys', 're'])
+ assert(prog_python.found() == true, 'python not found when should be')
+ prog_python = py.find_installation('python3', modules : ['thisbetternotexistmod'], required : false)
+ assert(prog_python.found() == false, 'python not found but reported as found')
+-- 
+2.39.2
+
+
+From d9abf4a97dc182b3c57204a792000d620f9f941e Mon Sep 17 00:00:00 2001
+From: Eli Schwartz <eschwartz93@gmail.com>
+Date: Tue, 3 Oct 2023 00:22:25 -0400
+Subject: [PATCH 6/6] mark the PyPI metadata as supporting python 3.12
+
+meson itself runs okay on 3.12, and the last issue for *probing* against
+3.12 is solved. Tests pass here locally.
+
+(cherry picked from commit 880f21281ee359e01de659fe7d45549d19e6b84d)
+---
+ setup.cfg | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/setup.cfg b/setup.cfg
+index dfaba76dd..2f2962eed 100644
+--- a/setup.cfg
++++ b/setup.cfg
+@@ -30,6 +30,7 @@ classifiers =
+   Programming Language :: Python :: 3.9
+   Programming Language :: Python :: 3.10
+   Programming Language :: Python :: 3.11
++  Programming Language :: Python :: 3.12
+   Topic :: Software Development :: Build Tools
+ long_description = Meson is a cross-platform build system designed to be both as fast and as user friendly as possible. It supports many languages and compilers, including GCC, Clang, PGI, Intel, and Visual Studio. Its build definitions are written in a simple non-Turing complete DSL.
+ 
+-- 
+2.39.2
+
diff --git a/var/spack/repos/builtin/packages/py-cffi/package.py b/var/spack/repos/builtin/packages/py-cffi/package.py
index 606c678369..ccbe262a15 100644
--- a/var/spack/repos/builtin/packages/py-cffi/package.py
+++ b/var/spack/repos/builtin/packages/py-cffi/package.py
@@ -24,6 +24,10 @@ class PyCffi(PythonPackage):
     version("1.10.0", sha256="b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5")
     version("1.1.2", sha256="390970b602708c91ddc73953bb6929e56291c18a4d80f360afa00fad8b6f3339")
 
+    # ./spack-src/cffi/ffiplatform.py has _hack_at_distutils which imports
+    # setuptools before distutils, but only on Windows. This could be made
+    # unconditional to support Python 3.12
+    depends_on("python@:3.11", type=("build", "run"))
     depends_on("pkgconfig", type="build")
     depends_on("py-setuptools", type="build")
     depends_on("py-pycparser", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-cryptography/package.py b/var/spack/repos/builtin/packages/py-cryptography/package.py
index caadcf16f3..c17e073777 100644
--- a/var/spack/repos/builtin/packages/py-cryptography/package.py
+++ b/var/spack/repos/builtin/packages/py-cryptography/package.py
@@ -13,6 +13,7 @@ class PyCryptography(PythonPackage):
     homepage = "https://github.com/pyca/cryptography"
     pypi = "cryptography/cryptography-1.8.1.tar.gz"
 
+    version("41.0.3", sha256="6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34")
     version("40.0.2", sha256="c33c0d32b8594fa647d2e01dbccc303478e16fdd7cf98652d5b3ed11aa5e5c99")
     version("38.0.1", sha256="1db3d807a14931fa317f96435695d9ec386be7b84b618cc61cfa5d08b0ae33d7")
     version("37.0.4", sha256="63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82")
@@ -28,12 +29,16 @@ class PyCryptography(PythonPackage):
 
     variant("idna", default=False, when="@2.5:3.0", description="Deprecated U-label support")
 
+    # distutils required in version <= 40
+    depends_on("python@:3.11", when="@:40", type=("build", "run"))
+    depends_on("py-setuptools@61.0:", when="@41:", type="build")
     depends_on("py-setuptools@40.6:60.8,60.9.1:", when="@37:", type="build")
     depends_on("py-setuptools@40.6:", when="@2.7:36", type="build")
     depends_on("py-setuptools@18.5:", when="@2.2:2.6", type="build")
     depends_on("py-setuptools@11.3:", when="@:2.1", type="build")
     depends_on("py-setuptools-rust@0.11.4:", when="@3.4.2:", type="build")
     depends_on("py-setuptools-rust@0.11.4:", when="@3.4:3.4.1", type=("build", "run"))
+    depends_on("rust@1.56:", when="@41:", type="build")
     depends_on("rust@1.48:", when="@38:", type="build")
     depends_on("rust@1.41:", when="@3.4.5:", type="build")
     depends_on("rust@1.45:", when="@3.4.3:3.4.4", type="build")
diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
index 2a904c7386..c07ad5d6ad 100644
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -40,7 +40,12 @@ class Python(Package):
     install_targets = ["install"]
     build_targets: List[str] = []
 
-    version("3.11.4", sha256="85c37a265e5c9dd9f75b35f954e31fbfc10383162417285e30ad25cc073a0d63")
+    version("3.12.0", sha256="51412956d24a1ef7c97f1cb5f70e185c13e3de1f50d131c0aac6338080687afb")
+    version(
+        "3.11.4",
+        sha256="85c37a265e5c9dd9f75b35f954e31fbfc10383162417285e30ad25cc073a0d63",
+        preferred=True,
+    )
     version("3.11.3", sha256="1a79f3df32265d9e6625f1a0b31c28eb1594df911403d11f3320ee1da1b3e048")
     version("3.11.2", sha256="2411c74bda5bbcfcddaf4531f66d1adc73f247f529aee981b029513aefdbf849")
     version("3.11.1", sha256="baed518e26b337d4d8105679caf68c5c32630d702614fc174e98cb95c46bdfa4")
@@ -272,7 +277,7 @@ class Python(Package):
     patch("python-3.7.2-distutils-C++.patch", when="@3.7.2")
     patch("python-3.7.3-distutils-C++.patch", when="@3.7.3")
     patch("python-3.7.4+-distutils-C++.patch", when="@3.7.4:3.10")
-    patch("python-3.7.4+-distutils-C++-testsuite.patch", when="@3.7.4:")
+    patch("python-3.7.4+-distutils-C++-testsuite.patch", when="@3.7.4:3.11")
     patch("python-3.11-distutils-C++.patch", when="@3.11.0:3.11")
     patch("cpython-windows-externals.patch", when="@:3.9.6 platform=windows")
     patch("tkinter-3.7.patch", when="@3.7 platform=darwin")
@@ -287,7 +292,7 @@ class Python(Package):
 
     # Ensure that distutils chooses correct compiler option for RPATH on fj:
     patch("fj-rpath-3.1.patch", when="@:3.9.7,3.10.0 %fj")
-    patch("fj-rpath-3.9.patch", when="@3.9.8:3.9,3.10.1: %fj")
+    patch("fj-rpath-3.9.patch", when="@3.9.8:3.9,3.10.1:3.11 %fj")
 
     # Fixes build with the Intel compilers
     # https://github.com/python/cpython/pull/16717