From 28533b2553237b388950774bb3262e1906be06ee Mon Sep 17 00:00:00 2001 From: John Parent Date: Wed, 21 Oct 2026 20:21:39 -0400 Subject: [PATCH 1/5] Python: ensure disabled variants disabled corresponding build elements Previously multiple Python package variants could be enabled or disabled, the only change reflected in the build was the inclusion of CFLAGS and LDFLAGS. This allowed python to find and link against system copies of dependencies required to build modules for which the corresponding variant was off, and build said module. This produces modules with potential system references and allows python builds to diverge from the spec requirements. This PR adds arguments to configure that disable the modules our variants configure to be off. Also updates the python requirements pip expects. Pip unconditonally imports zlib and expat, so python must have both those modules Signed-off-by: John Parent --- .../builtin/build_systems/python.py | 5 ++ .../builtin/packages/py_pip/package.py | 3 +- .../builtin/packages/python/package.py | 63 +++++++++++++++++-- 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/repos/spack_repo/builtin/build_systems/python.py b/repos/spack_repo/builtin/build_systems/python.py index a3ec26c3f9d..d131b5ac00a 100644 --- a/repos/spack_repo/builtin/build_systems/python.py +++ b/repos/spack_repo/builtin/build_systems/python.py @@ -18,6 +18,7 @@ Spec, build_system, classproperty, + conflicts, depends_on, determine_number_of_jobs, execute_install_time_tests, @@ -243,6 +244,10 @@ class PythonPackage(PythonExtension): with when("build_system=python_pip"): extends("python") depends_on("py-pip", type="build") + # pip unconditionally imports expat and zlib, we cannot run + # pip against a python without these modules + conflicts("^python~pyexpat", msg="pip cannot run on a python without the pyexpat module") + conflicts("^python~zlib", msg="pip cannot run on a python without the zlib module") # FIXME: technically wheel is only needed when building from source, not when # installing a downloaded wheel, but I don't want to add wheel as a dep to every # package manually diff --git a/repos/spack_repo/builtin/packages/py_pip/package.py b/repos/spack_repo/builtin/packages/py_pip/package.py index a7323fe3b91..9dce7a02029 100644 --- a/repos/spack_repo/builtin/packages/py_pip/package.py +++ b/repos/spack_repo/builtin/packages/py_pip/package.py @@ -54,7 +54,8 @@ class PyPip(Package, PythonExtension): version("10.0.1", sha256="717cdffb2833be8409433a93746744b59505f42146e8d37de6c62b430e25d6d7") version("9.0.1", sha256="690b762c0a8460c303c089d5d0be034fb15a5ea2b75bdf565f40421f542fefb0") - extends("python+zlib") + # pip needs zlib and expat as python modules + extends("python+zlib+pyexpat") with default_args(type=("build", "run")): depends_on("python@3.10:", when="@26.1:") diff --git a/repos/spack_repo/builtin/packages/python/package.py b/repos/spack_repo/builtin/packages/python/package.py index 745e0b55443..4c4a41a2773 100644 --- a/repos/spack_repo/builtin/packages/python/package.py +++ b/repos/spack_repo/builtin/packages/python/package.py @@ -11,7 +11,7 @@ import sys from pathlib import Path from shutil import copy -from typing import Dict, List +from typing import Dict, List, Tuple from spack_repo.builtin.build_systems.generic import Package @@ -210,6 +210,7 @@ class Python(Package): # Optional Python modules variant("readline", default=sys.platform != "win32", description="Build readline module") + variant("curses", default=sys.platform != "win32", description="Build curses modules") variant("ssl", default=True, description="Build ssl module") variant("sqlite3", default=True, description="Build sqlite3 module") variant("dbm", default=True, description="Build dbm module") @@ -245,6 +246,9 @@ class Python(Package): # See detect_modules() in setup.py for details depends_on("readline", when="+readline") depends_on("ncurses", when="+readline") + # python's configure looks for ncurses regardless of readline + # use, model this to prevent system ncurses being linked + depends_on("ncurses", when="+curses") depends_on("openssl", when="+ssl") # https://docs.python.org/3/whatsnew/3.7.html#build-changes depends_on("openssl@1.0.2:", when="+ssl") @@ -260,6 +264,7 @@ class Python(Package): depends_on("zstd libs=shared", when="+zstd") depends_on("expat", when="+pyexpat") depends_on("libffi", when="+ctypes") + depends_on("mpdecimal@2.5.0:2", when="@3.13:") # https://docs.python.org/3/whatsnew/3.11.html#build-changes depends_on("tk@8.5.12:", when="@3.11: +tkinter") depends_on("tk", when="+tkinter") @@ -326,6 +331,35 @@ class Python(Package): # Used to cache various attributes that are expensive to compute _config_vars: Dict[str, Dict[str, str]] = {} + # Map stdlib python modules to their Spack variants and the versions for which Python supports + # configuring these modules, so we can align the python builds with what we expect + # from the concretized python + # + # Python's configure only took over building these modules from setup.py in 3.11, and a + # few of them stayed in setup.py until 3.12, hence the version range on each entry + # These modules are disabled by setting a value, different from a more traditional --with- + # config option. Modules configured with that more traditional approach are handled as we + # normally would + + # (variant, versions in which configure controls the module, module names) + stdlib_modules: List[Tuple[str, str, List[str]]] = [ + ("bz2", "@3.11:", ["_bz2"]), + ("crypt", "@3.11:3.12", ["_crypt"]), + ("curses", "@3.12:", ["_curses", "_curses_panel"]), + ("ctypes", "@3.12:", ["_ctypes"]), + ("lzma", "@3.11:", ["_lzma"]), + ("pyexpat", "@3.11:", ["pyexpat", "_elementtree"]), + ("readline", "@3.12:", ["readline"]), + ("sqlite3", "@3.11:", ["_sqlite3"]), + ("ssl", "@3.11:", ["_ssl", "_hashlib"]), + ("tkinter", "@3.12:", ["_tkinter"]), + ("uuid", "@3.11:", ["_uuid"]), + # NOTE: binascii is built unconditionally and uses zlib for an optimized crc32 when + # configure finds it, so ~zlib still leaves a detected system zlib linked into it. + ("zlib", "@3.11:", ["zlib"]), + ("zstd", "@3.14:", ["_zstd"]), + ] + # An in-source build with --enable-optimizations fails for python@3.X build_directory = "spack-build" @@ -365,6 +399,7 @@ def determine_variants(cls, exes, version_str): for module in [ "readline", + "curses", "sqlite3", "dbm", "zlib", @@ -703,6 +738,12 @@ def configure_args(self): else: config_args.append("--without-system-expat") + # From 3.13 --with-system-libmpdec defaults to yes and will take any libmpdec it finds on + # the system, so pass it explicitly to pin _decimal to spack mpdecimal. Earlier + # versions vendor libmpdec with no configure option for system + if spec.satisfies("@3.13:"): + config_args.append("--with-system-libmpdec") + if self.version < Version("3.12.0"): if "+ctypes" in spec: config_args.append("--with-system-ffi") @@ -724,9 +765,17 @@ def configure_args(self): if "+freethreading" in spec: config_args.append("--disable-gil") - # Disable tkinter module in the configure script for Python 3.12 onwards if ~tkinter - if spec.satisfies("@3.12:") and spec.satisfies("~tkinter"): - config_args.append("py_cv_module__tkinter=n/a") + # Explicitly disable the extension modules that our variants turn off + # see stdlib_modules definition for more details + for variant_name, when, modules in self.stdlib_modules: + if variant_name in spec.variants and spec.satisfies(f"{when} ~{variant_name}"): + config_args.extend(f"py_cv_module_{module}=n/a" for module in modules) + + # In 3.10 and 3.11 setup.py still builds the readline module + # --with-readline=no is the only tuning + # defaults to true + if spec.satisfies("@3.10:3.11 ~readline"): + config_args.append("--with-readline=no") # Disable the nis module in the configure script for Python 3.11 and 3.12. It is deleted # in Python 3.13. See ``def patch`` for disabling the nis module in Python 3.10 and older. @@ -743,7 +792,11 @@ def configure_args(self): if cflags: config_args.append("CFLAGS={0}".format(" ".join(cflags))) - if self.version >= Version("3.12.0") and sys.platform == "darwin": + # Setting both of these makes configure skip pkg-conf for curses, + # ensures _curses is built against the spack ncurses + # args are only used by configure once _curses moved out of setup.py in 3.12. + if self.version >= Version("3.12.0") and spec.satisfies("+curses"): + config_args.append("CURSES_CFLAGS=-I{0}".format(spec["ncurses"].prefix.include)) config_args.append("CURSES_LIBS={0}".format(spec["ncurses"].libs.link_flags)) return config_args From e5ac7c1bebd9d466ff3a45cbd073c638ccb70f2d Mon Sep 17 00:00:00 2001 From: John Parent Date: Thu, 22 Oct 2026 14:26:32 -0400 Subject: [PATCH 2/5] remove redundant constraints Signed-off-by: John Parent --- repos/spack_repo/builtin/build_systems/python.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/repos/spack_repo/builtin/build_systems/python.py b/repos/spack_repo/builtin/build_systems/python.py index d131b5ac00a..adbd3ca4ae1 100644 --- a/repos/spack_repo/builtin/build_systems/python.py +++ b/repos/spack_repo/builtin/build_systems/python.py @@ -244,10 +244,6 @@ class PythonPackage(PythonExtension): with when("build_system=python_pip"): extends("python") depends_on("py-pip", type="build") - # pip unconditionally imports expat and zlib, we cannot run - # pip against a python without these modules - conflicts("^python~pyexpat", msg="pip cannot run on a python without the pyexpat module") - conflicts("^python~zlib", msg="pip cannot run on a python without the zlib module") # FIXME: technically wheel is only needed when building from source, not when # installing a downloaded wheel, but I don't want to add wheel as a dep to every # package manually From e2d17094172d63bddfef1fb833264e8d93010272 Mon Sep 17 00:00:00 2001 From: John Parent Date: Thu, 22 Oct 2026 14:31:06 -0400 Subject: [PATCH 3/5] readline's curses requirement expressed through readline's dependencies Signed-off-by: John Parent --- repos/spack_repo/builtin/packages/python/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/repos/spack_repo/builtin/packages/python/package.py b/repos/spack_repo/builtin/packages/python/package.py index 4c4a41a2773..38fd3e17a9c 100644 --- a/repos/spack_repo/builtin/packages/python/package.py +++ b/repos/spack_repo/builtin/packages/python/package.py @@ -245,7 +245,6 @@ class Python(Package): # Optional dependencies # See detect_modules() in setup.py for details depends_on("readline", when="+readline") - depends_on("ncurses", when="+readline") # python's configure looks for ncurses regardless of readline # use, model this to prevent system ncurses being linked depends_on("ncurses", when="+curses") From 1b51cfdad3c6074b0b83d5227ed5f7824ff1bb7f Mon Sep 17 00:00:00 2001 From: John Parent Date: Thu, 22 Oct 2026 14:33:28 -0400 Subject: [PATCH 4/5] drop unused import Signed-off-by: John Parent --- repos/spack_repo/builtin/build_systems/python.py | 1 - 1 file changed, 1 deletion(-) diff --git a/repos/spack_repo/builtin/build_systems/python.py b/repos/spack_repo/builtin/build_systems/python.py index adbd3ca4ae1..a3ec26c3f9d 100644 --- a/repos/spack_repo/builtin/build_systems/python.py +++ b/repos/spack_repo/builtin/build_systems/python.py @@ -18,7 +18,6 @@ Spec, build_system, classproperty, - conflicts, depends_on, determine_number_of_jobs, execute_install_time_tests, From 0d07d9d96a7c589008416f0027eb7769bab1a028 Mon Sep 17 00:00:00 2001 From: John Parent Date: Fri, 23 Oct 2026 15:11:01 -0400 Subject: [PATCH 5/5] Further update python dependencies version constraints Signed-off-by: John Parent --- repos/spack_repo/builtin/packages/python/package.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/repos/spack_repo/builtin/packages/python/package.py b/repos/spack_repo/builtin/packages/python/package.py index 38fd3e17a9c..86917b0c1f6 100644 --- a/repos/spack_repo/builtin/packages/python/package.py +++ b/repos/spack_repo/builtin/packages/python/package.py @@ -242,8 +242,8 @@ class Python(Package): depends_on("iconv", when="~libxml2") depends_on("gettext ~libxml2", when="~libxml2 ^[virtuals=iconv] libiconv") - # Optional dependencies - # See detect_modules() in setup.py for details + # Optional dependencies and the minimum versions they require + # https://docs.python.org/3/using/configure.html#optional-module-requirements depends_on("readline", when="+readline") # python's configure looks for ncurses regardless of readline # use, model this to prevent system ncurses being linked @@ -256,11 +256,12 @@ class Python(Package): depends_on("sqlite@3.0.8:", when="@:3.9+sqlite3") # https://docs.python.org/3.10/whatsnew/3.10.html#build-changes depends_on("sqlite@3.7.15:", when="@3.10:+sqlite3") + depends_on("sqlite@3.15.2:", when="@3.13:+sqlite3") depends_on("gdbm", when="+dbm") # alternatively ndbm or berkeley-db depends_on("zlib-api", when="+zlib") depends_on("bzip2", when="+bz2") depends_on("xz libs=shared", when="+lzma") - depends_on("zstd libs=shared", when="+zstd") + depends_on("zstd@1.4.5: libs=shared", when="+zstd") depends_on("expat", when="+pyexpat") depends_on("libffi", when="+ctypes") depends_on("mpdecimal@2.5.0:2", when="@3.13:")