Skip to content

Commit 266955f

Browse files
geofftgeofft
committed
Statically link libpython into interpreter (but keep building libpython3.x.so)
1 parent 6b8bdb0 commit 266955f

5 files changed

Lines changed: 350 additions & 0 deletions

‎cpython-unix/build-cpython.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,14 @@ if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_10}" ]; then
298298
patch -p1 -i ${ROOT}/patch-configure-crypt-no-modify-libs.patch
299299
fi
300300

301+
# Build a libpython3.x.so, but statically link the interpreter against
302+
# libpython.
303+
if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then
304+
patch -p1 -i "${ROOT}/patch-python-configure-add-enable-static-libpython-for-interpreter.patch"
305+
else
306+
patch -p1 -i "${ROOT}/patch-python-configure-add-enable-static-libpython-for-interpreter-${PYTHON_MAJMIN_VERSION}.patch"
307+
fi
308+
301309
# We patched configure.ac above. Reflect those changes.
302310
autoconf
303311

@@ -382,6 +390,7 @@ CONFIGURE_FLAGS="
382390
--with-system-expat
383391
--with-system-libmpdec
384392
--without-ensurepip
393+
--enable-static-libpython-for-interpreter
385394
${EXTRA_CONFIGURE_FLAGS}"
386395

387396

‎cpython-unix/patch-python-configure-add-enable-static-libpython-for-interpreter-3.10.patch‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
From 579a7cf9498ccfa656dd720a5db8dd6e04e97150 Mon Sep 17 00:00:00 2001
2+
From: Geoffrey Thomas <geofft@ldpreload.com>
3+
Date: Sat, 19 Apr 2025 11:13:40 -0400
4+
Subject: [PATCH 1/1] configure: add --enable-static-libpython-for-interpreter
5+
6+
This option changes the behavior of --enable-shared to continue to build
7+
the libpython3.x.so shared library, but not use it for linking the
8+
python3 interpreter executable. Instead, the executable is linked
9+
directly against the libpython .o files as it would be with
10+
--disable-shared [in newer versions of Python].
11+
12+
There are two benefits of this change. First, libpython uses
13+
thread-local storage, which is noticeably slower when used in a loaded
14+
module instead of in the main program, because the main program can take
15+
advantage of constant offsets from the thread state pointer but loaded
16+
modules have to dynamically call a function __tls_get_addr() to
17+
potentially allocate their thread-local storage area. (There is another
18+
thread-local storage model for dynamic libraries which mitigates most of
19+
this performance hit, but it comes at the cost of preventing
20+
dlopen("libpython3.x.so"), which is a use case we want to preserve.)
21+
22+
Second, this improves the user experience around relocatable Python a
23+
little bit, in that we don't need to use an $ORIGIN-relative path to
24+
locate libpython3.x.so, which has some mild benefits around musl (which
25+
does not support $ORIGIN-relative DT_NEEDED, only $ORIGIN-relative
26+
DT_RPATH/DT_RUNPATH), users who want to make the interpreter setuid or
27+
setcap (which prevents processing $ORIGIN), etc.
28+
---
29+
Makefile.pre.in | 4 +++-
30+
configure.ac | 18 ++++++++++++++++++
31+
2 files changed, 21 insertions(+), 1 deletion(-)
32+
33+
diff --git a/Makefile.pre.in b/Makefile.pre.in
34+
index fa99dd86c41..84c00a5c071 100644
35+
--- a/Makefile.pre.in
36+
+++ b/Makefile.pre.in
37+
@@ -458,6 +458,8 @@ LIBRARY_OBJS= \
38+
$(LIBRARY_OBJS_OMIT_FROZEN) \
39+
Python/frozen.o
40+
41+
+LINK_PYTHON_OBJS=@LINK_PYTHON_OBJS@
42+
+
43+
##########################################################################
44+
# DTrace
45+
46+
@@ -586,7 +588,7 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c
47+
48+
# Build the interpreter
49+
$(BUILDPYTHON): Programs/python.o $(LIBRARY_DEPS)
50+
- $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
51+
+ $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS)
52+
53+
platform: $(BUILDPYTHON) pybuilddir.txt
54+
$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform
55+
diff --git a/configure.ac b/configure.ac
56+
index ac3be3850a9..a07003a24ed 100644
57+
--- a/configure.ac
58+
+++ b/configure.ac
59+
@@ -1093,6 +1093,17 @@ then
60+
fi
61+
AC_MSG_RESULT($enable_shared)
62+
63+
+AC_MSG_CHECKING([for --enable-static-libpython-for-interpreter])
64+
+AC_ARG_ENABLE([static-libpython-for-interpreter],
65+
+ AS_HELP_STRING([--enable-static-libpython-for-interpreter],
66+
+ [even with --enable-shared, statically link libpython into the interpreter (default is to use the shared library)]))
67+
+
68+
+if test -z "$enable_static_libpython_for_interpreter"
69+
+then
70+
+ enable_static_libpython_for_interpreter="no"
71+
+fi
72+
+AC_MSG_RESULT([$enable_static_libpython_for_interpreter])
73+
+
74+
AC_MSG_CHECKING(for --enable-profiling)
75+
AC_ARG_ENABLE(profiling,
76+
AS_HELP_STRING([--enable-profiling], [enable C-level code profiling with gprof (default is no)]))
77+
@@ -1198,6 +1209,13 @@ fi
78+
79+
AC_MSG_RESULT($LDLIBRARY)
80+
81+
+if test "$enable_static_libpython_for_interpreter" = "yes"; then
82+
+ LINK_PYTHON_OBJS='$(LIBRARY_OBJS)'
83+
+else
84+
+ LINK_PYTHON_OBJS='$(BLDLIBRARY)'
85+
+fi
86+
+AC_SUBST(LINK_PYTHON_OBJS)
87+
+
88+
AC_SUBST(AR)
89+
AC_CHECK_TOOLS(AR, ar aal, ar)
90+
91+
--
92+
2.39.5 (Apple Git-154)
93+

‎cpython-unix/patch-python-configure-add-enable-static-libpython-for-interpreter-3.11.patch‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
From a5182aec2c0597adb8a01298af120809fcf3187b Mon Sep 17 00:00:00 2001
2+
From: Geoffrey Thomas <geofft@ldpreload.com>
3+
Date: Sat, 19 Apr 2025 11:13:40 -0400
4+
Subject: [PATCH 1/1] configure: add --enable-static-libpython-for-interpreter
5+
6+
This option changes the behavior of --enable-shared to continue to build
7+
the libpython3.x.so shared library, but not use it for linking the
8+
python3 interpreter executable. Instead, the executable is linked
9+
directly against the libpython .o files as it would be with
10+
--disable-shared.
11+
12+
There are two benefits of this change. First, libpython uses
13+
thread-local storage, which is noticeably slower when used in a loaded
14+
module instead of in the main program, because the main program can take
15+
advantage of constant offsets from the thread state pointer but loaded
16+
modules have to dynamically call a function __tls_get_addr() to
17+
potentially allocate their thread-local storage area. (There is another
18+
thread-local storage model for dynamic libraries which mitigates most of
19+
this performance hit, but it comes at the cost of preventing
20+
dlopen("libpython3.x.so"), which is a use case we want to preserve.)
21+
22+
Second, this improves the user experience around relocatable Python a
23+
little bit, in that we don't need to use an $ORIGIN-relative path to
24+
locate libpython3.x.so, which has some mild benefits around musl (which
25+
does not support $ORIGIN-relative DT_NEEDED, only $ORIGIN-relative
26+
DT_RPATH/DT_RUNPATH), users who want to make the interpreter setuid or
27+
setcap (which prevents processing $ORIGIN), etc.
28+
---
29+
configure.ac | 17 ++++++++++++++++-
30+
1 file changed, 16 insertions(+), 1 deletion(-)
31+
32+
diff --git a/configure.ac b/configure.ac
33+
index ab5e1de6fab..6783c36da4d 100644
34+
--- a/configure.ac
35+
+++ b/configure.ac
36+
@@ -1419,6 +1419,17 @@ fi],
37+
[AC_MSG_RESULT(yes)])
38+
AC_SUBST(STATIC_LIBPYTHON)
39+
40+
+AC_MSG_CHECKING([for --enable-static-libpython-for-interpreter])
41+
+AC_ARG_ENABLE([static-libpython-for-interpreter],
42+
+ AS_HELP_STRING([--enable-static-libpython-for-interpreter],
43+
+ [even with --enable-shared, statically link libpython into the interpreter (default is to use the shared library)]))
44+
+
45+
+if test -z "$enable_static_libpython_for_interpreter"
46+
+then
47+
+ enable_static_libpython_for_interpreter="no"
48+
+fi
49+
+AC_MSG_RESULT([$enable_static_libpython_for_interpreter])
50+
+
51+
AC_MSG_CHECKING(for --enable-profiling)
52+
AC_ARG_ENABLE(profiling,
53+
AS_HELP_STRING([--enable-profiling], [enable C-level code profiling with gprof (default is no)]))
54+
@@ -1563,7 +1574,11 @@ if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then
55+
LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS"
56+
fi
57+
# Link Python program to the shared library
58+
- LINK_PYTHON_OBJS='$(BLDLIBRARY)'
59+
+ if test "$enable_static_libpython_for_interpreter" = "yes"; then
60+
+ LINK_PYTHON_OBJS='$(LIBRARY_OBJS)'
61+
+ else
62+
+ LINK_PYTHON_OBJS='$(BLDLIBRARY)'
63+
+ fi
64+
else
65+
if test "$STATIC_LIBPYTHON" = 0; then
66+
# Build Python needs object files but don't need to build
67+
--
68+
2.39.5 (Apple Git-154)
69+

‎cpython-unix/patch-python-configure-add-enable-static-libpython-for-interpreter-3.9.patch‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
From 5ae9112a87d45c3aff5ee269ff8e2e49ca278ed3 Mon Sep 17 00:00:00 2001
2+
From: Geoffrey Thomas <geofft@ldpreload.com>
3+
Date: Sat, 19 Apr 2025 11:13:40 -0400
4+
Subject: [PATCH 1/1] configure: add --enable-static-libpython-for-interpreter
5+
6+
This option changes the behavior of --enable-shared to continue to build
7+
the libpython3.x.so shared library, but not use it for linking the
8+
python3 interpreter executable. Instead, the executable is linked
9+
directly against the libpython .o files as it would be with
10+
--disable-shared [in newer versions of Python].
11+
12+
There are two benefits of this change. First, libpython uses
13+
thread-local storage, which is noticeably slower when used in a loaded
14+
module instead of in the main program, because the main program can take
15+
advantage of constant offsets from the thread state pointer but loaded
16+
modules have to dynamically call a function __tls_get_addr() to
17+
potentially allocate their thread-local storage area. (There is another
18+
thread-local storage model for dynamic libraries which mitigates most of
19+
this performance hit, but it comes at the cost of preventing
20+
dlopen("libpython3.x.so"), which is a use case we want to preserve.)
21+
22+
Second, this improves the user experience around relocatable Python a
23+
little bit, in that we don't need to use an $ORIGIN-relative path to
24+
locate libpython3.x.so, which has some mild benefits around musl (which
25+
does not support $ORIGIN-relative DT_NEEDED, only $ORIGIN-relative
26+
DT_RPATH/DT_RUNPATH), users who want to make the interpreter setuid or
27+
setcap (which prevents processing $ORIGIN), etc.
28+
---
29+
Makefile.pre.in | 4 +++-
30+
configure.ac | 18 ++++++++++++++++++
31+
2 files changed, 21 insertions(+), 1 deletion(-)
32+
33+
diff --git a/Makefile.pre.in b/Makefile.pre.in
34+
index a276d535c7f..193439aa73e 100644
35+
--- a/Makefile.pre.in
36+
+++ b/Makefile.pre.in
37+
@@ -460,6 +460,8 @@ LIBRARY_OBJS= \
38+
$(LIBRARY_OBJS_OMIT_FROZEN) \
39+
Python/frozen.o
40+
41+
+LINK_PYTHON_OBJS=@LINK_PYTHON_OBJS@
42+
+
43+
##########################################################################
44+
# DTrace
45+
46+
@@ -589,7 +591,7 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c
47+
48+
# Build the interpreter
49+
$(BUILDPYTHON): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) $(EXPORTSYMS)
50+
- $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
51+
+ $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS)
52+
53+
platform: $(BUILDPYTHON) pybuilddir.txt
54+
$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform
55+
diff --git a/configure.ac b/configure.ac
56+
index aa515da4655..122b11def62 100644
57+
--- a/configure.ac
58+
+++ b/configure.ac
59+
@@ -1106,6 +1106,17 @@ then
60+
fi
61+
AC_MSG_RESULT($enable_shared)
62+
63+
+AC_MSG_CHECKING([for --enable-static-libpython-for-interpreter])
64+
+AC_ARG_ENABLE([static-libpython-for-interpreter],
65+
+ AS_HELP_STRING([--enable-static-libpython-for-interpreter],
66+
+ [even with --enable-shared, statically link libpython into the interpreter (default is to use the shared library)]))
67+
+
68+
+if test -z "$enable_static_libpython_for_interpreter"
69+
+then
70+
+ enable_static_libpython_for_interpreter="no"
71+
+fi
72+
+AC_MSG_RESULT([$enable_static_libpython_for_interpreter])
73+
+
74+
AC_MSG_CHECKING(for --enable-profiling)
75+
AC_ARG_ENABLE(profiling,
76+
AS_HELP_STRING([--enable-profiling], [enable C-level code profiling with gprof (default is no)]))
77+
@@ -1211,6 +1222,13 @@ fi
78+
79+
AC_MSG_RESULT($LDLIBRARY)
80+
81+
+if test "$enable_static_libpython_for_interpreter" = "yes"; then
82+
+ LINK_PYTHON_OBJS='$(LIBRARY_OBJS)'
83+
+else
84+
+ LINK_PYTHON_OBJS='$(BLDLIBRARY)'
85+
+fi
86+
+AC_SUBST(LINK_PYTHON_OBJS)
87+
+
88+
AC_SUBST(AR)
89+
AC_CHECK_TOOLS(AR, ar aal, ar)
90+
91+
--
92+
2.39.5 (Apple Git-154)
93+

‎cpython-unix/patch-python-configure-add-enable-static-libpython-for-interpreter.patch‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
From 439f6e6bb62482a98fb6765d723cedea12f3b10f Mon Sep 17 00:00:00 2001
2+
From: Geoffrey Thomas <geofft@ldpreload.com>
3+
Date: Sat, 19 Apr 2025 11:13:40 -0400
4+
Subject: [PATCH 1/1] configure: add --enable-static-libpython-for-interpreter
5+
6+
This option changes the behavior of --enable-shared to continue to build
7+
the libpython3.x.so shared library, but not use it for linking the
8+
python3 interpreter executable. Instead, the executable is linked
9+
directly against the libpython .o files as it would be with
10+
--disable-shared.
11+
12+
There are two benefits of this change. First, libpython uses
13+
thread-local storage, which is noticeably slower when used in a loaded
14+
module instead of in the main program, because the main program can take
15+
advantage of constant offsets from the thread state pointer but loaded
16+
modules have to dynamically call a function __tls_get_addr() to
17+
potentially allocate their thread-local storage area. (There is another
18+
thread-local storage model for dynamic libraries which mitigates most of
19+
this performance hit, but it comes at the cost of preventing
20+
dlopen("libpython3.x.so"), which is a use case we want to preserve.)
21+
22+
Second, this improves the user experience around relocatable Python a
23+
little bit, in that we don't need to use an $ORIGIN-relative path to
24+
locate libpython3.x.so, which has some mild benefits around musl (which
25+
does not support $ORIGIN-relative DT_NEEDED, only $ORIGIN-relative
26+
DT_RPATH/DT_RUNPATH), users who want to make the interpreter setuid or
27+
setcap (which prevents processing $ORIGIN), etc.
28+
---
29+
configure.ac | 24 +++++++++++++++++++++---
30+
1 file changed, 21 insertions(+), 3 deletions(-)
31+
32+
diff --git a/configure.ac b/configure.ac
33+
index 004797b5233..a3a5ac1cdce 100644
34+
--- a/configure.ac
35+
+++ b/configure.ac
36+
@@ -1502,6 +1502,17 @@ fi],
37+
[AC_MSG_RESULT([yes])])
38+
AC_SUBST([STATIC_LIBPYTHON])
39+
40+
+AC_MSG_CHECKING([for --enable-static-libpython-for-interpreter])
41+
+AC_ARG_ENABLE([static-libpython-for-interpreter],
42+
+ AS_HELP_STRING([--enable-static-libpython-for-interpreter],
43+
+ [even with --enable-shared, statically link libpython into the interpreter (default is to use the shared library)]))
44+
+
45+
+if test -z "$enable_static_libpython_for_interpreter"
46+
+then
47+
+ enable_static_libpython_for_interpreter="no"
48+
+fi
49+
+AC_MSG_RESULT([$enable_static_libpython_for_interpreter])
50+
+
51+
AC_MSG_CHECKING([for --enable-profiling])
52+
AC_ARG_ENABLE([profiling],
53+
AS_HELP_STRING([--enable-profiling], [enable C-level code profiling with gprof (default is no)]))
54+
@@ -1660,7 +1671,11 @@ if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then
55+
LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS"
56+
fi
57+
# Link Python program to the shared library
58+
- LINK_PYTHON_OBJS='$(BLDLIBRARY)'
59+
+ if test "$enable_static_libpython_for_interpreter" = "yes"; then
60+
+ LINK_PYTHON_OBJS='$(LIBRARY_OBJS)'
61+
+ else
62+
+ LINK_PYTHON_OBJS='$(BLDLIBRARY)'
63+
+ fi
64+
else
65+
if test "$STATIC_LIBPYTHON" = 0; then
66+
# Build Python needs object files but don't need to build
67+
@@ -2166,11 +2181,14 @@ if test "$Py_BOLT" = 'true' ; then
68+
fi
69+
fi
70+
71+
-dnl Enable BOLT of libpython if built.
72+
+dnl Enable BOLT of libpython if built and used by the python3 binary.
73+
+dnl (If it is built but not used, we cannot profile it.)
74+
AC_SUBST([BOLT_BINARIES])
75+
BOLT_BINARIES='$(BUILDPYTHON)'
76+
AS_VAR_IF([enable_shared], [yes], [
77+
- BOLT_BINARIES="${BOLT_BINARIES} \$(INSTSONAME)"
78+
+ AS_VAR_IF([enable_static_libpython_for_interpreter], [no], [
79+
+ BOLT_BINARIES="${BOLT_BINARIES} \$(INSTSONAME)"
80+
+ ])
81+
])
82+
83+
AC_ARG_VAR(
84+
--
85+
2.39.5 (Apple Git-154)
86+

0 commit comments

Comments
 (0)