bpo-45548: Some test modules must be built as shared libs (GH-29268)

Some test cases don't work when test modules are static extensions.

Add dependency on Modules/config.c to trigger a rebuild whenever a
module build type is changed.

``makesetup`` puts shared extensions into ``Modules/`` directory. Create
symlinks from pybuilddir so the extensions can be imported.

Note: It is not possible to use the content of pybuilddir.txt as a build
target. Makefile evaluates target variables in the first pass. The
pybuilddir.txt file does not exist at that point.

Author: tiran

Doc/whatsnew/3.11.rst

@@ -507,6 +507,13 @@ Build Changes
   except empty tuple singleton.
   (Contributed by Christian Heimes in :issue:`45522`)
 
+* ``Modules/Setup`` and ``Modules/makesetup`` have been improved and tied up.
+  Extension modules can now be built through ``makesetup``. All except some
+  test modules can be linked statically into main binary or library.
+  (Contributed by Brett Cannon and Christian Heimes in :issue:`45548`,
+  :issue:`45570`, :issue:`45571`, and :issue:`43974`.)
+
+
 C API Changes
 =============
 

Makefile.pre.in

@@ -705,8 +705,17 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
 	fi
 
 
-oldsharedmods: $(SHAREDMODS)
-
+# create relative links from build/lib.platform/egg.so to Modules/egg.so
+# pybuilddir.txt is created too late. We cannot use it in Makefile
+# targets. ln --relative is not portable.
+oldsharedmods: $(SHAREDMODS) pybuilddir.txt
+	@target=`cat pybuilddir.txt`; \
+	$(MKDIR_P) $$target; \
+	for mod in X $(SHAREDMODS); do \
+		if test $$mod != X; then \
+			$(LN) -sf ../../$$mod $$target/`basename $$mod`; \
+		fi; \
+	done
 
 Makefile Modules/config.c: Makefile.pre \
 				$(srcdir)/Modules/config.c.in \

Misc/NEWS.d/next/Build/2021-10-28-14-47-22.bpo-45548.mdCBxB.rst

@@ -0,0 +1,4 @@
+``Modules/Setup`` and ``Modules/makesetup`` have been improved. The
+``Setup`` file now contains working rules for all extensions. Outdated
+comments have been removed. Rules defined by ``makesetup`` track
+dependencies correctly.

Modules/Setup

@@ -297,14 +297,16 @@ xxsubtype xxsubtype.c  # Required for the test suite to pass!
 
 #_xxsubinterpreters _xxsubinterpretersmodule.c
 #_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
-#_ctypes_test _ctypes/_ctypes_test.c
 #_testbuffer _testbuffer.c
-#_testimportmultiple _testimportmultiple.c
 #_testinternalcapi _testinternalcapi.c
-#_testmultiphase _testmultiphase.c
+
+# Some testing modules MUST be built as shared libraries.
 
 #*shared*
-#_testcapi _testcapimodule.c  # CANNOT be statically compiled!
+#_ctypes_test _ctypes/_ctypes_test.c
+#_testcapi _testcapimodule.c
+#_testimportmultiple _testimportmultiple.c
+#_testmultiphase _testmultiphase.c
 
 # ---
 # Uncommenting the following line tells makesetup that all following modules

Modules/makesetup

@@ -241,7 +241,8 @@ sed -e 's/[ 	]*#.*//' -e '/^[ 	]*$/d' |
 				cc="$cc \$(PY_BUILTIN_MODULE_CFLAGS)";;
 			esac
 			mods_upper=$(echo $mods | tr '[a-z]' '[A-Z]')
-			rule="$obj: $src \$(MODULE_${mods_upper}_DEPS) \$(PYTHON_HEADERS); $cc $cpps -c $src -o $obj"
+			# force rebuild when header file or module build flavor (static/shared) is changed
+			rule="$obj: $src \$(MODULE_${mods_upper}_DEPS) \$(PYTHON_HEADERS) Modules/config.c; $cc $cpps -c $src -o $obj"
 			echo "$rule" >>$rulesf
 		done
 		case $doconfig in

setup.py

@@ -426,12 +426,13 @@ def update_sources_depends(self):
             # re-compile extensions if a header file has been changed
             ext.depends.extend(headers)
 
-    def remove_configured_extensions(self):
+    def handle_configured_extensions(self):
         # The sysconfig variables built by makesetup that list the already
         # built modules and the disabled modules as configured by the Setup
         # files.
-        sysconf_built = sysconfig.get_config_var('MODBUILT_NAMES').split()
-        sysconf_dis = sysconfig.get_config_var('MODDISABLED_NAMES').split()
+        sysconf_built = set(sysconfig.get_config_var('MODBUILT_NAMES').split())
+        sysconf_shared = set(sysconfig.get_config_var('MODSHARED_NAMES').split())
+        sysconf_dis = set(sysconfig.get_config_var('MODDISABLED_NAMES').split())
 
         mods_built = []
         mods_disabled = []
@@ -449,11 +450,15 @@ def remove_configured_extensions(self):
                                mods_configured]
             # Remove the shared libraries built by a previous build.
             for ext in mods_configured:
+                # Don't remove shared extensions which have been built
+                # by Modules/Setup
+                if ext.name in sysconf_shared:
+                    continue
                 fullpath = self.get_ext_fullpath(ext.name)
-                if os.path.exists(fullpath):
+                if os.path.lexists(fullpath):
                     os.unlink(fullpath)
 
-        return (mods_built, mods_disabled)
+        return mods_built, mods_disabled
 
     def set_compiler_executables(self):
         # When you run "make CC=altcc" or something similar, you really want
@@ -478,7 +483,7 @@ def build_extensions(self):
             self.remove_disabled()
 
         self.update_sources_depends()
-        mods_built, mods_disabled = self.remove_configured_extensions()
+        mods_built, mods_disabled = self.handle_configured_extensions()
         self.set_compiler_executables()
 
         if LIST_MODULE_NAMES: