gh-87135: threading.Lock: Raise rather than hang on Python finalization

After Python finalization gets to the point where no other thread
can attach thread state, attempting to acquire a Python lock must
hang.
Raise PythonFinalizationError instead of hanging.

Author: encukou

Include/internal/pycore_lock.h

@@ -51,6 +51,11 @@ typedef enum _PyLockFlags {
 
     // Fail if interrupted by a signal while waiting on the lock.
     _PY_FAIL_IF_INTERRUPTED = 4,
+
+    // Locking & unlocking this lock requires attached thread state.
+    // If locking returns PY_LOCK_FAILURE, a Python exception *may* be raised.
+    // Implies _PY_LOCK_HANDLE_SIGNALS and _PY_LOCK_DETACH.
+    _PY_LOCK_PYTHONLOCK = 8 | 2 | 1,
 } _PyLockFlags;
 
 // Lock a mutex with an optional timeout and additional options. See

Lib/test/test_threading.py

@@ -1247,6 +1247,51 @@ def __del__(self):
         self.assertEqual(err, b"")
         self.assertIn(b"all clear", out)
 
+    def test_acquire_daemon_thread_lock_in_finalization(self):
+        # gh-123940: Py_Finalize() prevents other threads from running Python
+        # code (and so, releasing locks), so acquiring a locked lock can not
+        # succeed.
+        # We raise an exception rather than hang.
+        for timeout in (None, 10):
+            with self.subTest(timeout=timeout):
+                code = textwrap.dedent(f"""
+                    import threading
+                    import time
+
+                    thread_started_event = threading.Event()
+
+                    lock = threading.Lock()
+                    def loop():
+                        with lock:
+                            thread_started_event.set()
+                            while True:
+                                time.sleep(1)
+
+                    class Cycle:
+                        def __init__(self):
+                            self.self_ref = self
+                            self.thr = threading.Thread(
+                                target=loop, daemon=True)
+                            self.thr.start()
+                            thread_started_event.wait()
+
+                        def __del__(self):
+                            assert self.thr.is_alive()
+                            try:
+                                lock.acquire()
+                            except PythonFinalizationError:
+                                assert self.thr.is_alive()
+                                print('got the correct exception!')
+
+                    # Cycle holds a reference to itself, which ensures it is
+                    # cleaned up during the GC that runs after daemon threads
+                    # have been forced to exit during finalization.
+                    Cycle()
+                """)
+                rc, out, err = assert_python_ok("-c", code)
+                self.assertEqual(err, b"")
+                self.assertIn(b"got the correct exception", out)
+
     def test_start_new_thread_failed(self):
         # gh-109746: if Python fails to start newly created thread
         # due to failure of underlying PyThread_start_new_thread() call,

Modules/_threadmodule.c

@@ -835,8 +835,12 @@ lock_PyThread_acquire_lock(PyObject *op, PyObject *args, PyObject *kwds)
     }
 
     PyLockStatus r = _PyMutex_LockTimed(&self->lock, timeout,
-                                        _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH);
+                                        _PY_LOCK_PYTHONLOCK);
     if (r == PY_LOCK_INTR) {
+        assert(PyErr_Occurred());
+        return NULL;
+    }
+    if (r == PY_LOCK_FAILURE && PyErr_Occurred()) {
         return NULL;
     }
 
@@ -1055,8 +1059,12 @@ rlock_acquire(PyObject *op, PyObject *args, PyObject *kwds)
     }
 
     PyLockStatus r = _PyRecursiveMutex_LockTimed(&self->lock, timeout,
-                                                 _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH);
+                                                 _PY_LOCK_PYTHONLOCK);
     if (r == PY_LOCK_INTR) {
+        assert(PyErr_Occurred());
+        return NULL;
+    }
+    if (r == PY_LOCK_FAILURE && PyErr_Occurred()) {
         return NULL;
     }
 

Python/lock.c

@@ -95,6 +95,18 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags)
         if (timeout == 0) {
             return PY_LOCK_FAILURE;
         }
+        if ((flags & _PY_LOCK_PYTHONLOCK) && Py_IsFinalizing()) {
+            // At this phase of runtime shutdown, only the finalization thread
+            // can have attached thread state; others hang if they try
+            // attaching. And since operations on this lock requires attached
+            // thread state (_PY_LOCK_PYTHONLOCK), the finalization thread is
+            // running this code, and no other thread can unlock.
+            // Raise rather than hang. (_PY_LOCK_PYTHONLOCK allows raising
+            // exceptons.)
+            PyErr_SetString(PyExc_PythonFinalizationError,
+                            "cannot acquire lock at interpreter finalization");
+            return PY_LOCK_FAILURE;
+        }
 
         uint8_t newv = v;
         if (!(v & _Py_HAS_PARKED)) {