Merge branch 'main' into gh-144986-fix-atexit-memleak

Author: ashm-dev

Include/internal/pycore_critical_section.h

@@ -50,6 +50,15 @@ extern "C" {
 // Asserts that the mutex for the given object is locked. The mutex must
 // be held by the top-most critical section otherwise there's the
 // possibility that the mutex would be swalled out in some code paths.
+//
+// NOTE: We use Py_REFCNT(op) != 1 instead of
+// !PyUnstable_Object_IsUniquelyReferenced(op) because the free threading
+// GC can change an object's ob_tid (it overwrites ob_tid and later
+// restores it from the mimalloc segment).  This means
+// PyUnstable_Object_IsUniquelyReferenced() may spuriously return false
+// after a GC collection, even though the thread may still have exclusive
+// access to the object.  The refcount check is a looser but still catches
+// most misuses.
 #ifdef Py_DEBUG
 
 # define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op)                           \

Include/internal/pycore_interp_structs.h

@@ -496,7 +496,7 @@ struct _py_func_state {
 
 /* For now we hard-code this to a value for which we are confident
    all the static builtin types will fit (for all builds). */
-#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 200
+#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 201
 #define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10
 #define _Py_MAX_MANAGED_STATIC_TYPES \
     (_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES + _Py_MAX_MANAGED_STATIC_EXT_TYPES)

InternalDocs/garbage_collector.md

@@ -153,7 +153,11 @@ pointer-sized (that is, eight bytes on a 64-bit platform).
 
 The garbage collector also temporarily repurposes the `ob_tid` (thread ID)
 and `ob_ref_local` (local reference count) fields for other purposes during
-collections.
+collections.  The `ob_tid` field is later restored from the containing
+mimalloc segment data structure.  In some cases, such as when the original
+allocating thread exits, this can result in a different `ob_tid` value.
+Code should not rely on `ob_tid` being stable across operations that may
+trigger garbage collection.
 
 
 C APIs

Lib/dataclasses.py

@@ -191,8 +191,8 @@ class _KW_ONLY_TYPE:
 KW_ONLY = _KW_ONLY_TYPE()
 
 # Since most per-field metadata will be unused, create an empty
-# read-only proxy that can be shared among all fields.
-_EMPTY_METADATA = types.MappingProxyType({})
+# read-only dictionary that can be shared among all fields.
+_EMPTY_METADATA = frozendict()
 
 # Markers for the various kinds of fields and pseudo-fields.
 class _FIELD_BASE:

Lib/email/headerregistry.py

@@ -3,8 +3,6 @@
 This module provides an implementation of the HeaderRegistry API.
 The implementation is designed to flexibly follow RFC5322 rules.
 """
-from types import MappingProxyType
-
 from email import utils
 from email import errors
 from email import _header_value_parser as parser
@@ -462,7 +460,7 @@ def init(self, *args, **kw):
 
     @property
     def params(self):
-        return MappingProxyType(self._params)
+        return frozendict(self._params)
 
 
 class ContentTypeHeader(ParameterizedMIMEHeader):

Lib/test/pickletester.py

@@ -1484,6 +1484,29 @@ def __hash__(self):
         # bad hashable dict key
         self.check_unpickling_error(CustomError, base + b'}c__main__\nBadKey1\n)\x81Nsb.')
 
+    def test_bad_types(self):
+        # APPEND
+        self.assertEqual(self.loads(b']Na.'), [None])
+        self.check_unpickling_error(AttributeError, b'NNa.')  # non-list
+        # APPENDS
+        self.assertEqual(self.loads(b'](Ne.'), [None])
+        self.check_unpickling_error(AttributeError, b'N(Ne.')  # non-list
+        self.check_unpickling_error(AttributeError, b'N(e.')
+        # SETITEM
+        self.assertEqual(self.loads(b'}NNs.'), {None: None})
+        self.check_unpickling_error(TypeError, b'NNNs.')  # non-dict
+        self.check_unpickling_error(TypeError, b'}]Ns.')  # non-hashable key
+        # SETITEMS
+        self.assertEqual(self.loads(b'}(NNu.'), {None: None})
+        self.check_unpickling_error(TypeError, b'N(NNu.')  # non-dict
+        self.assertEqual(self.loads(b'N(u.'), None)  # no validation for empty items
+        self.check_unpickling_error(TypeError, b'}(]Nu.')  # non-hashable key
+        # ADDITEMS
+        self.assertEqual(self.loads(b'\x8f(N\x90.'), {None})
+        self.check_unpickling_error(AttributeError, b'N(N\x90.')  # non-set
+        self.check_unpickling_error(AttributeError, b'N(\x90.')
+        self.check_unpickling_error(TypeError, b'\x8f(]\x90.')  # non-hashable element
+
     def test_bad_stack(self):
         badpickles = [
             b'.',                       # STOP

Lib/test/support/hashlib_helper.py

@@ -6,7 +6,6 @@
 import unittest
 import unittest.mock
 from test.support import import_helper
-from types import MappingProxyType
 
 
 def _parse_fullname(fullname, *, strict=False):
@@ -351,7 +350,7 @@ def __init__(
         )
 
 
-_HASHINFO_DATABASE = MappingProxyType({
+_HASHINFO_DATABASE = frozendict({
     _HashId.md5: _HashInfo(
         _HashId.md5,
         "_md5.MD5Type",
@@ -500,7 +499,7 @@ def _iter_hash_func_info(excluded):
 # keyed hash function. However, as it's exposed by HACL*, we test it.
 _HMACINFO_DATABASE[_HashId.blake2s] = _HashInfoItem('_hmac.compute_blake2s_32')
 _HMACINFO_DATABASE[_HashId.blake2b] = _HashInfoItem('_hmac.compute_blake2b_32')
-_HMACINFO_DATABASE = MappingProxyType(_HMACINFO_DATABASE)
+_HMACINFO_DATABASE = frozendict(_HMACINFO_DATABASE)
 assert _HMACINFO_DATABASE.keys() == CANONICAL_DIGEST_NAMES
 
 

Lib/test/test_dataclasses/__init__.py

@@ -71,7 +71,7 @@ def test_field_repr(self):
         expected_output = "Field(name='id',type=None," \
                            f"default=1,default_factory={MISSING!r}," \
                            "init=True,repr=False,hash=None," \
-                           "compare=True,metadata=mappingproxy({})," \
+                           "compare=True,metadata=frozendict()," \
                            f"kw_only={MISSING!r}," \
                            "doc='Docstring'," \
                            "_field_type=None)"

Lib/test/test_dict.py

@@ -1787,6 +1787,44 @@ def test_hash(self):
         with self.assertRaisesRegex(TypeError, "unhashable type: 'list'"):
             hash(fd)
 
+    def test_fromkeys(self):
+        self.assertEqual(frozendict.fromkeys('abc'),
+                         frozendict(a=None, b=None, c=None))
+
+        # Subclass which overrides the constructor
+        created = frozendict(x=1)
+        class FrozenDictSubclass(frozendict):
+            def __new__(self):
+                return created
+
+        fd = FrozenDictSubclass.fromkeys("abc")
+        self.assertEqual(fd, frozendict(x=1, a=None, b=None, c=None))
+        self.assertEqual(type(fd), FrozenDictSubclass)
+        self.assertEqual(created, frozendict(x=1))
+
+        fd = FrozenDictSubclass.fromkeys(frozendict(y=2))
+        self.assertEqual(fd, frozendict(x=1, y=None))
+        self.assertEqual(type(fd), FrozenDictSubclass)
+        self.assertEqual(created, frozendict(x=1))
+
+        # Subclass which doesn't override the constructor
+        class FrozenDictSubclass2(frozendict):
+            pass
+
+        fd = FrozenDictSubclass2.fromkeys("abc")
+        self.assertEqual(fd, frozendict(a=None, b=None, c=None))
+        self.assertEqual(type(fd), FrozenDictSubclass2)
+
+        # Dict subclass which overrides the constructor
+        class DictSubclass(dict):
+            def __new__(self):
+                return created
+
+        fd = DictSubclass.fromkeys("abc")
+        self.assertEqual(fd, frozendict(x=1, a=None, b=None, c=None))
+        self.assertEqual(type(fd), DictSubclass)
+        self.assertEqual(created, frozendict(x=1))
+
 
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_hmac.py

@@ -21,7 +21,6 @@
 import hmac
 import hashlib
 import random
-import types
 import unittest
 import warnings
 from _operator import _compare_digest as operator_compare_digest
@@ -303,7 +302,7 @@ def assert_hmac_new_by_name(
 
     def check_hmac_new(
         self, key, msg, hexdigest, hashname, digest_size, block_size,
-        hmac_new_func, hmac_new_kwds=types.MappingProxyType({}),
+        hmac_new_func, hmac_new_kwds=frozendict(),
     ):
         """Check that HMAC(key, msg) == digest.
 
@@ -349,7 +348,7 @@ def assert_hmac_hexdigest_by_name(
 
     def check_hmac_hexdigest(
         self, key, msg, hexdigest, digest_size,
-        hmac_digest_func, hmac_digest_kwds=types.MappingProxyType({}),
+        hmac_digest_func, hmac_digest_kwds=frozendict(),
     ):
         """Check and return a HMAC digest computed by hmac_digest_func().