Merge branch 'main' into local-annos

Author: JelleZijlstra

.github/workflows/tail-call.yml

@@ -87,8 +87,8 @@ jobs:
           set PlatformToolset=clangcl
           set LLVMToolsVersion=${{ matrix.llvm }}.1.5
           set LLVMInstallDir=C:\Program Files\LLVM
-          ./PCbuild/build.bat --tail-call-interp -d -p ${{ matrix.architecture }}
-          ./PCbuild/rt.bat -d -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          call ./PCbuild/build.bat --tail-call-interp -d -p ${{ matrix.architecture }}
+          call ./PCbuild/rt.bat -d -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # No tests (yet):
       - name: Emulated Windows (release)

Doc/library/os.rst

@@ -3693,16 +3693,16 @@ features:
 
    This example displays the number of bytes taken by non-directory files in each
    directory under the starting directory, except that it doesn't look under any
-   CVS subdirectory::
+   ``__pycache__`` subdirectory::
 
       import os
       from os.path import join, getsize
-      for root, dirs, files in os.walk('python/Lib/email'):
+      for root, dirs, files in os.walk('python/Lib/xml'):
           print(root, "consumes", end=" ")
           print(sum(getsize(join(root, name)) for name in files), end=" ")
           print("bytes in", len(files), "non-directory files")
-          if 'CVS' in dirs:
-              dirs.remove('CVS')  # don't visit CVS directories
+          if '__pycache__' in dirs:
+              dirs.remove('__pycache__')  # don't visit __pycache__ directories
 
    In the next example (simple implementation of :func:`shutil.rmtree`),
    walking the tree bottom-up is essential, :func:`rmdir` doesn't allow
@@ -3755,16 +3755,16 @@ features:
 
    This example displays the number of bytes taken by non-directory files in each
    directory under the starting directory, except that it doesn't look under any
-   CVS subdirectory::
+   ``__pycache__`` subdirectory::
 
       import os
-      for root, dirs, files, rootfd in os.fwalk('python/Lib/email'):
+      for root, dirs, files, rootfd in os.fwalk('python/Lib/xml'):
           print(root, "consumes", end="")
           print(sum([os.stat(name, dir_fd=rootfd).st_size for name in files]),
                 end="")
           print("bytes in", len(files), "non-directory files")
-          if 'CVS' in dirs:
-              dirs.remove('CVS')  # don't visit CVS directories
+          if '__pycache__' in dirs:
+              dirs.remove('__pycache__')  # don't visit __pycache__ directories
 
    In the next example, walking the tree bottom-up is essential:
    :func:`rmdir` doesn't allow deleting a directory before the directory is

Include/internal/pycore_dict.h

@@ -150,6 +150,10 @@ extern int _PyDict_Pop_KnownHash(
     Py_hash_t hash,
     PyObject **result);
 
+#ifdef Py_GIL_DISABLED
+PyAPI_FUNC(void) _PyDict_EnsureSharedOnRead(PyDictObject *mp);
+#endif
+
 #define DKIX_EMPTY (-1)
 #define DKIX_DUMMY (-2)  /* Used internally */
 #define DKIX_ERROR (-3)

Lib/os.py

@@ -345,12 +345,12 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
 
     import os
     from os.path import join, getsize
-    for root, dirs, files in os.walk('python/Lib/email'):
+    for root, dirs, files in os.walk('python/Lib/xml'):
         print(root, "consumes ")
         print(sum(getsize(join(root, name)) for name in files), end=" ")
         print("bytes in", len(files), "non-directory files")
-        if 'CVS' in dirs:
-            dirs.remove('CVS')  # don't visit CVS directories
+        if '__pycache__' in dirs:
+            dirs.remove('__pycache__')  # don't visit __pycache__ directories
 
     """
     sys.audit("os.walk", top, topdown, onerror, followlinks)
@@ -460,13 +460,13 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd=
         Example:
 
         import os
-        for root, dirs, files, rootfd in os.fwalk('python/Lib/email'):
+        for root, dirs, files, rootfd in os.fwalk('python/Lib/xml'):
             print(root, "consumes", end="")
             print(sum(os.stat(name, dir_fd=rootfd).st_size for name in files),
                   end="")
             print("bytes in", len(files), "non-directory files")
-            if 'CVS' in dirs:
-                dirs.remove('CVS')  # don't visit CVS directories
+            if '__pycache__' in dirs:
+                dirs.remove('__pycache__')  # don't visit __pycache__ directories
         """
         sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd)
         top = fspath(top)

Lib/test/audit-tests.py

@@ -208,7 +208,15 @@ def test_open(testfn):
             if not fn:
                 continue
             with assertRaises(RuntimeError):
-                fn(*args)
+                try:
+                    fn(*args)
+                except NotImplementedError:
+                    if fn == load_dh_params:
+                        # Not callable in some builds
+                        load_dh_params = None
+                        raise RuntimeError
+                    else:
+                        raise
 
     actual_mode = [(a[0], a[1]) for e, a in hook.seen if e == "open" and a[1]]
     actual_flag = [(a[0], a[2]) for e, a in hook.seen if e == "open" and not a[1]]

Lib/test/libregrtest/tsan.py

@@ -2,7 +2,7 @@
 # chosen because they use threads and run in a reasonable amount of time.
 
 TSAN_TESTS = [
-    'test_asyncio.test_free_threading',
+    'test_asyncio',
     # TODO: enable more of test_capi once bugs are fixed (GH-116908, GH-116909).
     'test_capi.test_mem',
     'test_capi.test_pyatomic',

Lib/test/test_asyncio/test_events.py

@@ -2652,33 +2652,46 @@ def callback(*args):
 
         h1 = asyncio.TimerHandle(when, callback, (), self.loop)
         h2 = asyncio.TimerHandle(when, callback, (), self.loop)
-        # TODO: Use assertLess etc.
-        self.assertFalse(h1 < h2)
-        self.assertFalse(h2 < h1)
-        self.assertTrue(h1 <= h2)
-        self.assertTrue(h2 <= h1)
-        self.assertFalse(h1 > h2)
-        self.assertFalse(h2 > h1)
-        self.assertTrue(h1 >= h2)
-        self.assertTrue(h2 >= h1)
-        self.assertTrue(h1 == h2)
-        self.assertFalse(h1 != h2)
+        with self.assertRaises(AssertionError):
+            self.assertLess(h1, h2)
+        with self.assertRaises(AssertionError):
+            self.assertLess(h2, h1)
+        with self.assertRaises(AssertionError):
+            self.assertGreater(h1, h2)
+        with self.assertRaises(AssertionError):
+            self.assertGreater(h2, h1)
+        with self.assertRaises(AssertionError):
+            self.assertNotEqual(h1, h2)
+
+        self.assertLessEqual(h1, h2)
+        self.assertLessEqual(h2, h1)
+        self.assertGreaterEqual(h1, h2)
+        self.assertGreaterEqual(h2, h1)
+        self.assertEqual(h1, h2)
 
         h2.cancel()
-        self.assertFalse(h1 == h2)
+        with self.assertRaises(AssertionError):
+            self.assertEqual(h1, h2)
+        self.assertNotEqual(h1, h2)
 
         h1 = asyncio.TimerHandle(when, callback, (), self.loop)
         h2 = asyncio.TimerHandle(when + 10.0, callback, (), self.loop)
-        self.assertTrue(h1 < h2)
-        self.assertFalse(h2 < h1)
-        self.assertTrue(h1 <= h2)
-        self.assertFalse(h2 <= h1)
-        self.assertFalse(h1 > h2)
-        self.assertTrue(h2 > h1)
-        self.assertFalse(h1 >= h2)
-        self.assertTrue(h2 >= h1)
-        self.assertFalse(h1 == h2)
-        self.assertTrue(h1 != h2)
+        with self.assertRaises(AssertionError):
+            self.assertLess(h2, h1)
+        with self.assertRaises(AssertionError):
+            self.assertLessEqual(h2, h1)
+        with self.assertRaises(AssertionError):
+            self.assertGreater(h1, h2)
+        with self.assertRaises(AssertionError):
+            self.assertGreaterEqual(h1, h2)
+        with self.assertRaises(AssertionError):
+            self.assertEqual(h1, h2)
+
+        self.assertLess(h1, h2)
+        self.assertGreater(h2, h1)
+        self.assertLessEqual(h1, h2)
+        self.assertGreaterEqual(h2, h1)
+        self.assertNotEqual(h1, h2)
 
         h3 = asyncio.Handle(callback, (), self.loop)
         self.assertIs(NotImplemented, h1.__eq__(h3))
@@ -2692,19 +2705,25 @@ def callback(*args):
             h1 <= ()
         with self.assertRaises(TypeError):
             h1 >= ()
-        self.assertFalse(h1 == ())
-        self.assertTrue(h1 != ())
-
-        self.assertTrue(h1 == ALWAYS_EQ)
-        self.assertFalse(h1 != ALWAYS_EQ)
-        self.assertTrue(h1 < LARGEST)
-        self.assertFalse(h1 > LARGEST)
-        self.assertTrue(h1 <= LARGEST)
-        self.assertFalse(h1 >= LARGEST)
-        self.assertFalse(h1 < SMALLEST)
-        self.assertTrue(h1 > SMALLEST)
-        self.assertFalse(h1 <= SMALLEST)
-        self.assertTrue(h1 >= SMALLEST)
+        with self.assertRaises(AssertionError):
+            self.assertEqual(h1, ())
+        with self.assertRaises(AssertionError):
+            self.assertNotEqual(h1, ALWAYS_EQ)
+        with self.assertRaises(AssertionError):
+            self.assertGreater(h1, LARGEST)
+        with self.assertRaises(AssertionError):
+            self.assertGreaterEqual(h1, LARGEST)
+        with self.assertRaises(AssertionError):
+            self.assertLess(h1, SMALLEST)
+        with self.assertRaises(AssertionError):
+            self.assertLessEqual(h1, SMALLEST)
+
+        self.assertNotEqual(h1, ())
+        self.assertEqual(h1, ALWAYS_EQ)
+        self.assertLess(h1, LARGEST)
+        self.assertLessEqual(h1, LARGEST)
+        self.assertGreaterEqual(h1, SMALLEST)
+        self.assertGreater(h1, SMALLEST)
 
 
 class AbstractEventLoopTests(unittest.TestCase):

Lib/test/test_audit.py

@@ -23,6 +23,7 @@ def run_test_in_subprocess(self, *args):
         with subprocess.Popen(
             [sys.executable, "-X utf8", AUDIT_TESTS_PY, *args],
             encoding="utf-8",
+            errors="backslashreplace",
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         ) as p:

Lib/test/test_pyrepl/test_interact.py

@@ -58,7 +58,7 @@ def test_multiple_statements_fail_early(self):
         console = InteractiveColoredConsole()
         code = dedent("""\
         raise Exception('foobar')
-        print('spam&eggs')
+        print('spam', 'eggs', sep='&')
         """)
         f = io.StringIO()
         with contextlib.redirect_stderr(f):

Lib/test/test_ssl.py

@@ -1321,10 +1321,14 @@ def test_load_verify_cadata(self):
         with self.assertRaises(ssl.SSLError):
             ctx.load_verify_locations(cadata=cacert_der + b"A")
 
-    @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows")
     def test_load_dh_params(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
-        ctx.load_dh_params(DHFILE)
+        try:
+            ctx.load_dh_params(DHFILE)
+        except RuntimeError:
+            if Py_DEBUG_WIN32:
+                self.skipTest("not supported on Win32 debug build")
+            raise
         ctx.load_dh_params(BYTES_DHFILE)
         self.assertRaises(TypeError, ctx.load_dh_params)
         self.assertRaises(TypeError, ctx.load_dh_params, None)
@@ -1648,12 +1652,17 @@ def test_str(self):
         self.assertEqual(str(e), "foo")
         self.assertEqual(e.errno, 1)
 
-    @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows")
     def test_lib_reason(self):
         # Test the library and reason attributes
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-        with self.assertRaises(ssl.SSLError) as cm:
-            ctx.load_dh_params(CERTFILE)
+        try:
+            with self.assertRaises(ssl.SSLError) as cm:
+                ctx.load_dh_params(CERTFILE)
+        except RuntimeError:
+            if Py_DEBUG_WIN32:
+                self.skipTest("not supported on Win32 debug build")
+            raise
+
         self.assertEqual(cm.exception.library, 'PEM')
         regex = "(NO_START_LINE|UNSUPPORTED_PUBLIC_KEY_TYPE)"
         self.assertRegex(cm.exception.reason, regex)
@@ -2773,6 +2782,14 @@ def try_protocol_combo(server_protocol, client_protocol, expect_success,
                                  % (expect_success, stats['version']))
 
 
+def supports_kx_alias(ctx, aliases):
+    for cipher in ctx.get_ciphers():
+        for alias in aliases:
+            if f"Kx={alias}" in cipher['description']:
+                return True
+    return False
+
+
 class ThreadedTests(unittest.TestCase):
 
     @support.requires_resource('walltime')
@@ -4032,21 +4049,30 @@ def test_no_legacy_server_connect(self):
                                    chatty=True, connectionchatty=True,
                                    sni_name=hostname)
 
-    @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows")
     def test_dh_params(self):
-        # Check we can get a connection with ephemeral Diffie-Hellman
+        # Check we can get a connection with ephemeral finite-field
+        # Diffie-Hellman (if supported).
         client_context, server_context, hostname = testing_context()
+        dhe_aliases = {"ADH", "EDH", "DHE"}
+        if not (supports_kx_alias(client_context, dhe_aliases)
+                and supports_kx_alias(server_context, dhe_aliases)):
+            self.skipTest("libssl doesn't support ephemeral DH")
         # test scenario needs TLS <= 1.2
         client_context.maximum_version = ssl.TLSVersion.TLSv1_2
-        server_context.load_dh_params(DHFILE)
+        try:
+            server_context.load_dh_params(DHFILE)
+        except RuntimeError:
+            if Py_DEBUG_WIN32:
+                self.skipTest("not supported on Win32 debug build")
+            raise
         server_context.set_ciphers("kEDH")
         server_context.maximum_version = ssl.TLSVersion.TLSv1_2
         stats = server_params_test(client_context, server_context,
                                    chatty=True, connectionchatty=True,
                                    sni_name=hostname)
         cipher = stats["cipher"][0]
         parts = cipher.split("-")
-        if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts:
+        if not dhe_aliases.intersection(parts):
             self.fail("Non-DH key exchange: " + cipher[0])
 
     def test_ecdh_curve(self):
@@ -4819,14 +4845,18 @@ def keylog_lines(self, fname=os_helper.TESTFN):
             return len(list(f))
 
     @requires_keylog
-    @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows")
     def test_keylog_defaults(self):
         self.addCleanup(os_helper.unlink, os_helper.TESTFN)
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
         self.assertEqual(ctx.keylog_filename, None)
 
         self.assertFalse(os.path.isfile(os_helper.TESTFN))
-        ctx.keylog_filename = os_helper.TESTFN
+        try:
+            ctx.keylog_filename = os_helper.TESTFN
+        except RuntimeError:
+            if Py_DEBUG_WIN32:
+                self.skipTest("not supported on Win32 debug build")
+            raise
         self.assertEqual(ctx.keylog_filename, os_helper.TESTFN)
         self.assertTrue(os.path.isfile(os_helper.TESTFN))
         self.assertEqual(self.keylog_lines(), 1)
@@ -4843,12 +4873,17 @@ def test_keylog_defaults(self):
             ctx.keylog_filename = 1
 
     @requires_keylog
-    @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows")
     def test_keylog_filename(self):
         self.addCleanup(os_helper.unlink, os_helper.TESTFN)
         client_context, server_context, hostname = testing_context()
 
-        client_context.keylog_filename = os_helper.TESTFN
+        try:
+            client_context.keylog_filename = os_helper.TESTFN
+        except RuntimeError:
+            if Py_DEBUG_WIN32:
+                self.skipTest("not supported on Win32 debug build")
+            raise
+
         server = ThreadedEchoServer(context=server_context, chatty=False)
         with server:
             with client_context.wrap_socket(socket.socket(),
@@ -4881,7 +4916,6 @@ def test_keylog_filename(self):
     @requires_keylog
     @unittest.skipIf(sys.flags.ignore_environment,
                      "test is not compatible with ignore_environment")
-    @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows")
     def test_keylog_env(self):
         self.addCleanup(os_helper.unlink, os_helper.TESTFN)
         with unittest.mock.patch.dict(os.environ):
@@ -4891,7 +4925,12 @@ def test_keylog_env(self):
             ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
             self.assertEqual(ctx.keylog_filename, None)
 
-            ctx = ssl.create_default_context()
+            try:
+                ctx = ssl.create_default_context()
+            except RuntimeError:
+                if Py_DEBUG_WIN32:
+                    self.skipTest("not supported on Win32 debug build")
+                raise
             self.assertEqual(ctx.keylog_filename, os_helper.TESTFN)
 
             ctx = ssl._create_stdlib_context()