gh-42400: Fix buffer overflow in _Py_wrealpath for long paths

Author: ashm-dev

Doc/library/xml.etree.elementtree.rst

@@ -656,6 +656,10 @@ Functions
    .. versionchanged:: 3.13
       Added the :meth:`!close` method.
 
+   .. versionchanged:: next
+      A :exc:`ResourceWarning` is now emitted if the iterator opened a file
+      and is not explicitly closed.
+
 
 .. function:: parse(source, parser=None)
 

Doc/whatsnew/3.15.rst

@@ -1244,3 +1244,9 @@ that may require changes to your code.
 
 * :meth:`~mmap.mmap.resize` has been removed on platforms that don't support the
   underlying syscall, instead of raising a :exc:`SystemError`.
+
+* Resource warning is now emitted for unclosed
+  :func:`xml.etree.ElementTree.iterparse` iterator if it opened a file.
+  Use its :meth:`!close` method or the :func:`contextlib.closing` context
+  manager to close it.
+  (Contributed by Osama Abdelkader and Serhiy Storchaka in :gh:`140601`.)

Lib/test/test_fileutils.py

@@ -1,11 +1,9 @@
-# Run tests for functions in Python/fileutils.c.
-
 import os
 import os.path
 import unittest
 import tempfile
 import shutil
-from test.support import import_helper
+from test.support import import_helper, os_helper
 
 # Skip this test if the _testcapi module isn't available.
 _testcapi = import_helper.import_module('_testinternalcapi')
@@ -29,86 +27,79 @@ def test_capi_normalize_path(self):
 
 
 class RealpathTests(unittest.TestCase):
-    """Tests for _Py_wrealpath used by os.path.realpath"""
-
-    def test_realpath_long_path(self):
-        """Test that realpath handles paths longer than MAXPATHLEN (4096)"""
-        if os.name == 'nt':
-            raise unittest.SkipTest('POSIX-specific test')
+    """Tests for _Py_wrealpath used by os.path.realpath."""
 
-        base = tempfile.mkdtemp()
-        original_cwd = os.getcwd()
-        try:
-            os.chdir(base)
+    def setUp(self):
+        self.base = None
 
-            for i in range(85):
-                dirname = f"d{i:03d}_" + "x" * 44
-                os.mkdir(dirname)
-                os.chdir(dirname)
+    def tearDown(self):
+        if self.base and os.path.exists(self.base):
+            shutil.rmtree(self.base, ignore_errors=True)
 
-            full_path = os.getcwd()
+    def test_realpath_long_path(self):
+        """Test that realpath handles paths longer than MAXPATHLEN."""
+        if os.name == 'nt':
+            self.skipTest('POSIX-specific test')
 
-            self.assertGreater(len(full_path), 4096,
-                              f"Path should exceed MAXPATHLEN, got {len(full_path)}")
+        self.base = tempfile.mkdtemp()
+        current_path = self.base
 
-            # Main test: realpath should not crash on long paths
-            result = os.path.realpath(full_path)
+        for i in range(25):
+            dirname = f"d{i:03d}_" + "x" * 195
+            current_path = os.path.join(current_path, dirname)
+            try:
+                os.mkdir(current_path)
+            except OSError as e:
+                self.skipTest(f"Cannot create long paths on this platform: {e}")
 
-            self.assertTrue(os.path.isabs(result))
-            self.assertGreater(len(result), 4096)
-            # Note: os.path.exists() may fail on very long paths
-            # The important thing is realpath() doesn't crash
+        full_path = os.path.abspath(current_path)
+        if len(full_path) <= 4096:
+            self.skipTest(f"Path not long enough ({len(full_path)} bytes)")
 
-        finally:
-            os.chdir(original_cwd)
-            shutil.rmtree(base, ignore_errors=True)
+        result = os.path.realpath(full_path)
+        self.assertTrue(os.path.isabs(result))
+        self.assertGreater(len(result), 4096)
 
     def test_realpath_nonexistent_with_strict(self):
-        """Test that realpath with strict=True raises for nonexistent paths"""
+        """Test that realpath with strict=True raises for nonexistent paths."""
         if os.name == 'nt':
-            raise unittest.SkipTest('POSIX-specific test')
-
-        base = tempfile.mkdtemp()
-        try:
-            nonexistent = os.path.join(base, "does_not_exist", "subdir")
+            self.skipTest('POSIX-specific test')
 
-            # Without strict, should return the path
-            result = os.path.realpath(nonexistent, strict=False)
-            self.assertIsNotNone(result)
+        self.base = tempfile.mkdtemp()
+        nonexistent = os.path.join(self.base, "does_not_exist", "subdir")
 
-            # With strict=True, should raise an error
-            with self.assertRaises(OSError):
-                os.path.realpath(nonexistent, strict=True)
+        result = os.path.realpath(nonexistent, strict=False)
+        self.assertIsNotNone(result)
 
-        finally:
-            shutil.rmtree(base, ignore_errors=True)
+        with self.assertRaises(OSError):
+            os.path.realpath(nonexistent, strict=True)
 
+    @os_helper.skip_unless_symlink
     def test_realpath_symlink_long_path(self):
-        """Test realpath with symlinks in long paths"""
+        """Test realpath with symlinks in long paths."""
         if os.name == 'nt':
-            raise unittest.SkipTest('POSIX-specific test')
+            self.skipTest('POSIX-specific test')
 
-        base = tempfile.mkdtemp()
-        try:
-            # Create a long path
-            current = base
-            for i in range(30):
-                dirname = f"d{i:03d}_" + "x" * 44
-                current = os.path.join(current, dirname)
-                os.mkdir(current)
-
-            # Create a symlink pointing to the long path
-            symlink = os.path.join(base, "link")
-            os.symlink(current, symlink)
+        self.base = tempfile.mkdtemp()
+        current_path = self.base
 
-            # Resolve the symlink
-            result = os.path.realpath(symlink)
+        for i in range(15):
+            dirname = f"d{i:03d}_" + "x" * 195
+            current_path = os.path.join(current_path, dirname)
+            try:
+                os.mkdir(current_path)
+            except OSError as e:
+                self.skipTest(f"Cannot create long paths on this platform: {e}")
 
-            self.assertEqual(os.path.normpath(result), os.path.normpath(current))
-            self.assertGreater(len(result), 1500)
+        symlink = os.path.join(self.base, "link")
+        try:
+            os.symlink(current_path, symlink)
+        except (OSError, NotImplementedError) as e:
+            self.skipTest(f"Cannot create symlinks on this platform: {e}")
 
-        finally:
-            shutil.rmtree(base, ignore_errors=True)
+        result = os.path.realpath(symlink)
+        self.assertEqual(os.path.normpath(result), os.path.normpath(current_path))
+        self.assertGreater(len(result), 1500)
 
 
 if __name__ == "__main__":

Lib/test/test_xml_etree.py

@@ -1436,18 +1436,40 @@ def test_nonexistent_file(self):
 
     def test_resource_warnings_not_exhausted(self):
         # Not exhausting the iterator still closes the underlying file (bpo-43292)
+        # Not closing before del should emit ResourceWarning
         it = ET.iterparse(SIMPLE_XMLFILE)
         with warnings_helper.check_no_resource_warning(self):
+            it.close()
+            del it
+            gc_collect()
+
+        it = ET.iterparse(SIMPLE_XMLFILE)
+        with self.assertWarns(ResourceWarning) as wm:
             del it
             gc_collect()
+        # Not 'unclosed file'.
+        self.assertIn('unclosed iterparse iterator', str(wm.warning))
+        self.assertIn(repr(SIMPLE_XMLFILE), str(wm.warning))
+        self.assertEqual(wm.filename, __file__)
 
         it = ET.iterparse(SIMPLE_XMLFILE)
         with warnings_helper.check_no_resource_warning(self):
             action, elem = next(it)
+            it.close()
             self.assertEqual((action, elem.tag), ('end', 'element'))
             del it, elem
             gc_collect()
 
+        it = ET.iterparse(SIMPLE_XMLFILE)
+        with self.assertWarns(ResourceWarning) as wm:
+            action, elem = next(it)
+            self.assertEqual((action, elem.tag), ('end', 'element'))
+            del it, elem
+            gc_collect()
+        self.assertIn('unclosed iterparse iterator', str(wm.warning))
+        self.assertIn(repr(SIMPLE_XMLFILE), str(wm.warning))
+        self.assertEqual(wm.filename, __file__)
+
     def test_resource_warnings_failed_iteration(self):
         self.addCleanup(os_helper.unlink, TESTFN)
         with open(TESTFN, "wb") as f:
@@ -1461,15 +1483,40 @@ def test_resource_warnings_failed_iteration(self):
                 next(it)
             self.assertEqual(str(cm.exception),
                     'junk after document element: line 1, column 12')
+            it.close()
             del cm, it
             gc_collect()
 
+        it = ET.iterparse(TESTFN)
+        action, elem = next(it)
+        self.assertEqual((action, elem.tag), ('end', 'document'))
+        with self.assertWarns(ResourceWarning) as wm:
+            with self.assertRaises(ET.ParseError) as cm:
+                next(it)
+            self.assertEqual(str(cm.exception),
+                    'junk after document element: line 1, column 12')
+            del cm, it
+            gc_collect()
+        self.assertIn('unclosed iterparse iterator', str(wm.warning))
+        self.assertIn(repr(TESTFN), str(wm.warning))
+        self.assertEqual(wm.filename, __file__)
+
     def test_resource_warnings_exhausted(self):
         it = ET.iterparse(SIMPLE_XMLFILE)
         with warnings_helper.check_no_resource_warning(self):
+            list(it)
+            it.close()
+            del it
+            gc_collect()
+
+        it = ET.iterparse(SIMPLE_XMLFILE)
+        with self.assertWarns(ResourceWarning) as wm:
             list(it)
             del it
             gc_collect()
+        self.assertIn('unclosed iterparse iterator', str(wm.warning))
+        self.assertIn(repr(SIMPLE_XMLFILE), str(wm.warning))
+        self.assertEqual(wm.filename, __file__)
 
     def test_close_not_exhausted(self):
         iterparse = ET.iterparse

Lib/xml/etree/ElementTree.py

@@ -1261,16 +1261,20 @@ def iterator(source):
     gen = iterator(source)
     class IterParseIterator(collections.abc.Iterator):
         __next__ = gen.__next__
+
         def close(self):
+            nonlocal close_source
             if close_source:
                 source.close()
+                close_source = False
             gen.close()
 
-        def __del__(self):
-            # TODO: Emit a ResourceWarning if it was not explicitly closed.
-            # (When the close() method will be supported in all maintained Python versions.)
+        def __del__(self, _warn=warnings.warn):
             if close_source:
-                source.close()
+                try:
+                    _warn(f"unclosed iterparse iterator {source.name!r}", ResourceWarning, stacklevel=2)
+                finally:
+                    source.close()
 
     it = IterParseIterator()
     it.root = None

Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst

@@ -0,0 +1,4 @@
+:func:`xml.etree.ElementTree.iterparse` now emits a :exc:`ResourceWarning`
+when the iterator is not explicitly closed and was opened with a filename.
+This helps developers identify and fix resource leaks. Patch by Osama
+Abdelkader.