[3.14] gh-83895: Accept input larger than 2 GiB in the C implementation of ElementTree (GH-156746) (GH-156772)

XMLParser.feed() and ElementTree.parse() raised OverflowError, because Expat
takes the length as an int.  The data is now fed to Expat in chunks of 1 MiB,
as xml.parsers.expat does since bpo-17089, so the pure Python implementation
already accepted such input.
(cherry picked from commit 9259e8b)

Author: serhiy-storchaka

Lib/test/test_xml_etree.py

@@ -1052,6 +1052,41 @@ def test_parse_text_source_multiple_chunks(self):
         xml = "<?xml version='1.0' encoding='ISO-8859-1'?><xml>%s</xml>" % body
         self.assertEqual(ET.parse(io.StringIO(xml)).getroot().text, body)
 
+    def test_parse_input_larger_than_chunk(self):
+        # gh-83895: the C implementation feeds Expat in chunks of 1 MiB
+        size = 3 * (1 << 20)
+        xml = '<r><a>%s</a><b/></r>' % ('x' * size)
+        for source in xml, xml.encode():
+            with self.subTest(type=type(source).__name__):
+                root = ET.fromstring(source)
+                self.assertEqual(len(root[0].text), size)
+                self.assertEqual(root[1].tag, 'b')
+
+    # gh-83895: input larger than INT_MAX is fed to Expat in chunks.
+    # memuse is 3 for the Python implementation, which joins the collected
+    # data, 2 would be enough for the C implementation.
+    @support.bigmemtest(size=support._2G + 100, memuse=3, dry_run=False)
+    def test_large_input(self, size):
+        data = b'<r>' + b'x' * size + b'</r>'
+        root = None
+        try:
+            parser = ET.XMLParser()
+            parser.feed(data)
+            data = None
+            root = parser.close()
+            self.assertEqual(len(root.text), size)
+        finally:
+            data = None
+            root = None
+
+    def test_parse_error_after_chunk_boundary(self):
+        # the reported position accounts for the preceding chunks
+        size = 2 * (1 << 20)
+        with self.assertRaises(ET.ParseError) as cm:
+            ET.fromstring('<r>%s<</r>' % ('x' * size))
+        self.assertEqual(cm.exception.position, (1, size + 4))
+
+
     def test_methods(self):
         # Test serialization methods.
 

Lib/test/test_xml_etree_c.py

@@ -15,16 +15,6 @@
 
 @unittest.skipUnless(cET, 'requires _elementtree')
 class MiscTests(unittest.TestCase):
-    # Issue #8651.
-    @support.bigmemtest(size=support._2G + 100, memuse=1, dry_run=False)
-    def test_length_overflow(self, size):
-        data = b'x' * size
-        parser = cET.XMLParser()
-        try:
-            self.assertRaises(OverflowError, parser.feed, data)
-        finally:
-            data = None
-
     def test_del_attribute(self):
         element = cET.Element('tag')
 

Misc/NEWS.d/next/Library/2026-08-31-23-10-00.gh-issue-83895.Jm5tR3.rst

@@ -0,0 +1,4 @@
+:mod:`xml.etree.ElementTree` now accepts input larger than 2 GiB
+in the C implementation.
+The data is fed to Expat in chunks, as :mod:`xml.parsers.expat` already did,
+instead of raising :exc:`OverflowError`.

Modules/_elementtree.c

@@ -3928,6 +3928,27 @@ expat_parse(elementtreestate *st, XMLParserObject *self, const char *data,
     Py_RETURN_NONE;
 }
 
+/* Expat takes the length as an int, feed larger data in chunks. */
+#define MAX_CHUNK_SIZE (1 << 20)
+
+LOCAL(PyObject*)
+expat_parse_large(elementtreestate *st, XMLParserObject *self,
+                  const char *data, Py_ssize_t data_len, int final)
+{
+    static_assert(MAX_CHUNK_SIZE <= INT_MAX,
+                  "MAX_CHUNK_SIZE is larger than INT_MAX");
+    while (data_len > MAX_CHUNK_SIZE) {
+        PyObject *res = expat_parse(st, self, data, MAX_CHUNK_SIZE, 0);
+        if (res == NULL) {
+            return NULL;
+        }
+        Py_DECREF(res);
+        data += MAX_CHUNK_SIZE;
+        data_len -= MAX_CHUNK_SIZE;
+    }
+    return expat_parse(st, self, data, (int)data_len, final);
+}
+
 /*[clinic input]
 _elementtree.XMLParser.close
 
@@ -4019,26 +4040,17 @@ _elementtree_XMLParser_feed_impl(XMLParserObject *self, PyObject *data)
         const char *data_ptr = PyUnicode_AsUTF8AndSize(data, &data_len);
         if (data_ptr == NULL)
             return NULL;
-        if (data_len > INT_MAX) {
-            PyErr_SetString(PyExc_OverflowError, "size does not fit in an int");
-            return NULL;
-        }
         /* Explicitly set UTF-8 encoding. Return code ignored. */
         (void)EXPAT(st, SetEncoding)(self->parser, "utf-8");
 
-        return expat_parse(st, self, data_ptr, (int)data_len, 0);
+        return expat_parse_large(st, self, data_ptr, data_len, 0);
     }
     else {
         Py_buffer view;
         PyObject *res;
         if (PyObject_GetBuffer(data, &view, PyBUF_SIMPLE) < 0)
             return NULL;
-        if (view.len > INT_MAX) {
-            PyBuffer_Release(&view);
-            PyErr_SetString(PyExc_OverflowError, "size does not fit in an int");
-            return NULL;
-        }
-        res = expat_parse(st, self, view.buf, (int)view.len, 0);
+        res = expat_parse_large(st, self, view.buf, view.len, 0);
         PyBuffer_Release(&view);
         return res;
     }
@@ -4108,14 +4120,8 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject *self,
             break;
         }
 
-        if (PyBytes_GET_SIZE(buffer) > INT_MAX) {
-            Py_DECREF(buffer);
-            Py_DECREF(reader);
-            PyErr_SetString(PyExc_OverflowError, "size does not fit in an int");
-            return NULL;
-        }
-        res = expat_parse(
-            st, self, PyBytes_AS_STRING(buffer), (int)PyBytes_GET_SIZE(buffer),
+        res = expat_parse_large(
+            st, self, PyBytes_AS_STRING(buffer), PyBytes_GET_SIZE(buffer),
             0);
         first = 0;