From c0c802dad7ab0a44a4ca3b09cda8a491685e9573 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 31 Aug 2026 21:15:06 +0300 Subject: [PATCH 1/3] gh-99064: Ignore the encoding declaration when parsing decoded text ElementTree.parse() with a text file mis-decoded the text in the C implementation: _parse_whole() encoded it as UTF-8, but left expat to honor the encoding declared in the document. It now overrides the encoding, as XMLParser.feed() already does for str data. --- Lib/test/test_xml_etree.py | 28 +++++++++++++++++++ ...6-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst | 5 ++++ Modules/_elementtree.c | 7 +++++ 3 files changed, 40 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 2af2d1fd64520b1..4f6820dcdc92daf 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1068,6 +1068,34 @@ def bxml(encoding, body=''): self.assertRaises(ValueError, ET.XML, xml('undefined').encode('ascii')) self.assertRaises(LookupError, ET.XML, xml('xxx').encode('ascii')) + def test_parse_text_source(self): + # gh-99064: The encoding declared in the document does not apply + # to a source which is already decoded. + def check(encoding, body): + xml = ("%s" % + (encoding, body)) + with self.subTest(encoding=encoding): + self.assertEqual(ET.parse(io.StringIO(xml)).getroot().text, + body) + # the same with an explicitly created parser + self.assertEqual( + ET.parse(io.StringIO(xml), ET.XMLParser()).getroot().text, + body) + check("ascii", 'a') + check("iso-8859-1", '\xbd') + check("iso-8859-15", '\u20ac') + check("cp437", '\u221a') + check("utf-8", '\u4e2d') + # not ASCII compatible, unsupported for a bytes source + check("utf-16", '\u4e2d') + check("utf-32", '\u4e2d') + + def test_parse_text_source_multiple_chunks(self): + # the encoding is overridden before the first chunk is parsed + body = '\xe4' * 100_000 + xml = "%s" % body + self.assertEqual(ET.parse(io.StringIO(xml)).getroot().text, body) + @support.subTests('sample,exception', [ (b' \xa1', UnicodeDecodeError), # crashed (b' \xa1state; + int first = 1; for (;;) { buffer = PyObject_CallFunction(reader, "i", 64*1024); @@ -4100,6 +4101,11 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject *self, Py_DECREF(buffer); break; } + if (first) { + /* The text is already decoded, the encoding declared in the + document does not apply to it. Return code ignored. */ + (void)EXPAT(st, SetEncoding)(self->parser, "utf-8"); + } temp = PyUnicode_AsEncodedString(buffer, "utf-8", "surrogatepass"); Py_DECREF(buffer); if (!temp) { @@ -4123,6 +4129,7 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject *self, res = expat_parse( st, self, PyBytes_AS_STRING(buffer), (int)PyBytes_GET_SIZE(buffer), 0); + first = 0; Py_DECREF(buffer); From dfbcdb42cd20f50d140267553ce92de19ad94807 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 31 Aug 2026 23:41:30 +0300 Subject: [PATCH 2/3] Apply suggestion from @vstinner Co-authored-by: Victor Stinner --- Lib/test/test_xml_etree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 4f6820dcdc92daf..dc7a4cd9782b1e8 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1072,8 +1072,8 @@ def test_parse_text_source(self): # gh-99064: The encoding declared in the document does not apply # to a source which is already decoded. def check(encoding, body): - xml = ("%s" % - (encoding, body)) + xml = (f"" + f"{body}") with self.subTest(encoding=encoding): self.assertEqual(ET.parse(io.StringIO(xml)).getroot().text, body) From 7b12b2015fc705c2e386f3211fb80bf6ddf5cbe6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 31 Aug 2026 23:51:47 +0300 Subject: [PATCH 3/3] Remove a trailing space --- Lib/test/test_xml_etree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index dc7a4cd9782b1e8..fb35bb6a5f442fa 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1072,7 +1072,7 @@ def test_parse_text_source(self): # gh-99064: The encoding declared in the document does not apply # to a source which is already decoded. def check(encoding, body): - xml = (f"" + xml = (f"" f"{body}") with self.subTest(encoding=encoding): self.assertEqual(ET.parse(io.StringIO(xml)).getroot().text,