diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 310ccd651e18c7..2e68f9fd31b6f0 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -1135,6 +1135,34 @@ ElementTree Objects of the XML *file* if given. + .. attribute:: children + + A sequence of the children of the document: + the root element and the comments and processing instructions + which surround it. + It can contain at most one element, + which is the root element of the tree; + adding a second one raises :exc:`ValueError`, + and adding anything which is not an element raises :exc:`TypeError`. + + It supports ``len()``, iteration, the :keyword:`in` operator, + :func:`reversed`, indexing and slicing (for getting, setting and + deleting), and the methods :meth:`!append`, :meth:`!insert`, + :meth:`!extend`, :meth:`!remove` and :meth:`!clear`, + which have the same signatures as the methods of :class:`list`. + It is a view of the tree: it changes when the tree changes, + and changing it changes the tree. + + Comments and processing instructions are only added to it when parsing + if the parser target collects them; see :class:`TreeBuilder`. + + :meth:`iter` iterates over all children of the document, + but :meth:`find`, :meth:`findall` and :meth:`iterfind` + search from the root element, so they never return the other children. + + .. versionadded:: next + + .. method:: _setroot(element) Replaces the root element for this tree. This discards the current @@ -1164,9 +1192,14 @@ ElementTree Objects .. method:: iter(tag=None) - Creates and returns a tree iterator for the root element. The iterator - loops over all elements in this tree, in section order. *tag* is the tag - to look for (default is to return all elements). + Creates and returns a tree iterator for the document. + The iterator loops over all children of the document + and their descendants, in document order. + *tag* is the tag to look for (default is to return all elements). + + .. versionchanged:: next + It iterates over all children of the document, + not only over the root element and its descendants. .. method:: iterfind(match, namespaces=None) @@ -1286,7 +1319,12 @@ TreeBuilder Objects create comments and processing instructions. When not given, the default factories will be used. When *insert_comments* and/or *insert_pis* is true, comments/pis will be inserted into the tree if they appear within the root - element (but not outside of it). + element. Those which appear outside of it are returned by + :meth:`document`. + + .. versionchanged:: next + Comments and processing instructions outside the root element + are no longer discarded. .. method:: close() @@ -1300,6 +1338,16 @@ TreeBuilder Objects either a bytestring, or a Unicode string. + .. method:: document() + + Returns the children of the document: + the root element, and the comments and processing instructions + which were inserted outside of it. + Returns a list of :class:`Element` instances. + + .. versionadded:: next + + .. method:: end(tag) Closes the current element. *tag* is the element name. Returns the diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index f3ddae7a2b2fdd..ebbeded4da32cd 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -642,6 +642,16 @@ xml and :meth:`!Document.createEntityReference`. (Contributed by Jason Orendorff and Serhiy Storchaka in :gh:`44871`.) +* Comments and processing instructions which occur outside of the root element + are no longer lost in :mod:`xml.etree.ElementTree`. + :class:`~xml.etree.ElementTree.ElementTree` now has the + :attr:`~xml.etree.ElementTree.ElementTree.children` attribute, + a sequence of the children of the document, + and :class:`~xml.etree.ElementTree.TreeBuilder` returns them + from the new :meth:`!document` method + when *insert_comments* or *insert_pis* is true. + (Contributed by Serhiy Storchaka in :gh:`68475`.) + * Add :meth:`!GetSpecifiedAttributeCount` method to the :mod:`XML parser ` objects. It tells how many of the reported attributes were given in the start tag diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 2af2d1fd64520b..dcb55d98472bf7 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -3739,7 +3739,7 @@ def test_basic(self): self.assertEqual(next(ET.iterparse(sourcefile, parser=parser))[0], 'end') tree = ET.ElementTree(None) - self.assertRaises(AttributeError, tree.iter) + self.assertEqual(list(tree.iter()), []) # Issue #16913 doc = ET.XML("a&b&c&") @@ -3836,6 +3836,179 @@ def test_pickle(self): pickle.dumps(it, proto) + +class DocumentChildrenTest(unittest.TestCase): + # gh-68475: comments and processing instructions outside the root element + + sample = ('') + + def parse(self, text=None): + builder = ET.TreeBuilder(insert_comments=True, insert_pis=True) + tree = ET.ElementTree() + tree.parse(io.StringIO(text if text is not None else self.sample), + ET.XMLParser(target=builder)) + return tree + + def test_only_the_root_by_default(self): + tree = ET.ElementTree() + tree.parse(io.StringIO(self.sample)) + self.assertEqual(summarize_list(tree.children), ['r']) + self.assertEqual(len(tree.children), 1) + self.assertIs(tree.children[0], tree.getroot()) + + def test_parse_keeps_the_prolog_and_the_epilog(self): + tree = self.parse() + self.assertEqual(summarize_list(tree.children), + [ET.Comment, ET.ProcessingInstruction, 'r', + ET.ProcessingInstruction, ET.Comment]) + self.assertEqual(tree.children[0].text, 'lead') + self.assertEqual(tree.children[-1].text, 'tail') + self.assertIs(tree.getroot(), tree.children[2]) + + def test_write(self): + tree = self.parse() + file = io.StringIO() + tree.write(file, encoding='unicode') + self.assertEqual(file.getvalue(), + '' + '') + + def test_iter(self): + tree = self.parse() + self.assertEqual(summarize_list(tree.iter()), + [ET.Comment, ET.ProcessingInstruction, 'r', + ET.ProcessingInstruction, 'a', + ET.ProcessingInstruction, ET.Comment]) + self.assertEqual(summarize_list(tree.iter('*')), + [ET.Comment, ET.ProcessingInstruction, 'r', + ET.ProcessingInstruction, 'a', + ET.ProcessingInstruction, ET.Comment]) + self.assertEqual(summarize_list(tree.iter('a')), ['a']) + # comments and processing instructions can be selected by the factory + self.assertEqual(summarize_list(tree.iter(ET.ProcessingInstruction)), + [ET.ProcessingInstruction] * 3) + self.assertEqual(summarize_list(tree.iter(ET.Comment)), + [ET.Comment, ET.Comment]) + + def test_find_searches_from_the_root(self): + tree = self.parse() + # find() and friends search from the root element, so they return + # the processing instruction inside it, but not those outside + self.assertEqual(summarize_list(tree.findall('*')), + [ET.ProcessingInstruction, 'a']) + self.assertEqual(summarize_list(tree.findall('.//*')), + [ET.ProcessingInstruction, 'a']) + self.assertEqual(tree.find('a').tag, 'a') + + def test_append_and_insert(self): + tree = ET.ElementTree(ET.Element('r')) + tree.children.insert(0, ET.Comment('lead')) + tree.children.append(ET.ProcessingInstruction('pi', 'data')) + self.assertEqual(summarize_list(tree.children), + [ET.Comment, 'r', ET.ProcessingInstruction]) + self.assertIs(tree.getroot(), tree.children[1]) + + def test_the_first_element_becomes_the_root(self): + tree = ET.ElementTree() + tree.children.append(ET.Comment('lead')) + self.assertIsNone(tree.getroot()) + elem = ET.Element('r') + tree.children.append(elem) + self.assertIs(tree.getroot(), elem) + + def test_only_one_element(self): + tree = ET.ElementTree(ET.Element('r')) + children = tree.children + children.insert(0, ET.Comment('lead')) + self.assertRaises(ValueError, children.append, ET.Element('second')) + self.assertRaises(ValueError, children.insert, 0, ET.Element('second')) + self.assertRaises(ValueError, children.extend, [ET.Element('second')]) + # the comment cannot be replaced by an element either + self.assertRaises(ValueError, children.__setitem__, 0, + ET.Element('second')) + self.assertEqual(summarize_list(tree.children), [ET.Comment, 'r']) + self.assertEqual(tree.getroot().tag, 'r') + # but the root element can be replaced + children[1] = ET.Element('other') + self.assertEqual(tree.getroot().tag, 'other') + + def test_not_an_element(self): + tree = ET.ElementTree(ET.Element('r')) + self.assertRaises(TypeError, tree.children.append, 'text') + self.assertRaises(TypeError, tree.children.insert, 0, None) + self.assertEqual(summarize_list(tree.children), ['r']) + + def test_remove_and_delete(self): + tree = self.parse() + root = tree.getroot() + tree.children.remove(root) + self.assertIsNone(tree.getroot()) + self.assertEqual(summarize_list(tree.children), + [ET.Comment, ET.ProcessingInstruction, + ET.ProcessingInstruction, ET.Comment]) + del tree.children[0] + self.assertEqual(summarize_list(tree.children), + [ET.ProcessingInstruction, ET.ProcessingInstruction, + ET.Comment]) + tree.children.clear() + self.assertEqual(summarize_list(tree.children), []) + self.assertIsNone(tree.getroot()) + + def test_slices(self): + tree = self.parse() + root = tree.getroot() + tree.children[0:2] = [ET.Comment('one')] + self.assertEqual(summarize_list(tree.children), + [ET.Comment, 'r', ET.ProcessingInstruction, + ET.Comment]) + self.assertIs(tree.getroot(), root) + # the slice which replaces the root element can add another one + new = ET.Element('new') + tree.children[1:2] = [new] + self.assertIs(tree.getroot(), new) + # but not two + self.assertRaises(ValueError, tree.children.__setitem__, + slice(0, 2), [ET.Element('a'), ET.Element('b')]) + self.assertIs(tree.getroot(), new) + del tree.children[1:2] + self.assertIsNone(tree.getroot()) + + def test_the_root_cannot_be_a_comment(self): + self.assertRaises(ValueError, ET.ElementTree, ET.Comment('c')) + self.assertRaises(ValueError, ET.ElementTree, + ET.ProcessingInstruction('pi')) + tree = ET.ElementTree(ET.Element('r')) + self.assertRaises(ValueError, tree._setroot, ET.Comment('c')) + + def test_tostring_of_a_comment(self): + # tostring() serializes a single node, which can be a comment + self.assertEqual(ET.tostring(ET.Comment('c')), b'') + self.assertEqual(ET.tostring(ET.ProcessingInstruction('t', 'd')), + b'') + + def test_document_without_the_root(self): + tree = ET.ElementTree() + tree.children.extend([ET.Comment('a'), ET.ProcessingInstruction('p')]) + file = io.StringIO() + tree.write(file, encoding='unicode') + self.assertEqual(file.getvalue(), '') + + def test_builder_document(self): + builder = ET.TreeBuilder(insert_comments=True, insert_pis=True) + parser = ET.XMLParser(target=builder) + parser.feed(self.sample) + root = parser.close() + self.assertEqual(root.tag, 'r') + self.assertEqual(len(builder.document()), 5) + self.assertIs(builder.document()[2], root) + + def test_builder_document_without_inserting(self): + builder = ET.TreeBuilder() + parser = ET.XMLParser(target=builder) + parser.feed(self.sample) + root = parser.close() + self.assertEqual(builder.document(), [root]) + class TreeBuilderTest(unittest.TestCase): sample1 = ('' % (self._tree._children,) + + def _check(self, value, root): + # Check a new child of a document whose root element is *root*, + # and return the root element after adding it. + if not iselement(value): + raise TypeError('expected an Element, not %s' + % type(value).__name__) + if _is_misc(value): + return root + if root is not None: + raise ValueError('a document can have only one element child') + return value + + def _contains_root(self, nodes): + root = self._tree._root + return root is not None and any(node is root for node in nodes) + + def append(self, value): + """Add a child at the end of the document.""" + root = self._check(value, self._tree._root) + self._tree._children.append(value) + self._tree._root = root + + def insert(self, index, value): + """Add a child at the given position.""" + root = self._check(value, self._tree._root) + self._tree._children.insert(index, value) + self._tree._root = root + + def extend(self, values): + """Add several children at the end of the document.""" + values = list(values) + root = self._tree._root + for item in values: + root = self._check(item, root) + self._tree._children.extend(values) + self._tree._root = root + + def remove(self, value): + """Remove the first child equal to the value.""" + self._tree._children.remove(value) + if value is self._tree._root: + self._tree._root = None + + def clear(self): + """Remove all children of the document.""" + self._tree._children.clear() + self._tree._root = None + + def __setitem__(self, index, value): + """Replace the child at the index, or the children in the slice.""" + children = self._tree._children + root = self._tree._root + if isinstance(index, slice): + value = list(value) + if self._contains_root(children[index]): + root = None + for item in value: + root = self._check(item, root) + else: + if children[index] is root: + root = None + root = self._check(value, root) + children[index] = value + self._tree._root = root + + def __delitem__(self, index): + """Remove the child at the index, or the children in the slice.""" + children = self._tree._children + if isinstance(index, slice): + root_removed = self._contains_root(children[index]) + else: + root_removed = children[index] is self._tree._root + del children[index] + if root_removed: + self._tree._root = None + + class ElementTree: """An XML element hierarchy. @@ -531,13 +639,18 @@ class ElementTree: """ def __init__(self, element=None, file=None): - if element is not None and not iselement(element): - raise TypeError('expected an Element, not %s' % - type(element).__name__) - self._root = element # first node + self._root = None # the root element + self._children = [] + if element is not None: + self._setroot(element) if file: self.parse(file) + @property + def children(self): + """A view of the children of the document.""" + return _DocumentChildren(self) + def getroot(self): """Return root element of this tree.""" return self._root @@ -552,6 +665,13 @@ def _setroot(self, element): if not iselement(element): raise TypeError('expected an Element, not %s' % type(element).__name__) + if _is_misc(element): + raise ValueError('the root element cannot be a comment ' + 'or a processing instruction') + if self._root is None: + self._children.append(element) + else: + self._children[self._children.index(self._root)] = element self._root = element def parse(self, source, parser=None): @@ -578,28 +698,36 @@ def parse(self, source, parser=None): # can define an internal _parse_whole API for efficiency. # It can be used to parse the whole source without feeding # it with chunks. - self._root = parser._parse_whole(source) + self._setroot(parser._parse_whole(source)) return self._root while data := source.read(65536): parser.feed(data) - self._root = parser.close() + # close() releases the target, ask it for the document first + document = getattr(getattr(parser, 'target', None), 'document', None) + result = parser.close() + # a custom target can return anything, even None + self._root = result + if document is not None: + self._children[:] = document() + else: + self._children = [result] if iselement(result) else [] return self._root finally: if close_source: source.close() def iter(self, tag=None): - """Create and return tree iterator for the root element. + """Create and return tree iterator for the document. - The iterator loops over all elements in this tree, in document - order. + The iterator loops over all children of the document and their + descendants, in document order. *tag* is a string with the tag name to iterate over (default is to return all elements). """ - # assert self._root is not None - return self._root.iter(tag) + for child in self._children: + yield from child.iter(tag) def find(self, path, namespaces=None): """Find first matching element by tag name or path. @@ -723,7 +851,7 @@ def write(self, file_or_filename, emitted as a pair of start/end tags """ - if self._root is None: + if not self._children: raise TypeError('ElementTree not initialized') if not method: method = "xml" @@ -741,12 +869,20 @@ def write(self, file_or_filename, write("\n" % ( declared_encoding,)) if method == "text": - _serialize_text(write, self._root) + for child in self._children: + _serialize_text(write, child) else: - qnames, namespaces = _namespaces(self._root, default_namespace) + root = self._root + if root is None: + # the document has no element child + qnames, namespaces = {None: None}, {} + else: + qnames, namespaces = _namespaces(root, default_namespace) serialize = _serialize[method] - serialize(write, self._root, qnames, namespaces, - short_empty_elements=short_empty_elements) + for child in self._children: + serialize(write, child, qnames, + namespaces if child is root else None, + short_empty_elements=short_empty_elements) # -------------------------------------------------------------------- # serialization support @@ -1100,7 +1236,10 @@ def tostring(element, encoding=None, method=None, *, """ stream = io.StringIO() if encoding == 'unicode' else io.BytesIO() - ElementTree(element).write(stream, encoding, + # the element can also be a comment or a processing instruction + tree = ElementTree() + tree.children.append(element) + tree.write(stream, encoding, xml_declaration=xml_declaration, default_namespace=default_namespace, method=method, @@ -1129,7 +1268,10 @@ def tostringlist(element, encoding=None, method=None, *, short_empty_elements=True): lst = [] stream = _ListDataStream(lst) - ElementTree(element).write(stream, encoding, + # the element can also be a comment or a processing instruction + tree = ElementTree() + tree.children.append(element) + tree.write(stream, encoding, xml_declaration=xml_declaration, default_namespace=default_namespace, method=method, @@ -1436,6 +1578,7 @@ def __init__(self, element_factory=None, *, self._elem = [] # element stack self._last = None # last element self._root = None # root element + self._document = [] # the children of the document self._tail = None # true if we're after an end tag if comment_factory is None: comment_factory = Comment @@ -1484,6 +1627,7 @@ def start(self, tag, attrs): self._elem[-1].append(elem) elif self._root is None: self._root = elem + self._document.append(elem) self._elem.append(elem) self._tail = 0 return elem @@ -1526,9 +1670,20 @@ def _handle_single(self, factory, insert, *args): self._last = elem if self._elem: self._elem[-1].append(elem) + else: + # outside the root element: the prolog or the epilog + self._document.append(elem) self._tail = 1 return elem + def document(self): + """Return the children of the document. + + These are the root element and the comments and processing + instructions which were inserted outside of it. + """ + return list(self._document) + # also see ElementTree and TreeBuilder class XMLParser: @@ -2110,6 +2265,7 @@ def _escape_attrib_c14n(text): # the Python version of it accessible for some "creative" by external code # (see tests) _Element_Py = Element + _TreeBuilder_Py = TreeBuilder # Element, SubElement, ParseError, TreeBuilder, XMLParser, _set_factories from _elementtree import * diff --git a/Misc/NEWS.d/next/Library/2026-08-31-12-00-00.gh-issue-68475.Nq7Vt2.rst b/Misc/NEWS.d/next/Library/2026-08-31-12-00-00.gh-issue-68475.Nq7Vt2.rst new file mode 100644 index 00000000000000..9868772ea29b19 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-31-12-00-00.gh-issue-68475.Nq7Vt2.rst @@ -0,0 +1,9 @@ +:class:`~xml.etree.ElementTree.ElementTree` now has +the :attr:`~xml.etree.ElementTree.ElementTree.children` attribute, +a sequence of the children of the document: +the root element and the comments and processing instructions around it. +When *insert_comments* or *insert_pis* is true, +:class:`~xml.etree.ElementTree.TreeBuilder` no longer discards comments +and processing instructions which occur outside of the root element; +they are returned by the new :meth:`!document` method +and are used by :func:`~xml.etree.ElementTree.parse`. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index f827274eeffba8..a460b3901003f2 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2419,6 +2419,7 @@ typedef struct { char insert_comments; char insert_pis; + PyObject *document; /* the children of the document, or NULL */ elementtreestate *state; } TreeBuilderObject; @@ -2455,6 +2456,14 @@ treebuilder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) t->start_ns_event_obj = t->end_ns_event_obj = NULL; t->comment_event_obj = t->pi_event_obj = NULL; t->insert_comments = t->insert_pis = 0; + t->document = PyList_New(0); + if (!t->document) { + Py_DECREF(t->this); + Py_DECREF(t->last); + Py_DECREF(t->stack); + Py_DECREF((PyObject *) t); + return NULL; + } t->state = get_elementtree_state_by_type(type); } return (PyObject *)t; @@ -2525,6 +2534,7 @@ treebuilder_gc_traverse(PyObject *op, visitproc visit, void *arg) Py_VISIT(self->end_event_obj); Py_VISIT(self->start_event_obj); Py_VISIT(self->events_append); + Py_VISIT(self->document); Py_VISIT(self->root); Py_VISIT(self->this); Py_VISIT(self->last); @@ -2557,6 +2567,7 @@ treebuilder_gc_clear(PyObject *op) Py_CLEAR(self->comment_factory); Py_CLEAR(self->element_factory); Py_CLEAR(self->root); + Py_CLEAR(self->document); return 0; } @@ -2785,6 +2796,9 @@ treebuilder_handle_start(TreeBuilderObject* self, PyObject* tag, goto error; } self->root = Py_NewRef(node); + if (PyList_Append(self->document, node) < 0) { + goto error; + } } if (self->index < PyList_GET_SIZE(self->stack)) { @@ -2897,11 +2911,17 @@ treebuilder_handle_comment(TreeBuilderObject* self, PyObject* text) return NULL; this = self->this; - if (self->insert_comments && this != Py_None) { - if (treebuilder_add_subelement(self->state, this, comment) < 0) { + if (self->insert_comments) { + if (this != Py_None) { + if (treebuilder_add_subelement(self->state, this, comment) < 0) { + goto error; + } + Py_XSETREF(self->last_for_tail, Py_NewRef(comment)); + } + /* outside the root element: the prolog or the epilog */ + else if (PyList_Append(self->document, comment) < 0) { goto error; } - Py_XSETREF(self->last_for_tail, Py_NewRef(comment)); } } else { comment = Py_NewRef(text); @@ -2937,11 +2957,17 @@ treebuilder_handle_pi(TreeBuilderObject* self, PyObject* target, PyObject* text) } this = self->this; - if (self->insert_pis && this != Py_None) { - if (treebuilder_add_subelement(self->state, this, pi) < 0) { + if (self->insert_pis) { + if (this != Py_None) { + if (treebuilder_add_subelement(self->state, this, pi) < 0) { + goto error; + } + Py_XSETREF(self->last_for_tail, Py_NewRef(pi)); + } + /* outside the root element: the prolog or the epilog */ + else if (PyList_Append(self->document, pi) < 0) { goto error; } - Py_XSETREF(self->last_for_tail, Py_NewRef(pi)); } } else { pi = _PyTuple_FromPair(target, text); @@ -4372,7 +4398,21 @@ static PyType_Spec element_spec = { .slots = element_slots, }; +/*[clinic input] +_elementtree.TreeBuilder.document + +Return the children of the document. +[clinic start generated code]*/ + +static PyObject * +_elementtree_TreeBuilder_document_impl(TreeBuilderObject *self) +/*[clinic end generated code: output=23aee8fbdee7bb53 input=8e3b119052a148d8]*/ +{ + return PyList_GetSlice(self->document, 0, PyList_GET_SIZE(self->document)); +} + static PyMethodDef treebuilder_methods[] = { + _ELEMENTTREE_TREEBUILDER_DOCUMENT_METHODDEF _ELEMENTTREE_TREEBUILDER_DATA_METHODDEF _ELEMENTTREE_TREEBUILDER_START_METHODDEF _ELEMENTTREE_TREEBUILDER_END_METHODDEF diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index c9e77a4c2b92d8..7e319e643e7698 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -1331,4 +1331,22 @@ _elementtree_XMLParser__setevents(PyObject *self, PyObject *const *args, Py_ssiz exit: return return_value; } -/*[clinic end generated code: output=c863ce16d8566291 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_elementtree_TreeBuilder_document__doc__, +"document($self, /)\n" +"--\n" +"\n" +"Return the children of the document."); + +#define _ELEMENTTREE_TREEBUILDER_DOCUMENT_METHODDEF \ + {"document", (PyCFunction)_elementtree_TreeBuilder_document, METH_NOARGS, _elementtree_TreeBuilder_document__doc__}, + +static PyObject * +_elementtree_TreeBuilder_document_impl(TreeBuilderObject *self); + +static PyObject * +_elementtree_TreeBuilder_document(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _elementtree_TreeBuilder_document_impl((TreeBuilderObject *)self); +} +/*[clinic end generated code: output=c26dab038e641a6c input=a9049054013a1b77]*/