gh-68475: Keep comments and processing instructions outside the root element

ElementTree gets the children attribute, a view of the children of the
document, containing the root element and any number of comments and
processing instructions around it.  Adding a second element is an error.
iter() iterates over all of them, but find(), findall() and iterfind()
still search from the root element.

TreeBuilder collects the comments and processing instructions which occur
outside the root element and returns them, together with the root element,
from the new document() method.  This only happens when insert_comments or
insert_pis is set, so nothing changes for existing code.  parse() asks the
target for the document before close(), which releases it.

The C accelerator implements document() too, so that the feature works at
full parsing speed.

Author: serhiy-storchaka

Doc/library/xml.etree.elementtree.rst

@@ -1135,6 +1135,34 @@ ElementTree Objects
    of the XML *file* if given.
 
 
+   .. attribute:: children
+
+      A sequence of the children of the document:
+      the root element and the comments and processing instructions
+      which surround it.
+      It can contain at most one element,
+      which is the root element of the tree;
+      adding a second one raises :exc:`ValueError`,
+      and adding anything which is not an element raises :exc:`TypeError`.
+
+      It supports ``len()``, iteration, the :keyword:`in` operator,
+      :func:`reversed`, indexing and slicing (for getting, setting and
+      deleting), and the methods :meth:`!append`, :meth:`!insert`,
+      :meth:`!extend`, :meth:`!remove` and :meth:`!clear`,
+      which have the same signatures as the methods of :class:`list`.
+      It is a view of the tree: it changes when the tree changes,
+      and changing it changes the tree.
+
+      Comments and processing instructions are only added to it when parsing
+      if the parser target collects them; see :class:`TreeBuilder`.
+
+      :meth:`iter` iterates over all children of the document,
+      but :meth:`find`, :meth:`findall` and :meth:`iterfind`
+      search from the root element, so they never return the other children.
+
+      .. versionadded:: next
+
+
    .. method:: _setroot(element)
 
       Replaces the root element for this tree.  This discards the current
@@ -1164,9 +1192,14 @@ ElementTree Objects
 
    .. method:: iter(tag=None)
 
-      Creates and returns a tree iterator for the root element.  The iterator
-      loops over all elements in this tree, in section order.  *tag* is the tag
-      to look for (default is to return all elements).
+      Creates and returns a tree iterator for the document.
+      The iterator loops over all children of the document
+      and their descendants, in document order.
+      *tag* is the tag to look for (default is to return all elements).
+
+      .. versionchanged:: next
+         It iterates over all children of the document,
+         not only over the root element and its descendants.
 
 
    .. method:: iterfind(match, namespaces=None)
@@ -1286,7 +1319,12 @@ TreeBuilder Objects
    create comments and processing instructions.  When not given, the default
    factories will be used.  When *insert_comments* and/or *insert_pis* is true,
    comments/pis will be inserted into the tree if they appear within the root
-   element (but not outside of it).
+   element.  Those which appear outside of it are returned by
+   :meth:`document`.
+
+   .. versionchanged:: next
+      Comments and processing instructions outside the root element
+      are no longer discarded.
 
    .. method:: close()
 
@@ -1300,6 +1338,16 @@ TreeBuilder Objects
       either a bytestring, or a Unicode string.
 
 
+   .. method:: document()
+
+      Returns the children of the document:
+      the root element, and the comments and processing instructions
+      which were inserted outside of it.
+      Returns a list of :class:`Element` instances.
+
+      .. versionadded:: next
+
+
    .. method:: end(tag)
 
       Closes the current element.  *tag* is the element name.  Returns the

Doc/whatsnew/3.16.rst

@@ -642,6 +642,16 @@ xml
   and :meth:`!Document.createEntityReference`.
   (Contributed by Jason Orendorff and Serhiy Storchaka in :gh:`44871`.)
 
+* Comments and processing instructions which occur outside of the root element
+  are no longer lost in :mod:`xml.etree.ElementTree`.
+  :class:`~xml.etree.ElementTree.ElementTree` now has the
+  :attr:`~xml.etree.ElementTree.ElementTree.children` attribute,
+  a sequence of the children of the document,
+  and :class:`~xml.etree.ElementTree.TreeBuilder` returns them
+  from the new :meth:`!document` method
+  when *insert_comments* or *insert_pis* is true.
+  (Contributed by Serhiy Storchaka in :gh:`68475`.)
+
 * Add :meth:`!GetSpecifiedAttributeCount` method
   to the :mod:`XML parser <xml.parsers.expat>` objects.
   It tells how many of the reported attributes were given in the start tag

Lib/test/test_xml_etree.py

@@ -3739,7 +3739,7 @@ def test_basic(self):
         self.assertEqual(next(ET.iterparse(sourcefile, parser=parser))[0], 'end')
 
         tree = ET.ElementTree(None)
-        self.assertRaises(AttributeError, tree.iter)
+        self.assertEqual(list(tree.iter()), [])
 
         # Issue #16913
         doc = ET.XML("<root>a&amp;<sub>b&amp;</sub>c&amp;</root>")
@@ -3836,6 +3836,179 @@ def test_pickle(self):
                 pickle.dumps(it, proto)
 
 
+
+class DocumentChildrenTest(unittest.TestCase):
+    # gh-68475: comments and processing instructions outside the root element
+
+    sample = ('<!--lead--><?pi data?><r><?in?><a/></r><?after?><!--tail-->')
+
+    def parse(self, text=None):
+        builder = ET.TreeBuilder(insert_comments=True, insert_pis=True)
+        tree = ET.ElementTree()
+        tree.parse(io.StringIO(text if text is not None else self.sample),
+                   ET.XMLParser(target=builder))
+        return tree
+
+    def test_only_the_root_by_default(self):
+        tree = ET.ElementTree()
+        tree.parse(io.StringIO(self.sample))
+        self.assertEqual(summarize_list(tree.children), ['r'])
+        self.assertEqual(len(tree.children), 1)
+        self.assertIs(tree.children[0], tree.getroot())
+
+    def test_parse_keeps_the_prolog_and_the_epilog(self):
+        tree = self.parse()
+        self.assertEqual(summarize_list(tree.children),
+                         [ET.Comment, ET.ProcessingInstruction, 'r',
+                          ET.ProcessingInstruction, ET.Comment])
+        self.assertEqual(tree.children[0].text, 'lead')
+        self.assertEqual(tree.children[-1].text, 'tail')
+        self.assertIs(tree.getroot(), tree.children[2])
+
+    def test_write(self):
+        tree = self.parse()
+        file = io.StringIO()
+        tree.write(file, encoding='unicode')
+        self.assertEqual(file.getvalue(),
+                '<!--lead--><?pi data?><r><?in?><a /></r>'
+                '<?after?><!--tail-->')
+
+    def test_iter(self):
+        tree = self.parse()
+        self.assertEqual(summarize_list(tree.iter()),
+                         [ET.Comment, ET.ProcessingInstruction, 'r',
+                          ET.ProcessingInstruction, 'a',
+                          ET.ProcessingInstruction, ET.Comment])
+        self.assertEqual(summarize_list(tree.iter('*')),
+                         [ET.Comment, ET.ProcessingInstruction, 'r',
+                          ET.ProcessingInstruction, 'a',
+                          ET.ProcessingInstruction, ET.Comment])
+        self.assertEqual(summarize_list(tree.iter('a')), ['a'])
+        # comments and processing instructions can be selected by the factory
+        self.assertEqual(summarize_list(tree.iter(ET.ProcessingInstruction)),
+                         [ET.ProcessingInstruction] * 3)
+        self.assertEqual(summarize_list(tree.iter(ET.Comment)),
+                         [ET.Comment, ET.Comment])
+
+    def test_find_searches_from_the_root(self):
+        tree = self.parse()
+        # find() and friends search from the root element, so they return
+        # the processing instruction inside it, but not those outside
+        self.assertEqual(summarize_list(tree.findall('*')),
+                         [ET.ProcessingInstruction, 'a'])
+        self.assertEqual(summarize_list(tree.findall('.//*')),
+                         [ET.ProcessingInstruction, 'a'])
+        self.assertEqual(tree.find('a').tag, 'a')
+
+    def test_append_and_insert(self):
+        tree = ET.ElementTree(ET.Element('r'))
+        tree.children.insert(0, ET.Comment('lead'))
+        tree.children.append(ET.ProcessingInstruction('pi', 'data'))
+        self.assertEqual(summarize_list(tree.children),
+                         [ET.Comment, 'r', ET.ProcessingInstruction])
+        self.assertIs(tree.getroot(), tree.children[1])
+
+    def test_the_first_element_becomes_the_root(self):
+        tree = ET.ElementTree()
+        tree.children.append(ET.Comment('lead'))
+        self.assertIsNone(tree.getroot())
+        elem = ET.Element('r')
+        tree.children.append(elem)
+        self.assertIs(tree.getroot(), elem)
+
+    def test_only_one_element(self):
+        tree = ET.ElementTree(ET.Element('r'))
+        children = tree.children
+        children.insert(0, ET.Comment('lead'))
+        self.assertRaises(ValueError, children.append, ET.Element('second'))
+        self.assertRaises(ValueError, children.insert, 0, ET.Element('second'))
+        self.assertRaises(ValueError, children.extend, [ET.Element('second')])
+        # the comment cannot be replaced by an element either
+        self.assertRaises(ValueError, children.__setitem__, 0,
+                          ET.Element('second'))
+        self.assertEqual(summarize_list(tree.children), [ET.Comment, 'r'])
+        self.assertEqual(tree.getroot().tag, 'r')
+        # but the root element can be replaced
+        children[1] = ET.Element('other')
+        self.assertEqual(tree.getroot().tag, 'other')
+
+    def test_not_an_element(self):
+        tree = ET.ElementTree(ET.Element('r'))
+        self.assertRaises(TypeError, tree.children.append, 'text')
+        self.assertRaises(TypeError, tree.children.insert, 0, None)
+        self.assertEqual(summarize_list(tree.children), ['r'])
+
+    def test_remove_and_delete(self):
+        tree = self.parse()
+        root = tree.getroot()
+        tree.children.remove(root)
+        self.assertIsNone(tree.getroot())
+        self.assertEqual(summarize_list(tree.children),
+                         [ET.Comment, ET.ProcessingInstruction,
+                          ET.ProcessingInstruction, ET.Comment])
+        del tree.children[0]
+        self.assertEqual(summarize_list(tree.children),
+                         [ET.ProcessingInstruction, ET.ProcessingInstruction,
+                          ET.Comment])
+        tree.children.clear()
+        self.assertEqual(summarize_list(tree.children), [])
+        self.assertIsNone(tree.getroot())
+
+    def test_slices(self):
+        tree = self.parse()
+        root = tree.getroot()
+        tree.children[0:2] = [ET.Comment('one')]
+        self.assertEqual(summarize_list(tree.children),
+                         [ET.Comment, 'r', ET.ProcessingInstruction,
+                          ET.Comment])
+        self.assertIs(tree.getroot(), root)
+        # the slice which replaces the root element can add another one
+        new = ET.Element('new')
+        tree.children[1:2] = [new]
+        self.assertIs(tree.getroot(), new)
+        # but not two
+        self.assertRaises(ValueError, tree.children.__setitem__,
+                          slice(0, 2), [ET.Element('a'), ET.Element('b')])
+        self.assertIs(tree.getroot(), new)
+        del tree.children[1:2]
+        self.assertIsNone(tree.getroot())
+
+    def test_the_root_cannot_be_a_comment(self):
+        self.assertRaises(ValueError, ET.ElementTree, ET.Comment('c'))
+        self.assertRaises(ValueError, ET.ElementTree,
+                          ET.ProcessingInstruction('pi'))
+        tree = ET.ElementTree(ET.Element('r'))
+        self.assertRaises(ValueError, tree._setroot, ET.Comment('c'))
+
+    def test_tostring_of_a_comment(self):
+        # tostring() serializes a single node, which can be a comment
+        self.assertEqual(ET.tostring(ET.Comment('c')), b'<!--c-->')
+        self.assertEqual(ET.tostring(ET.ProcessingInstruction('t', 'd')),
+                         b'<?t d?>')
+
+    def test_document_without_the_root(self):
+        tree = ET.ElementTree()
+        tree.children.extend([ET.Comment('a'), ET.ProcessingInstruction('p')])
+        file = io.StringIO()
+        tree.write(file, encoding='unicode')
+        self.assertEqual(file.getvalue(), '<!--a--><?p?>')
+
+    def test_builder_document(self):
+        builder = ET.TreeBuilder(insert_comments=True, insert_pis=True)
+        parser = ET.XMLParser(target=builder)
+        parser.feed(self.sample)
+        root = parser.close()
+        self.assertEqual(root.tag, 'r')
+        self.assertEqual(len(builder.document()), 5)
+        self.assertIs(builder.document()[2], root)
+
+    def test_builder_document_without_inserting(self):
+        builder = ET.TreeBuilder()
+        parser = ET.XMLParser(target=builder)
+        parser.feed(self.sample)
+        root = parser.close()
+        self.assertEqual(builder.document(), [root])
+
 class TreeBuilderTest(unittest.TestCase):
     sample1 = ('<!DOCTYPE html PUBLIC'
         ' "-//W3C//DTD XHTML 1.0 Transitional//EN"'