gh-156072: Fix tokenize.untokenize to preserve the original line's whitespace

Lib/test/test_tokenize.py

@@ -1974,6 +1974,28 @@ def test_iter_compat(self):
         self.assertEqual(u.encoding, 'utf-8')
         self.assertEqual(tokenize.untokenize(iter(tokens)), b'Hello ')
 
+    def test_preserve_token_gap(self):
+        for ws in ("\t", "\f", " ", "\t \f"):
+            with self.subTest(whitespace=ws):
+                u = tokenize.Untokenizer()
+                u.prev_col = 2
+                u.add_whitespace((1, 2 + len(ws)), f"a,{ws}b = 1, 2")
+                self.assertEqual(u.tokens, [ws])
+
+    def test_preserve_backslash_whitespace(self):
+        for ws in ("\t", "\f", " ", "\t \f"):
+            with self.subTest(whitespace=ws):
+                u = tokenize.Untokenizer()
+                u.prev_line = "x = 1 + \\\n"
+                u.add_whitespace((2, len(ws)), f"{ws}y\n")
+                self.assertEqual(u.tokens, [' \\\n', ws])
+
+    def test_preserve_pre_continuation_whitespace_order(self):
+        u = tokenize.Untokenizer()
+        u.prev_line = "x = 1 +\t \f\\\n"
+        u.add_whitespace((2, 0), "y\n")
+        self.assertEqual(u.tokens, ['\t \f\\\n'])
+
 
 def contains_ambiguous_backslash(source):
     """Return `True` if the source contains a backslash on a
@@ -2206,6 +2228,25 @@ def test_indentation_semantics_retained(self):
         self.assertEqual(codelines[1], codelines[2])
         self.check_roundtrip(code)
 
+    def test_whitespace_between_tokens(self):
+        for whitespace in ("\t", "\f", " ", "\t \f"):
+            with self.subTest(whitespace=whitespace):
+                self.check_roundtrip(f"x{whitespace}={whitespace}1\n")
+
+    def test_whitespace_after_indent(self):
+        for indent in ("\t", "    "):
+            with self.subTest(indent=indent):
+                code = f"if False:\n{indent}print(\n{indent}{indent}'hello'\n{indent})"
+                self.check_roundtrip(code)
+
+    def test_whitespace_after_backslash(self):
+        for whitespace in ("\t", "\f", " ", "\t \f"):
+            with self.subTest(whitespace=whitespace):
+                self.check_roundtrip(f"x = 1 + \\\n{whitespace}y\n")
+
+    def test_whitespace_before_backslash(self):
+        self.check_roundtrip("x = 1 +\t \f\\\ny\n")
+
 
 class InvalidPythonTests(TestCase):
     def test_number_followed_by_name(self):

Lib/tokenize.py

@@ -173,15 +173,18 @@ def __init__(self):
         self.prev_line = ""
         self.encoding = None
 
-    def add_whitespace(self, start):
+    def add_whitespace(self, start, line=""):
         row, col = start
         if row < self.prev_row or row == self.prev_row and col < self.prev_col:
             raise ValueError("start ({},{}) precedes previous end ({},{})"
                              .format(row, col, self.prev_row, self.prev_col))
         self.add_backslash_continuation(start)
         col_offset = col - self.prev_col
         if col_offset:
-            self.tokens.append(" " * col_offset)
+            if line:
+                self.tokens.append(line[self.prev_col:col])
+            else:
+                self.tokens.append(" " * col_offset)
 
     def add_backslash_continuation(self, start):
         """Add backslash continuation characters if the row has increased
@@ -196,7 +199,7 @@ def add_backslash_continuation(self, start):
 
         newline = '\r\n' if self.prev_line.endswith('\r\n') else '\n'
         line = self.prev_line.rstrip('\\\r\n')
-        ws = ''.join(_itertools.takewhile(str.isspace, reversed(line)))
+        ws = line[len(line.rstrip()):]
         self.tokens.append(ws + f"\\{newline}" * row_offset)
         self.prev_col = 0
 
@@ -260,7 +263,7 @@ def untokenize(self, iterable):
                     extra_chars = last_line.count("{{") + last_line.count("}}")
                     end = (end_line, end_col + extra_chars)
 
-            self.add_whitespace(start)
+            self.add_whitespace(start, line)
             self.tokens.append(token)
             self.prev_row, self.prev_col = end
             if tok_type in (NEWLINE, NL):

Misc/NEWS.d/next/Library/2026-08-20-08-52-05.gh-issue-156072.TOT3kM.rst

@@ -0,0 +1,2 @@
+Fix :func:`tokenize.untokenize` to preserve the original line's whitespace
+when reconstructing source from a token iterator.