From 120cbd5f52dc84953b12ca832d5032c71a363264 Mon Sep 17 00:00:00 2001 From: Kelly Date: Wed, 19 Aug 2026 23:56:17 +1000 Subject: [PATCH 1/5] Add tests for `tokenize.untokenize` whitespace preservation --- Lib/test/test_tokenize.py | 30 ++++++++++++++++++++++++++++++ Lib/tokenize.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index e2db09d61f409bd..94f6e96afadc2aa 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1974,6 +1974,21 @@ def test_iter_compat(self): self.assertEqual(u.encoding, 'utf-8') self.assertEqual(tokenize.untokenize(iter(tokens)), b'Hello ') + def test_preserve_token_gap(self): + for ws in ("\t", "\f", " ", "\t \f"): + with self.subTest(whitespace=ws): + u = tokenize.Untokenizer() + u.prev_col = 2 + u.add_whitespace((1, 2 + len(ws)), f"a,{ws}b = 1, 2") + self.assertEqual(u.tokens, [ws]) + + def test_preserve_backslash_whitespace(self): + for ws in ("\t", "\f", " ", "\t \f"): + with self.subTest(whitespace=ws): + u = tokenize.Untokenizer() + u.prev_line = "x = 1 + \\\n" + u.add_whitespace((2, len(ws)), f"{ws}y\n") + self.assertEqual(u.tokens, [' \\\n', ws]) def contains_ambiguous_backslash(source): """Return `True` if the source contains a backslash on a @@ -2206,6 +2221,21 @@ def test_indentation_semantics_retained(self): self.assertEqual(codelines[1], codelines[2]) self.check_roundtrip(code) + def test_whitespace_between_tokens(self): + for whitespace in ("\t", "\f", " ", "\t \f"): + with self.subTest(whitespace=whitespace): + self.check_roundtrip(f"x{whitespace}={whitespace}1\n") + + def test_whitespace_after_indent(self): + for indent in ("\t", " "): + with self.subTest(indent=indent): + code = f"if False:\n{indent}print(\n{indent}{indent}'hello'\n{indent})" + self.check_roundtrip(code) + + def test_whitespace_after_backslash(self): + for whitespace in ("\t", "\f", " ", "\t \f"): + with self.subTest(whitespace=whitespace): + self.check_roundtrip(f"x = 1 + \\\n{whitespace}y\n") class InvalidPythonTests(TestCase): def test_number_followed_by_name(self): diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 3545d92c4f5d7ff..d6dceeee45c3051 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -173,7 +173,7 @@ def __init__(self): self.prev_line = "" self.encoding = None - def add_whitespace(self, start): + def add_whitespace(self, start, line=""): row, col = start if row < self.prev_row or row == self.prev_row and col < self.prev_col: raise ValueError("start ({},{}) precedes previous end ({},{})" From 0b691c154c23260e5ee4cdcc31fd01f48ef2fedc Mon Sep 17 00:00:00 2001 From: Kelly Date: Wed, 19 Aug 2026 23:59:07 +1000 Subject: [PATCH 2/5] Fix `tokenize.untokenize` to preserve the original line's whitespace --- Lib/tokenize.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index d6dceeee45c3051..215f259199c1bfc 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -181,7 +181,10 @@ def add_whitespace(self, start, line=""): self.add_backslash_continuation(start) col_offset = col - self.prev_col if col_offset: - self.tokens.append(" " * col_offset) + if line: + self.tokens.append(line[self.prev_col:col]) + else: + self.tokens.append(" " * col_offset) def add_backslash_continuation(self, start): """Add backslash continuation characters if the row has increased @@ -260,7 +263,7 @@ def untokenize(self, iterable): extra_chars = last_line.count("{{") + last_line.count("}}") end = (end_line, end_col + extra_chars) - self.add_whitespace(start) + self.add_whitespace(start, line) self.tokens.append(token) self.prev_row, self.prev_col = end if tok_type in (NEWLINE, NL): From d387d114c2ec111f9e063b8f7904d78b93928a08 Mon Sep 17 00:00:00 2001 From: Kelly Date: Wed, 19 Aug 2026 23:58:07 +1000 Subject: [PATCH 3/5] Add tests for `tokenize.untokenize` pre-continuation whitespace ordering --- Lib/test/test_tokenize.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 94f6e96afadc2aa..e452aab26554c6f 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1990,6 +1990,13 @@ def test_preserve_backslash_whitespace(self): u.add_whitespace((2, len(ws)), f"{ws}y\n") self.assertEqual(u.tokens, [' \\\n', ws]) + def test_preserve_pre_continuation_whitespace_order(self): + u = tokenize.Untokenizer() + u.prev_line = "x = 1 +\t \f\\\n" + u.add_whitespace((2, 0), "y\n") + self.assertEqual(u.tokens, ['\t \f\\\n']) + + def contains_ambiguous_backslash(source): """Return `True` if the source contains a backslash on a line by itself. For example: @@ -2237,6 +2244,10 @@ def test_whitespace_after_backslash(self): with self.subTest(whitespace=whitespace): self.check_roundtrip(f"x = 1 + \\\n{whitespace}y\n") + def test_whitespace_before_backslash(self): + self.check_roundtrip("x = 1 +\t \f\\\ny\n") + + class InvalidPythonTests(TestCase): def test_number_followed_by_name(self): # See issue #gh-105549 From 0f1268c381f73831a76e3c47adafb0a4e7363180 Mon Sep 17 00:00:00 2001 From: Kelly Date: Wed, 19 Aug 2026 23:59:48 +1000 Subject: [PATCH 4/5] Fix `tokenize.untokenize` to preserve the order of whitespace that comes before a continuation backslash --- Lib/tokenize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 215f259199c1bfc..647d334c9ceebc8 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -199,7 +199,7 @@ def add_backslash_continuation(self, start): newline = '\r\n' if self.prev_line.endswith('\r\n') else '\n' line = self.prev_line.rstrip('\\\r\n') - ws = ''.join(_itertools.takewhile(str.isspace, reversed(line))) + ws = line[len(line.rstrip()):] self.tokens.append(ws + f"\\{newline}" * row_offset) self.prev_col = 0 From f8151422d72862ed5bc0cbcb25ce2be2f3d9d1a6 Mon Sep 17 00:00:00 2001 From: Kelly Date: Thu, 20 Aug 2026 08:52:42 +1000 Subject: [PATCH 5/5] Add NEWS entry --- .../next/Library/2026-08-20-08-52-05.gh-issue-156072.TOT3kM.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-08-20-08-52-05.gh-issue-156072.TOT3kM.rst diff --git a/Misc/NEWS.d/next/Library/2026-08-20-08-52-05.gh-issue-156072.TOT3kM.rst b/Misc/NEWS.d/next/Library/2026-08-20-08-52-05.gh-issue-156072.TOT3kM.rst new file mode 100644 index 000000000000000..668af61eb410d9f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-20-08-52-05.gh-issue-156072.TOT3kM.rst @@ -0,0 +1,2 @@ +Fix :func:`tokenize.untokenize` to preserve the original line's whitespace +when reconstructing source from a token iterator.