diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index e2db09d61f409b..e452aab26554c6 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1974,6 +1974,28 @@ def test_iter_compat(self): self.assertEqual(u.encoding, 'utf-8') self.assertEqual(tokenize.untokenize(iter(tokens)), b'Hello ') + def test_preserve_token_gap(self): + for ws in ("\t", "\f", " ", "\t \f"): + with self.subTest(whitespace=ws): + u = tokenize.Untokenizer() + u.prev_col = 2 + u.add_whitespace((1, 2 + len(ws)), f"a,{ws}b = 1, 2") + self.assertEqual(u.tokens, [ws]) + + def test_preserve_backslash_whitespace(self): + for ws in ("\t", "\f", " ", "\t \f"): + with self.subTest(whitespace=ws): + u = tokenize.Untokenizer() + u.prev_line = "x = 1 + \\\n" + u.add_whitespace((2, len(ws)), f"{ws}y\n") + self.assertEqual(u.tokens, [' \\\n', ws]) + + def test_preserve_pre_continuation_whitespace_order(self): + u = tokenize.Untokenizer() + u.prev_line = "x = 1 +\t \f\\\n" + u.add_whitespace((2, 0), "y\n") + self.assertEqual(u.tokens, ['\t \f\\\n']) + def contains_ambiguous_backslash(source): """Return `True` if the source contains a backslash on a @@ -2206,6 +2228,25 @@ def test_indentation_semantics_retained(self): self.assertEqual(codelines[1], codelines[2]) self.check_roundtrip(code) + def test_whitespace_between_tokens(self): + for whitespace in ("\t", "\f", " ", "\t \f"): + with self.subTest(whitespace=whitespace): + self.check_roundtrip(f"x{whitespace}={whitespace}1\n") + + def test_whitespace_after_indent(self): + for indent in ("\t", " "): + with self.subTest(indent=indent): + code = f"if False:\n{indent}print(\n{indent}{indent}'hello'\n{indent})" + self.check_roundtrip(code) + + def test_whitespace_after_backslash(self): + for whitespace in ("\t", "\f", " ", "\t \f"): + with self.subTest(whitespace=whitespace): + self.check_roundtrip(f"x = 1 + \\\n{whitespace}y\n") + + def test_whitespace_before_backslash(self): + self.check_roundtrip("x = 1 +\t \f\\\ny\n") + class InvalidPythonTests(TestCase): def test_number_followed_by_name(self): diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 3545d92c4f5d7f..647d334c9ceebc 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -173,7 +173,7 @@ def __init__(self): self.prev_line = "" self.encoding = None - def add_whitespace(self, start): + def add_whitespace(self, start, line=""): row, col = start if row < self.prev_row or row == self.prev_row and col < self.prev_col: raise ValueError("start ({},{}) precedes previous end ({},{})" @@ -181,7 +181,10 @@ def add_whitespace(self, start): self.add_backslash_continuation(start) col_offset = col - self.prev_col if col_offset: - self.tokens.append(" " * col_offset) + if line: + self.tokens.append(line[self.prev_col:col]) + else: + self.tokens.append(" " * col_offset) def add_backslash_continuation(self, start): """Add backslash continuation characters if the row has increased @@ -196,7 +199,7 @@ def add_backslash_continuation(self, start): newline = '\r\n' if self.prev_line.endswith('\r\n') else '\n' line = self.prev_line.rstrip('\\\r\n') - ws = ''.join(_itertools.takewhile(str.isspace, reversed(line))) + ws = line[len(line.rstrip()):] self.tokens.append(ws + f"\\{newline}" * row_offset) self.prev_col = 0 @@ -260,7 +263,7 @@ def untokenize(self, iterable): extra_chars = last_line.count("{{") + last_line.count("}}") end = (end_line, end_col + extra_chars) - self.add_whitespace(start) + self.add_whitespace(start, line) self.tokens.append(token) self.prev_row, self.prev_col = end if tok_type in (NEWLINE, NL): diff --git a/Misc/NEWS.d/next/Library/2026-08-20-08-52-05.gh-issue-156072.TOT3kM.rst b/Misc/NEWS.d/next/Library/2026-08-20-08-52-05.gh-issue-156072.TOT3kM.rst new file mode 100644 index 00000000000000..668af61eb410d9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-20-08-52-05.gh-issue-156072.TOT3kM.rst @@ -0,0 +1,2 @@ +Fix :func:`tokenize.untokenize` to preserve the original line's whitespace +when reconstructing source from a token iterator.