From 7b88fd4ee5c8985faf9a61a12ac729484f5b7436 Mon Sep 17 00:00:00 2001
From: gwosti <322952417+gwosti@users.noreply.github.com>
Date: Tue, 1 Sep 2026 07:51:57 +0200
Subject: [PATCH 1/2] gh-155525: Avoid quadratic f-string tokenization

---
 Lib/test/test_fstring.py                      |  21 ++++
 Lib/test/test_tstring.py                      |  24 ++++
 ...-09-01-07-15-20.gh-issue-155525.A7kP2m.rst |   2 +
 Parser/lexer/buffer.c                         |   4 +
 Parser/lexer/lexer.c                          |   3 -
 Parser/lexer/lexer.h                          |   2 -
 Parser/lexer/state.c                          |  19 ---
 Parser/lexer/state.h                          |   5 +-
 Parser/lexer/string.c                         | 108 ++++--------------
 Parser/tokenizer/reader.c                     |   6 -
 10 files changed, 76 insertions(+), 118 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-01-07-15-20.gh-issue-155525.A7kP2m.rst

diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index c1ef1a73f05c204..7b76ec76260f7da 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -832,6 +832,14 @@ def build_fstr(n, extra=''):
         s = "f'{1}' 'x' 'y'" * 1024
         self.assertEqual(eval(s), '1xy' * 1024)
 
+    @support.requires_resource('cpu')
+    def test_many_fstrings_in_module(self):
+        fields = ''.join(f'{{x{i}}}' for i in range(100))
+        source = ''.join(
+            f"value_{i} = f'{fields}'\n" for i in range(1_000)
+        )
+        compile(source, '<string>', 'exec')
+
     def test_format_specifier_expressions(self):
         width = 10
         precision = 4
@@ -1796,6 +1804,19 @@ def test_debug_in_file(self):
         self.assertEqual(stdout.decode('utf-8').strip().replace('\r\n', '\n').replace('\r', '\n'),
                          "3\n=3")
 
+    def test_debug_in_file_after_buffer_resize(self):
+        expression = "(\n" + (" " * 64 + "\n") * 256 + "1\n)"
+        expected = expression + "=1"
+        with temp_cwd():
+            script = 'script.py'
+            source = (
+                f"result = f'''{{{expression}=}}'''\n"
+                f"assert result == {expected!r}\n"
+            )
+            with open(script, 'w') as f:
+                f.write(source)
+            assert_python_ok(script)
+
     def test_syntax_warning_infinite_recursion_in_file(self):
         with temp_cwd():
             script = 'script.py'
diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py
index 74653c77c55de17..6ea0f5bcc157df5 100644
--- a/Lib/test/test_tstring.py
+++ b/Lib/test/test_tstring.py
@@ -1,5 +1,8 @@
 import unittest
 
+from test import support
+from test.support.os_helper import temp_cwd
+from test.support.script_helper import assert_python_ok
 from test.test_string._support import TStringBaseCase, fstring
 
 
@@ -79,6 +82,27 @@ def upper(self):
         )
         self.assertEqual(fstring(t), "Name: Bob, Age: 30")
 
+    def test_interpolation_expression_in_file_after_buffer_resize(self):
+        expression = "(\n" + (" " * 64 + "\n") * 256 + "1\n)"
+        with temp_cwd():
+            script = 'script.py'
+            source = (
+                f"template = t'''{{{expression}}}'''\n"
+                "interpolation = template.interpolations[0]\n"
+                f"assert interpolation.expression == {expression!r}\n"
+            )
+            with open(script, 'w') as f:
+                f.write(source)
+            assert_python_ok(script)
+
+    @support.requires_resource('cpu')
+    def test_many_tstrings_in_module(self):
+        fields = ''.join(f'{{x{i}}}' for i in range(100))
+        source = ''.join(
+            f"value_{i} = t'{fields}'\n" for i in range(1_000)
+        )
+        compile(source, '<string>', 'exec')
+
     def test_format_specifiers(self):
         # Test basic format specifiers
         value = 3.14159
diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-01-07-15-20.gh-issue-155525.A7kP2m.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-01-07-15-20.gh-issue-155525.A7kP2m.rst
new file mode 100644
index 000000000000000..9da3c38ce4691b1
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-01-07-15-20.gh-issue-155525.A7kP2m.rst
@@ -0,0 +1,2 @@
+Fix quadratic-time tokenization of modules containing many f-strings or
+t-strings.
diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c
index cd6885a7d01040a..6a45c69119d9217 100644
--- a/Parser/lexer/buffer.c
+++ b/Parser/lexer/buffer.c
@@ -15,6 +15,8 @@ _PyLexer_remember_fstring_buffers(struct tok_state *tok)
         mode = &(tok->tok_mode_stack[index]);
         mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf;
         mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf;
+        mode->last_expr_start_offset = mode->last_expr_start == NULL
+            ? -1 : mode->last_expr_start - tok->buf;
     }
 }
 
@@ -29,6 +31,8 @@ _PyLexer_restore_fstring_buffers(struct tok_state *tok)
         mode = &(tok->tok_mode_stack[index]);
         mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset;
         mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset;
+        mode->last_expr_start = mode->last_expr_start_offset < 0
+            ? NULL : tok->buf + mode->last_expr_start_offset;
     }
 }
 
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index a96362c8961023a..2efe41c312427da 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -542,9 +542,6 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
          int cursor_in_format_with_debug =
              cursor == 1 && (current_tok->in_debug || in_format_spec);
          int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
-        if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) {
-            return MAKE_TOKEN(ENDMARKER);
-        }
         if ((cursor_valid) && c != '{' && _PyLexer_set_ftstring_expr(tok, token, c)) {
             return MAKE_TOKEN(ERRORTOKEN);
         }
diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h
index 1d97ac57b745b09..73781eaa60cea88 100644
--- a/Parser/lexer/lexer.h
+++ b/Parser/lexer/lexer.h
@@ -3,8 +3,6 @@
 
 #include "state.h"
 
-int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur);
-
 int _PyTokenizer_Get(struct tok_state *, struct token *);
 
 #endif
diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c
index 2a6408bef927a36..86cecfd039c1f5e 100644
--- a/Parser/lexer/state.c
+++ b/Parser/lexer/state.c
@@ -61,24 +61,6 @@ _PyTokenizer_tok_new(void)
     return tok;
 }
 
-static void
-free_fstring_expressions(struct tok_state *tok)
-{
-    int index;
-    tokenizer_mode *mode;
-
-    for (index = tok->tok_mode_stack_index; index >= 0; --index) {
-        mode = &(tok->tok_mode_stack[index]);
-        if (mode->last_expr_buffer != NULL) {
-            PyMem_Free(mode->last_expr_buffer);
-            mode->last_expr_buffer = NULL;
-            mode->last_expr_size = 0;
-            mode->last_expr_end = -1;
-            mode->in_format_spec = 0;
-        }
-    }
-}
-
 /* Free a tok_state structure */
 void
 _PyTokenizer_Free(struct tok_state *tok)
@@ -90,7 +72,6 @@ _PyTokenizer_Free(struct tok_state *tok)
     Py_XDECREF(tok->module);
     _PyTok_ReaderFree(tok);
     _PyTok_SourceClear(&tok->source);
-    free_fstring_expressions(tok);
     PyMem_Free(tok);
 }
 
diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h
index 0824785195491ee..3d94725540d80be 100644
--- a/Parser/lexer/state.h
+++ b/Parser/lexer/state.h
@@ -56,9 +56,8 @@ typedef struct _tokenizer_mode {
     Py_ssize_t start_offset;
     Py_ssize_t multi_line_start_offset;
 
-    Py_ssize_t last_expr_size;
-    Py_ssize_t last_expr_end;
-    char* last_expr_buffer;
+    const char* last_expr_start;
+    Py_ssize_t last_expr_start_offset;
     int in_debug;
     int in_format_spec;
 
diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c
index d67c48f7f678eda..7b7955f06e049fa 100644
--- a/Parser/lexer/string.c
+++ b/Parser/lexer/string.c
@@ -16,6 +16,10 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
     if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) {
         return 0;
     }
+    const char *expression = tok_mode->last_expr_start;
+    assert(expression != NULL);
+    assert(expression <= tok->start);
+    Py_ssize_t expression_size = tok->start - expression;
     PyObject *res = NULL;
 
     // Look for a # character outside of string literals
@@ -23,8 +27,8 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
     int in_string = 0;
     char quote_char = 0;
 
-    for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) {
-        char ch = tok_mode->last_expr_buffer[i];
+    for (Py_ssize_t i = 0; i < expression_size; i++) {
+        char ch = expression[i];
 
         // Skip escaped characters
         if (ch == '\\') {
@@ -60,7 +64,8 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
     // If we found a # character in the expression, we need to handle comments
     if (hash_detected) {
         // Allocate buffer for processed result
-        char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char));
+        char *result = (char *)PyMem_Malloc(
+            (expression_size + 1) * sizeof(char));
         if (!result) {
             return -1;
         }
@@ -71,8 +76,8 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
         quote_char = 0;    // Current string quote char
 
         // Process each character
-        while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
-            char ch = tok_mode->last_expr_buffer[i];
+        while (i < expression_size) {
+            char ch = expression[i];
 
             // Handle string quotes
             if (ch == '"' || ch == '\'') {
@@ -87,11 +92,10 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
             }
             // Skip comments
             else if (ch == '#' && !in_string) {
-                while (i < tok_mode->last_expr_size - tok_mode->last_expr_end &&
-                       tok_mode->last_expr_buffer[i] != '\n') {
+                while (i < expression_size && expression[i] != '\n') {
                     i++;
                 }
-                if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
+                if (i < expression_size) {
                     result[j++] = '\n';
                 }
             }
@@ -106,11 +110,7 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
         res = PyUnicode_DecodeUTF8(result, j, NULL);
         PyMem_Free(result);
     } else {
-        res = PyUnicode_DecodeUTF8(
-            tok_mode->last_expr_buffer,
-            tok_mode->last_expr_size - tok_mode->last_expr_end,
-            NULL
-        );
+        res = PyUnicode_DecodeUTF8(expression, expression_size, NULL);
     }
 
     if (!res) {
@@ -120,61 +120,6 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
     return 0;
 }
 
-int
-_PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
-{
-    assert(tok->cur != NULL);
-
-    Py_ssize_t size = strlen(tok->cur);
-    tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
-
-    switch (cur) {
-       case 0:
-            if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) {
-                return 1;
-            }
-            char *new_buffer = PyMem_Realloc(
-                tok_mode->last_expr_buffer,
-                tok_mode->last_expr_size + size
-            );
-            if (new_buffer == NULL) {
-                PyMem_Free(tok_mode->last_expr_buffer);
-                goto error;
-            }
-            tok_mode->last_expr_buffer = new_buffer;
-            strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size);
-            tok_mode->last_expr_size += size;
-            break;
-        case '{':
-            if (tok_mode->last_expr_buffer != NULL) {
-                PyMem_Free(tok_mode->last_expr_buffer);
-            }
-            tok_mode->last_expr_buffer = PyMem_Malloc(size);
-            if (tok_mode->last_expr_buffer == NULL) {
-                goto error;
-            }
-            tok_mode->last_expr_size = size;
-            tok_mode->last_expr_end = -1;
-            strncpy(tok_mode->last_expr_buffer, tok->cur, size);
-            break;
-        case '}':
-        case '!':
-            tok_mode->last_expr_end = strlen(tok->start);
-            break;
-        case ':':
-            if (tok_mode->last_expr_end == -1) {
-               tok_mode->last_expr_end = strlen(tok->start);
-            }
-            break;
-        default:
-            Py_UNREACHABLE();
-    }
-    return 1;
-error:
-    tok->done = E_NOMEM;
-    return 0;
-}
-
 int
 _PyLexer_check_string_prefixes(struct tok_state *tok,
                                              int saw_b, int saw_r, int saw_u,
@@ -268,9 +213,8 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
     the_current_tok->first_line = tok->lineno;
     the_current_tok->start_offset = -1;
     the_current_tok->multi_line_start_offset = -1;
-    the_current_tok->last_expr_buffer = NULL;
-    the_current_tok->last_expr_size = 0;
-    the_current_tok->last_expr_end = -1;
+    the_current_tok->last_expr_start = NULL;
+    the_current_tok->last_expr_start_offset = -1;
     the_current_tok->in_format_spec = 0;
     the_current_tok->in_debug = 0;
 
@@ -436,6 +380,9 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
     if (start_char == '{') {
         int peek1 = tok_nextc(tok);
         tok_backup(tok, peek1);
+        if (peek1 != '{') {
+            current_tok->last_expr_start = tok->cur;
+        }
         tok_backup(tok, start_char);
         if (peek1 != '{') {
             current_tok->curly_bracket_expr_start_depth++;
@@ -460,13 +407,6 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
         }
     }
 
-    if (current_tok->last_expr_buffer != NULL) {
-        PyMem_Free(current_tok->last_expr_buffer);
-        current_tok->last_expr_buffer = NULL;
-        current_tok->last_expr_size = 0;
-        current_tok->last_expr_end = -1;
-    }
-
     p_start = tok->start;
     p_end = tok->cur;
     tok->tok_mode_stack_index--;
@@ -551,12 +491,10 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
         }
 
         if (c == '{') {
-            if (!_PyLexer_update_ftstring_expr(tok, c)) {
-                return MAKE_TOKEN(ENDMARKER);
-            }
             int peek = tok_nextc(tok);
             if (peek != '{' || in_format_spec) {
                 tok_backup(tok, peek);
+                current_tok->last_expr_start = tok->cur;
                 tok_backup(tok, c);
                 current_tok->curly_bracket_expr_start_depth++;
                 if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) {
@@ -580,10 +518,10 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
             }
             int peek = tok_nextc(tok);
 
-            // The tokenizer can only be in the format spec if we have already completed the expression
-            // scanning (indicated by the end of the expression being set) and we are not at the top level
-            // of the bracket stack (-1 is the top level). Since format specifiers can't legally use double
-            // brackets, we can bypass it here.
+            // The tokenizer can only be in the format spec if expression
+            // scanning is complete and we are not at the top level of the
+            // bracket stack (-1 is the top level). Since format specifiers
+            // can't legally use double brackets, we can bypass it here.
             int cursor = current_tok->curly_bracket_depth;
             if (peek == '}' && !in_format_spec && cursor == 0) {
                 p_start = tok->start;
diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c
index 82b824f56374fcf..e2973f111ae0f92 100644
--- a/Parser/tokenizer/reader.c
+++ b/Parser/tokenizer/reader.c
@@ -604,12 +604,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
     }
     tok->implicit_newline = chunk.implicit_newline;
 
-    if (!prepared && tok->tok_mode_stack_index &&
-            !_PyLexer_update_ftstring_expr(tok, 0)) {
-        _PyTok_ChunkClear(&chunk);
-        tok->input_error = 1;
-        return 0;
-    }
     ADVANCE_LINENO();
     if (tok->reader->kind == _PYTOK_READER_FILE &&
             (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&

From f39a8eb8bc4430c66f07d3f60aeac5c960e0395d Mon Sep 17 00:00:00 2001
From: gwosti <322952417+gwosti@users.noreply.github.com>
Date: Tue, 1 Sep 2026 12:59:06 +0200
Subject: [PATCH 2/2] fixup! gh-155525: Avoid quadratic f-string tokenization

---
 Lib/test/test_fstring.py |  9 ++++++++-
 Lib/test/test_tstring.py | 14 +++++++++++++-
 Parser/lexer/state.h     |  3 +++
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index 7b76ec76260f7da..ced210728a9e249 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -838,7 +838,11 @@ def test_many_fstrings_in_module(self):
         source = ''.join(
             f"value_{i} = f'{fields}'\n" for i in range(1_000)
         )
-        compile(source, '<string>', 'exec')
+        namespace = {f'x{i}': str(i) for i in range(100)}
+        expected = ''.join(str(i) for i in range(100))
+        exec(source, namespace)
+        self.assertEqual(namespace['value_0'], expected)
+        self.assertEqual(namespace['value_999'], expected)
 
     def test_format_specifier_expressions(self):
         width = 10
@@ -1356,6 +1360,9 @@ def test_not_equal(self):
         self.assertEqual(f'{3!=4:}', 'True')
         self.assertEqual(f'{3!=4!s}', 'True')
         self.assertEqual(f'{3!=4!s:.3}', 'Tru')
+        a = 3
+        b = 4
+        self.assertEqual(f'{a!=b=:>10}', 'a!=b=         1')
 
     def test_equal_equal(self):
         # Because an expression ending in = has special meaning,
diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py
index 6ea0f5bcc157df5..b9b2e77c8ffde1c 100644
--- a/Lib/test/test_tstring.py
+++ b/Lib/test/test_tstring.py
@@ -101,7 +101,11 @@ def test_many_tstrings_in_module(self):
         source = ''.join(
             f"value_{i} = t'{fields}'\n" for i in range(1_000)
         )
-        compile(source, '<string>', 'exec')
+        namespace = {f'x{i}': str(i) for i in range(100)}
+        expected = ''.join(str(i) for i in range(100))
+        exec(source, namespace)
+        self.assertEqual(fstring(namespace['value_0']), expected)
+        self.assertEqual(fstring(namespace['value_999']), expected)
 
     def test_format_specifiers(self):
         # Test basic format specifiers
@@ -112,6 +116,14 @@ def test_format_specifiers(self):
         )
         self.assertEqual(fstring(t), "Pi: 3.14")
 
+        a = 3
+        b = 4
+        t = t"{a!=b:>10}"
+        self.assertTStringEqual(
+            t, ("", ""), [(a != b, "a!=b", None, ">10")]
+        )
+        self.assertEqual(fstring(t), "         1")
+
     def test_conversions(self):
         # Test !s conversion (str)
         obj = object()
diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h
index 3d94725540d80be..2474e67e5d44bea 100644
--- a/Parser/lexer/state.h
+++ b/Parser/lexer/state.h
@@ -56,8 +56,11 @@ typedef struct _tokenizer_mode {
     Py_ssize_t start_offset;
     Py_ssize_t multi_line_start_offset;
 
+    /* Points into tok->buf: relies on _PyTok_ReaderUnderflow()
+       not resetting the buffer while INSIDE_FSTRING(tok) */
     const char* last_expr_start;
     Py_ssize_t last_expr_start_offset;
+
     int in_debug;
     int in_format_spec;