From f523730a96029408e681a163333f0b5e6a92889e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:13:21 +0100 Subject: [PATCH 1/5] feat(native): reproduce each supported Python's text semantics natively Native discovery deferred to Python whenever the running Python's Unicode data differed from the one table it carried (CPython 3.12), so 3.13 and 3.14 never ran natively. Generate a `str.isalnum()` table for the Unicode version of each supported CPython (15.0.0, 15.1.0, 16.0.0), select it by `unicodedata.unidata_version` per call, and reproduce `inspect.cleandoc` per Python minor version (3.13 strips spaces only). Unknown Python or Unicode versions still defer to Python. --- .../src/text/_helpers/alnum_tables.rs | 8 + crates/sqlbuild-core/src/text/_helpers/mod.rs | 2 +- .../unicode_15_0_0.in} | 12 +- .../src/text/alnum_ranges/unicode_15_1_0.in | 752 +++++++++++++++++ .../src/text/alnum_ranges/unicode_16_0_0.in | 774 ++++++++++++++++++ .../src/text/main/is_python_alnum.rs | 11 +- .../src/text/main/is_python_word.rs | 5 +- crates/sqlbuild-core/src/text/main/mod.rs | 2 +- .../text/main/python_alnum_unicode_version.rs | 8 - .../src/text/main/python_cleandoc.rs | 13 +- .../src/text/main/python_text.rs | 21 + crates/sqlbuild-core/src/text/models.rs | 17 + .../src/text/tests/test_python_compat.rs | 101 ++- .../src/text/tests/test_types.rs | 16 + .../src/_helpers/header_keys.rs | 37 +- .../src/_helpers/statement_headers.rs | 3 + .../model_files/_helpers/output_columns.rs | 19 +- .../src/model_files/_helpers/parsing.rs | 8 +- .../src/model_files/models.rs | 3 + .../src/model_files/tests/helpers.rs | 2 + .../sql_tests/_helpers/scenario_parsing.rs | 9 +- .../src/sql_tests/_helpers/test_blocks.rs | 8 +- .../sql_tests/main/discover_scenario_files.rs | 2 +- .../sql_tests/main/discover_sql_test_files.rs | 2 +- .../src/sql_tests/models.rs | 4 +- .../src/sql_tests/tests/helpers.rs | 53 +- .../src/bindings/_helpers/discovery.rs | 35 +- .../src/bindings/_helpers/oracles.rs | 31 +- crates/sqlbuild-python/src/bindings/models.rs | 4 + scripts/generate_python_char_table.py | 2 +- scripts/python_char_table/_helpers/table.py | 30 +- scripts/python_char_table/constants.py | 7 +- scripts/python_char_table/main/generate.py | 13 +- src/sqlbuild/_native.pyi | 11 +- .../discovery/_helpers/native/model_files.py | 2 + .../discovery/_helpers/native/payloads.py | 15 +- .../_helpers/native/sql_test_files.py | 2 + .../compiler/discovery/_test_types.py | 9 +- .../discovery/test_native_model_discovery.py | 16 + .../test_native_yaml_file_discovery.py | 3 +- .../src/sqlbuild/compiler/helpers.py | 1 - .../test_native_python_compat_oracle.py | 23 +- 42 files changed, 1953 insertions(+), 143 deletions(-) create mode 100644 crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs rename crates/sqlbuild-core/src/text/{_helpers/alnum_ranges.rs => alnum_ranges/unicode_15_0_0.in} (98%) create mode 100644 crates/sqlbuild-core/src/text/alnum_ranges/unicode_15_1_0.in create mode 100644 crates/sqlbuild-core/src/text/alnum_ranges/unicode_16_0_0.in delete mode 100644 crates/sqlbuild-core/src/text/main/python_alnum_unicode_version.rs create mode 100644 crates/sqlbuild-core/src/text/main/python_text.rs diff --git a/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs b/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs new file mode 100644 index 0000000000..cf4cef94a7 --- /dev/null +++ b/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs @@ -0,0 +1,8 @@ +//! The generated `str.isalnum()` tables, one per Unicode version of a supported CPython. + +/// `(unicodedata.unidata_version, ranges)` for every table native text helpers can select. +pub(crate) const ALNUM_TABLES: [(&str, &[(u32, u32)]); 3] = [ + ("15.0.0", include!("../alnum_ranges/unicode_15_0_0.in")), + ("15.1.0", include!("../alnum_ranges/unicode_15_1_0.in")), + ("16.0.0", include!("../alnum_ranges/unicode_16_0_0.in")), +]; diff --git a/crates/sqlbuild-core/src/text/_helpers/mod.rs b/crates/sqlbuild-core/src/text/_helpers/mod.rs index 579d227f76..0a2f6ee237 100644 --- a/crates/sqlbuild-core/src/text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/text/_helpers/mod.rs @@ -1,3 +1,3 @@ -pub(crate) mod alnum_ranges; +pub(crate) mod alnum_tables; pub(crate) mod cleandoc; pub(crate) mod sequence_matcher; diff --git a/crates/sqlbuild-core/src/text/_helpers/alnum_ranges.rs b/crates/sqlbuild-core/src/text/alnum_ranges/unicode_15_0_0.in similarity index 98% rename from crates/sqlbuild-core/src/text/_helpers/alnum_ranges.rs rename to crates/sqlbuild-core/src/text/alnum_ranges/unicode_15_0_0.in index 8c4880334d..ff978e1700 100644 --- a/crates/sqlbuild-core/src/text/_helpers/alnum_ranges.rs +++ b/crates/sqlbuild-core/src/text/alnum_ranges/unicode_15_0_0.in @@ -1,10 +1,6 @@ -//! Generated by `scripts/generate_python_char_table.py`; do not edit. - -/// The Unicode version of the Python that generated the table. -pub(crate) const PYTHON_ALNUM_UNICODE_VERSION: &str = "15.0.0"; - -/// Inclusive code point ranges where Python's `str.isalnum()` is true, in order. -pub(crate) const PYTHON_ALNUM_RANGES: &[(u32, u32)] = &[ +// Generated by `scripts/generate_python_char_table.py`; do not edit. +// Inclusive code point ranges where `str.isalnum()` is true in Unicode 15.0.0, in order. +&[ (0x0030, 0x0039), (0x0041, 0x005A), (0x0061, 0x007A), @@ -752,4 +748,4 @@ pub(crate) const PYTHON_ALNUM_RANGES: &[(u32, u32)] = &[ (0x2F800, 0x2FA1D), (0x30000, 0x3134A), (0x31350, 0x323AF), -]; +] diff --git a/crates/sqlbuild-core/src/text/alnum_ranges/unicode_15_1_0.in b/crates/sqlbuild-core/src/text/alnum_ranges/unicode_15_1_0.in new file mode 100644 index 0000000000..ca3d653f11 --- /dev/null +++ b/crates/sqlbuild-core/src/text/alnum_ranges/unicode_15_1_0.in @@ -0,0 +1,752 @@ +// Generated by `scripts/generate_python_char_table.py`; do not edit. +// Inclusive code point ranges where `str.isalnum()` is true in Unicode 15.1.0, in order. +&[ + (0x0030, 0x0039), + (0x0041, 0x005A), + (0x0061, 0x007A), + (0x00AA, 0x00AA), + (0x00B2, 0x00B3), + (0x00B5, 0x00B5), + (0x00B9, 0x00BA), + (0x00BC, 0x00BE), + (0x00C0, 0x00D6), + (0x00D8, 0x00F6), + (0x00F8, 0x02C1), + (0x02C6, 0x02D1), + (0x02E0, 0x02E4), + (0x02EC, 0x02EC), + (0x02EE, 0x02EE), + (0x0370, 0x0374), + (0x0376, 0x0377), + (0x037A, 0x037D), + (0x037F, 0x037F), + (0x0386, 0x0386), + (0x0388, 0x038A), + (0x038C, 0x038C), + (0x038E, 0x03A1), + (0x03A3, 0x03F5), + (0x03F7, 0x0481), + (0x048A, 0x052F), + (0x0531, 0x0556), + (0x0559, 0x0559), + (0x0560, 0x0588), + (0x05D0, 0x05EA), + (0x05EF, 0x05F2), + (0x0620, 0x064A), + (0x0660, 0x0669), + (0x066E, 0x066F), + (0x0671, 0x06D3), + (0x06D5, 0x06D5), + (0x06E5, 0x06E6), + (0x06EE, 0x06FC), + (0x06FF, 0x06FF), + (0x0710, 0x0710), + (0x0712, 0x072F), + (0x074D, 0x07A5), + (0x07B1, 0x07B1), + (0x07C0, 0x07EA), + (0x07F4, 0x07F5), + (0x07FA, 0x07FA), + (0x0800, 0x0815), + (0x081A, 0x081A), + (0x0824, 0x0824), + (0x0828, 0x0828), + (0x0840, 0x0858), + (0x0860, 0x086A), + (0x0870, 0x0887), + (0x0889, 0x088E), + (0x08A0, 0x08C9), + (0x0904, 0x0939), + (0x093D, 0x093D), + (0x0950, 0x0950), + (0x0958, 0x0961), + (0x0966, 0x096F), + (0x0971, 0x0980), + (0x0985, 0x098C), + (0x098F, 0x0990), + (0x0993, 0x09A8), + (0x09AA, 0x09B0), + (0x09B2, 0x09B2), + (0x09B6, 0x09B9), + (0x09BD, 0x09BD), + (0x09CE, 0x09CE), + (0x09DC, 0x09DD), + (0x09DF, 0x09E1), + (0x09E6, 0x09F1), + (0x09F4, 0x09F9), + (0x09FC, 0x09FC), + (0x0A05, 0x0A0A), + (0x0A0F, 0x0A10), + (0x0A13, 0x0A28), + (0x0A2A, 0x0A30), + (0x0A32, 0x0A33), + (0x0A35, 0x0A36), + (0x0A38, 0x0A39), + (0x0A59, 0x0A5C), + (0x0A5E, 0x0A5E), + (0x0A66, 0x0A6F), + (0x0A72, 0x0A74), + (0x0A85, 0x0A8D), + (0x0A8F, 0x0A91), + (0x0A93, 0x0AA8), + (0x0AAA, 0x0AB0), + (0x0AB2, 0x0AB3), + (0x0AB5, 0x0AB9), + (0x0ABD, 0x0ABD), + (0x0AD0, 0x0AD0), + (0x0AE0, 0x0AE1), + (0x0AE6, 0x0AEF), + (0x0AF9, 0x0AF9), + (0x0B05, 0x0B0C), + (0x0B0F, 0x0B10), + (0x0B13, 0x0B28), + (0x0B2A, 0x0B30), + (0x0B32, 0x0B33), + (0x0B35, 0x0B39), + (0x0B3D, 0x0B3D), + (0x0B5C, 0x0B5D), + (0x0B5F, 0x0B61), + (0x0B66, 0x0B6F), + (0x0B71, 0x0B77), + (0x0B83, 0x0B83), + (0x0B85, 0x0B8A), + (0x0B8E, 0x0B90), + (0x0B92, 0x0B95), + (0x0B99, 0x0B9A), + (0x0B9C, 0x0B9C), + (0x0B9E, 0x0B9F), + (0x0BA3, 0x0BA4), + (0x0BA8, 0x0BAA), + (0x0BAE, 0x0BB9), + (0x0BD0, 0x0BD0), + (0x0BE6, 0x0BF2), + (0x0C05, 0x0C0C), + (0x0C0E, 0x0C10), + (0x0C12, 0x0C28), + (0x0C2A, 0x0C39), + (0x0C3D, 0x0C3D), + (0x0C58, 0x0C5A), + (0x0C5D, 0x0C5D), + (0x0C60, 0x0C61), + (0x0C66, 0x0C6F), + (0x0C78, 0x0C7E), + (0x0C80, 0x0C80), + (0x0C85, 0x0C8C), + (0x0C8E, 0x0C90), + (0x0C92, 0x0CA8), + (0x0CAA, 0x0CB3), + (0x0CB5, 0x0CB9), + (0x0CBD, 0x0CBD), + (0x0CDD, 0x0CDE), + (0x0CE0, 0x0CE1), + (0x0CE6, 0x0CEF), + (0x0CF1, 0x0CF2), + (0x0D04, 0x0D0C), + (0x0D0E, 0x0D10), + (0x0D12, 0x0D3A), + (0x0D3D, 0x0D3D), + (0x0D4E, 0x0D4E), + (0x0D54, 0x0D56), + (0x0D58, 0x0D61), + (0x0D66, 0x0D78), + (0x0D7A, 0x0D7F), + (0x0D85, 0x0D96), + (0x0D9A, 0x0DB1), + (0x0DB3, 0x0DBB), + (0x0DBD, 0x0DBD), + (0x0DC0, 0x0DC6), + (0x0DE6, 0x0DEF), + (0x0E01, 0x0E30), + (0x0E32, 0x0E33), + (0x0E40, 0x0E46), + (0x0E50, 0x0E59), + (0x0E81, 0x0E82), + (0x0E84, 0x0E84), + (0x0E86, 0x0E8A), + (0x0E8C, 0x0EA3), + (0x0EA5, 0x0EA5), + (0x0EA7, 0x0EB0), + (0x0EB2, 0x0EB3), + (0x0EBD, 0x0EBD), + (0x0EC0, 0x0EC4), + (0x0EC6, 0x0EC6), + (0x0ED0, 0x0ED9), + (0x0EDC, 0x0EDF), + (0x0F00, 0x0F00), + (0x0F20, 0x0F33), + (0x0F40, 0x0F47), + (0x0F49, 0x0F6C), + (0x0F88, 0x0F8C), + (0x1000, 0x102A), + (0x103F, 0x1049), + (0x1050, 0x1055), + (0x105A, 0x105D), + (0x1061, 0x1061), + (0x1065, 0x1066), + (0x106E, 0x1070), + (0x1075, 0x1081), + (0x108E, 0x108E), + (0x1090, 0x1099), + (0x10A0, 0x10C5), + (0x10C7, 0x10C7), + (0x10CD, 0x10CD), + (0x10D0, 0x10FA), + (0x10FC, 0x1248), + (0x124A, 0x124D), + (0x1250, 0x1256), + (0x1258, 0x1258), + (0x125A, 0x125D), + (0x1260, 0x1288), + (0x128A, 0x128D), + (0x1290, 0x12B0), + (0x12B2, 0x12B5), + (0x12B8, 0x12BE), + (0x12C0, 0x12C0), + (0x12C2, 0x12C5), + (0x12C8, 0x12D6), + (0x12D8, 0x1310), + (0x1312, 0x1315), + (0x1318, 0x135A), + (0x1369, 0x137C), + (0x1380, 0x138F), + (0x13A0, 0x13F5), + (0x13F8, 0x13FD), + (0x1401, 0x166C), + (0x166F, 0x167F), + (0x1681, 0x169A), + (0x16A0, 0x16EA), + (0x16EE, 0x16F8), + (0x1700, 0x1711), + (0x171F, 0x1731), + (0x1740, 0x1751), + (0x1760, 0x176C), + (0x176E, 0x1770), + (0x1780, 0x17B3), + (0x17D7, 0x17D7), + (0x17DC, 0x17DC), + (0x17E0, 0x17E9), + (0x17F0, 0x17F9), + (0x1810, 0x1819), + (0x1820, 0x1878), + (0x1880, 0x1884), + (0x1887, 0x18A8), + (0x18AA, 0x18AA), + (0x18B0, 0x18F5), + (0x1900, 0x191E), + (0x1946, 0x196D), + (0x1970, 0x1974), + (0x1980, 0x19AB), + (0x19B0, 0x19C9), + (0x19D0, 0x19DA), + (0x1A00, 0x1A16), + (0x1A20, 0x1A54), + (0x1A80, 0x1A89), + (0x1A90, 0x1A99), + (0x1AA7, 0x1AA7), + (0x1B05, 0x1B33), + (0x1B45, 0x1B4C), + (0x1B50, 0x1B59), + (0x1B83, 0x1BA0), + (0x1BAE, 0x1BE5), + (0x1C00, 0x1C23), + (0x1C40, 0x1C49), + (0x1C4D, 0x1C7D), + (0x1C80, 0x1C88), + (0x1C90, 0x1CBA), + (0x1CBD, 0x1CBF), + (0x1CE9, 0x1CEC), + (0x1CEE, 0x1CF3), + (0x1CF5, 0x1CF6), + (0x1CFA, 0x1CFA), + (0x1D00, 0x1DBF), + (0x1E00, 0x1F15), + (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), + (0x1F48, 0x1F4D), + (0x1F50, 0x1F57), + (0x1F59, 0x1F59), + (0x1F5B, 0x1F5B), + (0x1F5D, 0x1F5D), + (0x1F5F, 0x1F7D), + (0x1F80, 0x1FB4), + (0x1FB6, 0x1FBC), + (0x1FBE, 0x1FBE), + (0x1FC2, 0x1FC4), + (0x1FC6, 0x1FCC), + (0x1FD0, 0x1FD3), + (0x1FD6, 0x1FDB), + (0x1FE0, 0x1FEC), + (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFC), + (0x2070, 0x2071), + (0x2074, 0x2079), + (0x207F, 0x2089), + (0x2090, 0x209C), + (0x2102, 0x2102), + (0x2107, 0x2107), + (0x210A, 0x2113), + (0x2115, 0x2115), + (0x2119, 0x211D), + (0x2124, 0x2124), + (0x2126, 0x2126), + (0x2128, 0x2128), + (0x212A, 0x212D), + (0x212F, 0x2139), + (0x213C, 0x213F), + (0x2145, 0x2149), + (0x214E, 0x214E), + (0x2150, 0x2189), + (0x2460, 0x249B), + (0x24EA, 0x24FF), + (0x2776, 0x2793), + (0x2C00, 0x2CE4), + (0x2CEB, 0x2CEE), + (0x2CF2, 0x2CF3), + (0x2CFD, 0x2CFD), + (0x2D00, 0x2D25), + (0x2D27, 0x2D27), + (0x2D2D, 0x2D2D), + (0x2D30, 0x2D67), + (0x2D6F, 0x2D6F), + (0x2D80, 0x2D96), + (0x2DA0, 0x2DA6), + (0x2DA8, 0x2DAE), + (0x2DB0, 0x2DB6), + (0x2DB8, 0x2DBE), + (0x2DC0, 0x2DC6), + (0x2DC8, 0x2DCE), + (0x2DD0, 0x2DD6), + (0x2DD8, 0x2DDE), + (0x2E2F, 0x2E2F), + (0x3005, 0x3007), + (0x3021, 0x3029), + (0x3031, 0x3035), + (0x3038, 0x303C), + (0x3041, 0x3096), + (0x309D, 0x309F), + (0x30A1, 0x30FA), + (0x30FC, 0x30FF), + (0x3105, 0x312F), + (0x3131, 0x318E), + (0x3192, 0x3195), + (0x31A0, 0x31BF), + (0x31F0, 0x31FF), + (0x3220, 0x3229), + (0x3248, 0x324F), + (0x3251, 0x325F), + (0x3280, 0x3289), + (0x32B1, 0x32BF), + (0x3400, 0x4DBF), + (0x4E00, 0xA48C), + (0xA4D0, 0xA4FD), + (0xA500, 0xA60C), + (0xA610, 0xA62B), + (0xA640, 0xA66E), + (0xA67F, 0xA69D), + (0xA6A0, 0xA6EF), + (0xA717, 0xA71F), + (0xA722, 0xA788), + (0xA78B, 0xA7CA), + (0xA7D0, 0xA7D1), + (0xA7D3, 0xA7D3), + (0xA7D5, 0xA7D9), + (0xA7F2, 0xA801), + (0xA803, 0xA805), + (0xA807, 0xA80A), + (0xA80C, 0xA822), + (0xA830, 0xA835), + (0xA840, 0xA873), + (0xA882, 0xA8B3), + (0xA8D0, 0xA8D9), + (0xA8F2, 0xA8F7), + (0xA8FB, 0xA8FB), + (0xA8FD, 0xA8FE), + (0xA900, 0xA925), + (0xA930, 0xA946), + (0xA960, 0xA97C), + (0xA984, 0xA9B2), + (0xA9CF, 0xA9D9), + (0xA9E0, 0xA9E4), + (0xA9E6, 0xA9FE), + (0xAA00, 0xAA28), + (0xAA40, 0xAA42), + (0xAA44, 0xAA4B), + (0xAA50, 0xAA59), + (0xAA60, 0xAA76), + (0xAA7A, 0xAA7A), + (0xAA7E, 0xAAAF), + (0xAAB1, 0xAAB1), + (0xAAB5, 0xAAB6), + (0xAAB9, 0xAABD), + (0xAAC0, 0xAAC0), + (0xAAC2, 0xAAC2), + (0xAADB, 0xAADD), + (0xAAE0, 0xAAEA), + (0xAAF2, 0xAAF4), + (0xAB01, 0xAB06), + (0xAB09, 0xAB0E), + (0xAB11, 0xAB16), + (0xAB20, 0xAB26), + (0xAB28, 0xAB2E), + (0xAB30, 0xAB5A), + (0xAB5C, 0xAB69), + (0xAB70, 0xABE2), + (0xABF0, 0xABF9), + (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), + (0xD7CB, 0xD7FB), + (0xF900, 0xFA6D), + (0xFA70, 0xFAD9), + (0xFB00, 0xFB06), + (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), + (0xFB1F, 0xFB28), + (0xFB2A, 0xFB36), + (0xFB38, 0xFB3C), + (0xFB3E, 0xFB3E), + (0xFB40, 0xFB41), + (0xFB43, 0xFB44), + (0xFB46, 0xFBB1), + (0xFBD3, 0xFD3D), + (0xFD50, 0xFD8F), + (0xFD92, 0xFDC7), + (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), + (0xFE76, 0xFEFC), + (0xFF10, 0xFF19), + (0xFF21, 0xFF3A), + (0xFF41, 0xFF5A), + (0xFF66, 0xFFBE), + (0xFFC2, 0xFFC7), + (0xFFCA, 0xFFCF), + (0xFFD2, 0xFFD7), + (0xFFDA, 0xFFDC), + (0x10000, 0x1000B), + (0x1000D, 0x10026), + (0x10028, 0x1003A), + (0x1003C, 0x1003D), + (0x1003F, 0x1004D), + (0x10050, 0x1005D), + (0x10080, 0x100FA), + (0x10107, 0x10133), + (0x10140, 0x10178), + (0x1018A, 0x1018B), + (0x10280, 0x1029C), + (0x102A0, 0x102D0), + (0x102E1, 0x102FB), + (0x10300, 0x10323), + (0x1032D, 0x1034A), + (0x10350, 0x10375), + (0x10380, 0x1039D), + (0x103A0, 0x103C3), + (0x103C8, 0x103CF), + (0x103D1, 0x103D5), + (0x10400, 0x1049D), + (0x104A0, 0x104A9), + (0x104B0, 0x104D3), + (0x104D8, 0x104FB), + (0x10500, 0x10527), + (0x10530, 0x10563), + (0x10570, 0x1057A), + (0x1057C, 0x1058A), + (0x1058C, 0x10592), + (0x10594, 0x10595), + (0x10597, 0x105A1), + (0x105A3, 0x105B1), + (0x105B3, 0x105B9), + (0x105BB, 0x105BC), + (0x10600, 0x10736), + (0x10740, 0x10755), + (0x10760, 0x10767), + (0x10780, 0x10785), + (0x10787, 0x107B0), + (0x107B2, 0x107BA), + (0x10800, 0x10805), + (0x10808, 0x10808), + (0x1080A, 0x10835), + (0x10837, 0x10838), + (0x1083C, 0x1083C), + (0x1083F, 0x10855), + (0x10858, 0x10876), + (0x10879, 0x1089E), + (0x108A7, 0x108AF), + (0x108E0, 0x108F2), + (0x108F4, 0x108F5), + (0x108FB, 0x1091B), + (0x10920, 0x10939), + (0x10980, 0x109B7), + (0x109BC, 0x109CF), + (0x109D2, 0x10A00), + (0x10A10, 0x10A13), + (0x10A15, 0x10A17), + (0x10A19, 0x10A35), + (0x10A40, 0x10A48), + (0x10A60, 0x10A7E), + (0x10A80, 0x10A9F), + (0x10AC0, 0x10AC7), + (0x10AC9, 0x10AE4), + (0x10AEB, 0x10AEF), + (0x10B00, 0x10B35), + (0x10B40, 0x10B55), + (0x10B58, 0x10B72), + (0x10B78, 0x10B91), + (0x10BA9, 0x10BAF), + (0x10C00, 0x10C48), + (0x10C80, 0x10CB2), + (0x10CC0, 0x10CF2), + (0x10CFA, 0x10D23), + (0x10D30, 0x10D39), + (0x10E60, 0x10E7E), + (0x10E80, 0x10EA9), + (0x10EB0, 0x10EB1), + (0x10F00, 0x10F27), + (0x10F30, 0x10F45), + (0x10F51, 0x10F54), + (0x10F70, 0x10F81), + (0x10FB0, 0x10FCB), + (0x10FE0, 0x10FF6), + (0x11003, 0x11037), + (0x11052, 0x1106F), + (0x11071, 0x11072), + (0x11075, 0x11075), + (0x11083, 0x110AF), + (0x110D0, 0x110E8), + (0x110F0, 0x110F9), + (0x11103, 0x11126), + (0x11136, 0x1113F), + (0x11144, 0x11144), + (0x11147, 0x11147), + (0x11150, 0x11172), + (0x11176, 0x11176), + (0x11183, 0x111B2), + (0x111C1, 0x111C4), + (0x111D0, 0x111DA), + (0x111DC, 0x111DC), + (0x111E1, 0x111F4), + (0x11200, 0x11211), + (0x11213, 0x1122B), + (0x1123F, 0x11240), + (0x11280, 0x11286), + (0x11288, 0x11288), + (0x1128A, 0x1128D), + (0x1128F, 0x1129D), + (0x1129F, 0x112A8), + (0x112B0, 0x112DE), + (0x112F0, 0x112F9), + (0x11305, 0x1130C), + (0x1130F, 0x11310), + (0x11313, 0x11328), + (0x1132A, 0x11330), + (0x11332, 0x11333), + (0x11335, 0x11339), + (0x1133D, 0x1133D), + (0x11350, 0x11350), + (0x1135D, 0x11361), + (0x11400, 0x11434), + (0x11447, 0x1144A), + (0x11450, 0x11459), + (0x1145F, 0x11461), + (0x11480, 0x114AF), + (0x114C4, 0x114C5), + (0x114C7, 0x114C7), + (0x114D0, 0x114D9), + (0x11580, 0x115AE), + (0x115D8, 0x115DB), + (0x11600, 0x1162F), + (0x11644, 0x11644), + (0x11650, 0x11659), + (0x11680, 0x116AA), + (0x116B8, 0x116B8), + (0x116C0, 0x116C9), + (0x11700, 0x1171A), + (0x11730, 0x1173B), + (0x11740, 0x11746), + (0x11800, 0x1182B), + (0x118A0, 0x118F2), + (0x118FF, 0x11906), + (0x11909, 0x11909), + (0x1190C, 0x11913), + (0x11915, 0x11916), + (0x11918, 0x1192F), + (0x1193F, 0x1193F), + (0x11941, 0x11941), + (0x11950, 0x11959), + (0x119A0, 0x119A7), + (0x119AA, 0x119D0), + (0x119E1, 0x119E1), + (0x119E3, 0x119E3), + (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), + (0x11A3A, 0x11A3A), + (0x11A50, 0x11A50), + (0x11A5C, 0x11A89), + (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), + (0x11C00, 0x11C08), + (0x11C0A, 0x11C2E), + (0x11C40, 0x11C40), + (0x11C50, 0x11C6C), + (0x11C72, 0x11C8F), + (0x11D00, 0x11D06), + (0x11D08, 0x11D09), + (0x11D0B, 0x11D30), + (0x11D46, 0x11D46), + (0x11D50, 0x11D59), + (0x11D60, 0x11D65), + (0x11D67, 0x11D68), + (0x11D6A, 0x11D89), + (0x11D98, 0x11D98), + (0x11DA0, 0x11DA9), + (0x11EE0, 0x11EF2), + (0x11F02, 0x11F02), + (0x11F04, 0x11F10), + (0x11F12, 0x11F33), + (0x11F50, 0x11F59), + (0x11FB0, 0x11FB0), + (0x11FC0, 0x11FD4), + (0x12000, 0x12399), + (0x12400, 0x1246E), + (0x12480, 0x12543), + (0x12F90, 0x12FF0), + (0x13000, 0x1342F), + (0x13441, 0x13446), + (0x14400, 0x14646), + (0x16800, 0x16A38), + (0x16A40, 0x16A5E), + (0x16A60, 0x16A69), + (0x16A70, 0x16ABE), + (0x16AC0, 0x16AC9), + (0x16AD0, 0x16AED), + (0x16B00, 0x16B2F), + (0x16B40, 0x16B43), + (0x16B50, 0x16B59), + (0x16B5B, 0x16B61), + (0x16B63, 0x16B77), + (0x16B7D, 0x16B8F), + (0x16E40, 0x16E96), + (0x16F00, 0x16F4A), + (0x16F50, 0x16F50), + (0x16F93, 0x16F9F), + (0x16FE0, 0x16FE1), + (0x16FE3, 0x16FE3), + (0x17000, 0x187F7), + (0x18800, 0x18CD5), + (0x18D00, 0x18D08), + (0x1AFF0, 0x1AFF3), + (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), + (0x1B000, 0x1B122), + (0x1B132, 0x1B132), + (0x1B150, 0x1B152), + (0x1B155, 0x1B155), + (0x1B164, 0x1B167), + (0x1B170, 0x1B2FB), + (0x1BC00, 0x1BC6A), + (0x1BC70, 0x1BC7C), + (0x1BC80, 0x1BC88), + (0x1BC90, 0x1BC99), + (0x1D2C0, 0x1D2D3), + (0x1D2E0, 0x1D2F3), + (0x1D360, 0x1D378), + (0x1D400, 0x1D454), + (0x1D456, 0x1D49C), + (0x1D49E, 0x1D49F), + (0x1D4A2, 0x1D4A2), + (0x1D4A5, 0x1D4A6), + (0x1D4A9, 0x1D4AC), + (0x1D4AE, 0x1D4B9), + (0x1D4BB, 0x1D4BB), + (0x1D4BD, 0x1D4C3), + (0x1D4C5, 0x1D505), + (0x1D507, 0x1D50A), + (0x1D50D, 0x1D514), + (0x1D516, 0x1D51C), + (0x1D51E, 0x1D539), + (0x1D53B, 0x1D53E), + (0x1D540, 0x1D544), + (0x1D546, 0x1D546), + (0x1D54A, 0x1D550), + (0x1D552, 0x1D6A5), + (0x1D6A8, 0x1D6C0), + (0x1D6C2, 0x1D6DA), + (0x1D6DC, 0x1D6FA), + (0x1D6FC, 0x1D714), + (0x1D716, 0x1D734), + (0x1D736, 0x1D74E), + (0x1D750, 0x1D76E), + (0x1D770, 0x1D788), + (0x1D78A, 0x1D7A8), + (0x1D7AA, 0x1D7C2), + (0x1D7C4, 0x1D7CB), + (0x1D7CE, 0x1D7FF), + (0x1DF00, 0x1DF1E), + (0x1DF25, 0x1DF2A), + (0x1E030, 0x1E06D), + (0x1E100, 0x1E12C), + (0x1E137, 0x1E13D), + (0x1E140, 0x1E149), + (0x1E14E, 0x1E14E), + (0x1E290, 0x1E2AD), + (0x1E2C0, 0x1E2EB), + (0x1E2F0, 0x1E2F9), + (0x1E4D0, 0x1E4EB), + (0x1E4F0, 0x1E4F9), + (0x1E7E0, 0x1E7E6), + (0x1E7E8, 0x1E7EB), + (0x1E7ED, 0x1E7EE), + (0x1E7F0, 0x1E7FE), + (0x1E800, 0x1E8C4), + (0x1E8C7, 0x1E8CF), + (0x1E900, 0x1E943), + (0x1E94B, 0x1E94B), + (0x1E950, 0x1E959), + (0x1EC71, 0x1ECAB), + (0x1ECAD, 0x1ECAF), + (0x1ECB1, 0x1ECB4), + (0x1ED01, 0x1ED2D), + (0x1ED2F, 0x1ED3D), + (0x1EE00, 0x1EE03), + (0x1EE05, 0x1EE1F), + (0x1EE21, 0x1EE22), + (0x1EE24, 0x1EE24), + (0x1EE27, 0x1EE27), + (0x1EE29, 0x1EE32), + (0x1EE34, 0x1EE37), + (0x1EE39, 0x1EE39), + (0x1EE3B, 0x1EE3B), + (0x1EE42, 0x1EE42), + (0x1EE47, 0x1EE47), + (0x1EE49, 0x1EE49), + (0x1EE4B, 0x1EE4B), + (0x1EE4D, 0x1EE4F), + (0x1EE51, 0x1EE52), + (0x1EE54, 0x1EE54), + (0x1EE57, 0x1EE57), + (0x1EE59, 0x1EE59), + (0x1EE5B, 0x1EE5B), + (0x1EE5D, 0x1EE5D), + (0x1EE5F, 0x1EE5F), + (0x1EE61, 0x1EE62), + (0x1EE64, 0x1EE64), + (0x1EE67, 0x1EE6A), + (0x1EE6C, 0x1EE72), + (0x1EE74, 0x1EE77), + (0x1EE79, 0x1EE7C), + (0x1EE7E, 0x1EE7E), + (0x1EE80, 0x1EE89), + (0x1EE8B, 0x1EE9B), + (0x1EEA1, 0x1EEA3), + (0x1EEA5, 0x1EEA9), + (0x1EEAB, 0x1EEBB), + (0x1F100, 0x1F10C), + (0x1FBF0, 0x1FBF9), + (0x20000, 0x2A6DF), + (0x2A700, 0x2B739), + (0x2B740, 0x2B81D), + (0x2B820, 0x2CEA1), + (0x2CEB0, 0x2EBE0), + (0x2EBF0, 0x2EE5D), + (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), + (0x31350, 0x323AF), +] diff --git a/crates/sqlbuild-core/src/text/alnum_ranges/unicode_16_0_0.in b/crates/sqlbuild-core/src/text/alnum_ranges/unicode_16_0_0.in new file mode 100644 index 0000000000..b0d7858687 --- /dev/null +++ b/crates/sqlbuild-core/src/text/alnum_ranges/unicode_16_0_0.in @@ -0,0 +1,774 @@ +// Generated by `scripts/generate_python_char_table.py`; do not edit. +// Inclusive code point ranges where `str.isalnum()` is true in Unicode 16.0.0, in order. +&[ + (0x0030, 0x0039), + (0x0041, 0x005A), + (0x0061, 0x007A), + (0x00AA, 0x00AA), + (0x00B2, 0x00B3), + (0x00B5, 0x00B5), + (0x00B9, 0x00BA), + (0x00BC, 0x00BE), + (0x00C0, 0x00D6), + (0x00D8, 0x00F6), + (0x00F8, 0x02C1), + (0x02C6, 0x02D1), + (0x02E0, 0x02E4), + (0x02EC, 0x02EC), + (0x02EE, 0x02EE), + (0x0370, 0x0374), + (0x0376, 0x0377), + (0x037A, 0x037D), + (0x037F, 0x037F), + (0x0386, 0x0386), + (0x0388, 0x038A), + (0x038C, 0x038C), + (0x038E, 0x03A1), + (0x03A3, 0x03F5), + (0x03F7, 0x0481), + (0x048A, 0x052F), + (0x0531, 0x0556), + (0x0559, 0x0559), + (0x0560, 0x0588), + (0x05D0, 0x05EA), + (0x05EF, 0x05F2), + (0x0620, 0x064A), + (0x0660, 0x0669), + (0x066E, 0x066F), + (0x0671, 0x06D3), + (0x06D5, 0x06D5), + (0x06E5, 0x06E6), + (0x06EE, 0x06FC), + (0x06FF, 0x06FF), + (0x0710, 0x0710), + (0x0712, 0x072F), + (0x074D, 0x07A5), + (0x07B1, 0x07B1), + (0x07C0, 0x07EA), + (0x07F4, 0x07F5), + (0x07FA, 0x07FA), + (0x0800, 0x0815), + (0x081A, 0x081A), + (0x0824, 0x0824), + (0x0828, 0x0828), + (0x0840, 0x0858), + (0x0860, 0x086A), + (0x0870, 0x0887), + (0x0889, 0x088E), + (0x08A0, 0x08C9), + (0x0904, 0x0939), + (0x093D, 0x093D), + (0x0950, 0x0950), + (0x0958, 0x0961), + (0x0966, 0x096F), + (0x0971, 0x0980), + (0x0985, 0x098C), + (0x098F, 0x0990), + (0x0993, 0x09A8), + (0x09AA, 0x09B0), + (0x09B2, 0x09B2), + (0x09B6, 0x09B9), + (0x09BD, 0x09BD), + (0x09CE, 0x09CE), + (0x09DC, 0x09DD), + (0x09DF, 0x09E1), + (0x09E6, 0x09F1), + (0x09F4, 0x09F9), + (0x09FC, 0x09FC), + (0x0A05, 0x0A0A), + (0x0A0F, 0x0A10), + (0x0A13, 0x0A28), + (0x0A2A, 0x0A30), + (0x0A32, 0x0A33), + (0x0A35, 0x0A36), + (0x0A38, 0x0A39), + (0x0A59, 0x0A5C), + (0x0A5E, 0x0A5E), + (0x0A66, 0x0A6F), + (0x0A72, 0x0A74), + (0x0A85, 0x0A8D), + (0x0A8F, 0x0A91), + (0x0A93, 0x0AA8), + (0x0AAA, 0x0AB0), + (0x0AB2, 0x0AB3), + (0x0AB5, 0x0AB9), + (0x0ABD, 0x0ABD), + (0x0AD0, 0x0AD0), + (0x0AE0, 0x0AE1), + (0x0AE6, 0x0AEF), + (0x0AF9, 0x0AF9), + (0x0B05, 0x0B0C), + (0x0B0F, 0x0B10), + (0x0B13, 0x0B28), + (0x0B2A, 0x0B30), + (0x0B32, 0x0B33), + (0x0B35, 0x0B39), + (0x0B3D, 0x0B3D), + (0x0B5C, 0x0B5D), + (0x0B5F, 0x0B61), + (0x0B66, 0x0B6F), + (0x0B71, 0x0B77), + (0x0B83, 0x0B83), + (0x0B85, 0x0B8A), + (0x0B8E, 0x0B90), + (0x0B92, 0x0B95), + (0x0B99, 0x0B9A), + (0x0B9C, 0x0B9C), + (0x0B9E, 0x0B9F), + (0x0BA3, 0x0BA4), + (0x0BA8, 0x0BAA), + (0x0BAE, 0x0BB9), + (0x0BD0, 0x0BD0), + (0x0BE6, 0x0BF2), + (0x0C05, 0x0C0C), + (0x0C0E, 0x0C10), + (0x0C12, 0x0C28), + (0x0C2A, 0x0C39), + (0x0C3D, 0x0C3D), + (0x0C58, 0x0C5A), + (0x0C5D, 0x0C5D), + (0x0C60, 0x0C61), + (0x0C66, 0x0C6F), + (0x0C78, 0x0C7E), + (0x0C80, 0x0C80), + (0x0C85, 0x0C8C), + (0x0C8E, 0x0C90), + (0x0C92, 0x0CA8), + (0x0CAA, 0x0CB3), + (0x0CB5, 0x0CB9), + (0x0CBD, 0x0CBD), + (0x0CDD, 0x0CDE), + (0x0CE0, 0x0CE1), + (0x0CE6, 0x0CEF), + (0x0CF1, 0x0CF2), + (0x0D04, 0x0D0C), + (0x0D0E, 0x0D10), + (0x0D12, 0x0D3A), + (0x0D3D, 0x0D3D), + (0x0D4E, 0x0D4E), + (0x0D54, 0x0D56), + (0x0D58, 0x0D61), + (0x0D66, 0x0D78), + (0x0D7A, 0x0D7F), + (0x0D85, 0x0D96), + (0x0D9A, 0x0DB1), + (0x0DB3, 0x0DBB), + (0x0DBD, 0x0DBD), + (0x0DC0, 0x0DC6), + (0x0DE6, 0x0DEF), + (0x0E01, 0x0E30), + (0x0E32, 0x0E33), + (0x0E40, 0x0E46), + (0x0E50, 0x0E59), + (0x0E81, 0x0E82), + (0x0E84, 0x0E84), + (0x0E86, 0x0E8A), + (0x0E8C, 0x0EA3), + (0x0EA5, 0x0EA5), + (0x0EA7, 0x0EB0), + (0x0EB2, 0x0EB3), + (0x0EBD, 0x0EBD), + (0x0EC0, 0x0EC4), + (0x0EC6, 0x0EC6), + (0x0ED0, 0x0ED9), + (0x0EDC, 0x0EDF), + (0x0F00, 0x0F00), + (0x0F20, 0x0F33), + (0x0F40, 0x0F47), + (0x0F49, 0x0F6C), + (0x0F88, 0x0F8C), + (0x1000, 0x102A), + (0x103F, 0x1049), + (0x1050, 0x1055), + (0x105A, 0x105D), + (0x1061, 0x1061), + (0x1065, 0x1066), + (0x106E, 0x1070), + (0x1075, 0x1081), + (0x108E, 0x108E), + (0x1090, 0x1099), + (0x10A0, 0x10C5), + (0x10C7, 0x10C7), + (0x10CD, 0x10CD), + (0x10D0, 0x10FA), + (0x10FC, 0x1248), + (0x124A, 0x124D), + (0x1250, 0x1256), + (0x1258, 0x1258), + (0x125A, 0x125D), + (0x1260, 0x1288), + (0x128A, 0x128D), + (0x1290, 0x12B0), + (0x12B2, 0x12B5), + (0x12B8, 0x12BE), + (0x12C0, 0x12C0), + (0x12C2, 0x12C5), + (0x12C8, 0x12D6), + (0x12D8, 0x1310), + (0x1312, 0x1315), + (0x1318, 0x135A), + (0x1369, 0x137C), + (0x1380, 0x138F), + (0x13A0, 0x13F5), + (0x13F8, 0x13FD), + (0x1401, 0x166C), + (0x166F, 0x167F), + (0x1681, 0x169A), + (0x16A0, 0x16EA), + (0x16EE, 0x16F8), + (0x1700, 0x1711), + (0x171F, 0x1731), + (0x1740, 0x1751), + (0x1760, 0x176C), + (0x176E, 0x1770), + (0x1780, 0x17B3), + (0x17D7, 0x17D7), + (0x17DC, 0x17DC), + (0x17E0, 0x17E9), + (0x17F0, 0x17F9), + (0x1810, 0x1819), + (0x1820, 0x1878), + (0x1880, 0x1884), + (0x1887, 0x18A8), + (0x18AA, 0x18AA), + (0x18B0, 0x18F5), + (0x1900, 0x191E), + (0x1946, 0x196D), + (0x1970, 0x1974), + (0x1980, 0x19AB), + (0x19B0, 0x19C9), + (0x19D0, 0x19DA), + (0x1A00, 0x1A16), + (0x1A20, 0x1A54), + (0x1A80, 0x1A89), + (0x1A90, 0x1A99), + (0x1AA7, 0x1AA7), + (0x1B05, 0x1B33), + (0x1B45, 0x1B4C), + (0x1B50, 0x1B59), + (0x1B83, 0x1BA0), + (0x1BAE, 0x1BE5), + (0x1C00, 0x1C23), + (0x1C40, 0x1C49), + (0x1C4D, 0x1C7D), + (0x1C80, 0x1C8A), + (0x1C90, 0x1CBA), + (0x1CBD, 0x1CBF), + (0x1CE9, 0x1CEC), + (0x1CEE, 0x1CF3), + (0x1CF5, 0x1CF6), + (0x1CFA, 0x1CFA), + (0x1D00, 0x1DBF), + (0x1E00, 0x1F15), + (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), + (0x1F48, 0x1F4D), + (0x1F50, 0x1F57), + (0x1F59, 0x1F59), + (0x1F5B, 0x1F5B), + (0x1F5D, 0x1F5D), + (0x1F5F, 0x1F7D), + (0x1F80, 0x1FB4), + (0x1FB6, 0x1FBC), + (0x1FBE, 0x1FBE), + (0x1FC2, 0x1FC4), + (0x1FC6, 0x1FCC), + (0x1FD0, 0x1FD3), + (0x1FD6, 0x1FDB), + (0x1FE0, 0x1FEC), + (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFC), + (0x2070, 0x2071), + (0x2074, 0x2079), + (0x207F, 0x2089), + (0x2090, 0x209C), + (0x2102, 0x2102), + (0x2107, 0x2107), + (0x210A, 0x2113), + (0x2115, 0x2115), + (0x2119, 0x211D), + (0x2124, 0x2124), + (0x2126, 0x2126), + (0x2128, 0x2128), + (0x212A, 0x212D), + (0x212F, 0x2139), + (0x213C, 0x213F), + (0x2145, 0x2149), + (0x214E, 0x214E), + (0x2150, 0x2189), + (0x2460, 0x249B), + (0x24EA, 0x24FF), + (0x2776, 0x2793), + (0x2C00, 0x2CE4), + (0x2CEB, 0x2CEE), + (0x2CF2, 0x2CF3), + (0x2CFD, 0x2CFD), + (0x2D00, 0x2D25), + (0x2D27, 0x2D27), + (0x2D2D, 0x2D2D), + (0x2D30, 0x2D67), + (0x2D6F, 0x2D6F), + (0x2D80, 0x2D96), + (0x2DA0, 0x2DA6), + (0x2DA8, 0x2DAE), + (0x2DB0, 0x2DB6), + (0x2DB8, 0x2DBE), + (0x2DC0, 0x2DC6), + (0x2DC8, 0x2DCE), + (0x2DD0, 0x2DD6), + (0x2DD8, 0x2DDE), + (0x2E2F, 0x2E2F), + (0x3005, 0x3007), + (0x3021, 0x3029), + (0x3031, 0x3035), + (0x3038, 0x303C), + (0x3041, 0x3096), + (0x309D, 0x309F), + (0x30A1, 0x30FA), + (0x30FC, 0x30FF), + (0x3105, 0x312F), + (0x3131, 0x318E), + (0x3192, 0x3195), + (0x31A0, 0x31BF), + (0x31F0, 0x31FF), + (0x3220, 0x3229), + (0x3248, 0x324F), + (0x3251, 0x325F), + (0x3280, 0x3289), + (0x32B1, 0x32BF), + (0x3400, 0x4DBF), + (0x4E00, 0xA48C), + (0xA4D0, 0xA4FD), + (0xA500, 0xA60C), + (0xA610, 0xA62B), + (0xA640, 0xA66E), + (0xA67F, 0xA69D), + (0xA6A0, 0xA6EF), + (0xA717, 0xA71F), + (0xA722, 0xA788), + (0xA78B, 0xA7CD), + (0xA7D0, 0xA7D1), + (0xA7D3, 0xA7D3), + (0xA7D5, 0xA7DC), + (0xA7F2, 0xA801), + (0xA803, 0xA805), + (0xA807, 0xA80A), + (0xA80C, 0xA822), + (0xA830, 0xA835), + (0xA840, 0xA873), + (0xA882, 0xA8B3), + (0xA8D0, 0xA8D9), + (0xA8F2, 0xA8F7), + (0xA8FB, 0xA8FB), + (0xA8FD, 0xA8FE), + (0xA900, 0xA925), + (0xA930, 0xA946), + (0xA960, 0xA97C), + (0xA984, 0xA9B2), + (0xA9CF, 0xA9D9), + (0xA9E0, 0xA9E4), + (0xA9E6, 0xA9FE), + (0xAA00, 0xAA28), + (0xAA40, 0xAA42), + (0xAA44, 0xAA4B), + (0xAA50, 0xAA59), + (0xAA60, 0xAA76), + (0xAA7A, 0xAA7A), + (0xAA7E, 0xAAAF), + (0xAAB1, 0xAAB1), + (0xAAB5, 0xAAB6), + (0xAAB9, 0xAABD), + (0xAAC0, 0xAAC0), + (0xAAC2, 0xAAC2), + (0xAADB, 0xAADD), + (0xAAE0, 0xAAEA), + (0xAAF2, 0xAAF4), + (0xAB01, 0xAB06), + (0xAB09, 0xAB0E), + (0xAB11, 0xAB16), + (0xAB20, 0xAB26), + (0xAB28, 0xAB2E), + (0xAB30, 0xAB5A), + (0xAB5C, 0xAB69), + (0xAB70, 0xABE2), + (0xABF0, 0xABF9), + (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), + (0xD7CB, 0xD7FB), + (0xF900, 0xFA6D), + (0xFA70, 0xFAD9), + (0xFB00, 0xFB06), + (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), + (0xFB1F, 0xFB28), + (0xFB2A, 0xFB36), + (0xFB38, 0xFB3C), + (0xFB3E, 0xFB3E), + (0xFB40, 0xFB41), + (0xFB43, 0xFB44), + (0xFB46, 0xFBB1), + (0xFBD3, 0xFD3D), + (0xFD50, 0xFD8F), + (0xFD92, 0xFDC7), + (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), + (0xFE76, 0xFEFC), + (0xFF10, 0xFF19), + (0xFF21, 0xFF3A), + (0xFF41, 0xFF5A), + (0xFF66, 0xFFBE), + (0xFFC2, 0xFFC7), + (0xFFCA, 0xFFCF), + (0xFFD2, 0xFFD7), + (0xFFDA, 0xFFDC), + (0x10000, 0x1000B), + (0x1000D, 0x10026), + (0x10028, 0x1003A), + (0x1003C, 0x1003D), + (0x1003F, 0x1004D), + (0x10050, 0x1005D), + (0x10080, 0x100FA), + (0x10107, 0x10133), + (0x10140, 0x10178), + (0x1018A, 0x1018B), + (0x10280, 0x1029C), + (0x102A0, 0x102D0), + (0x102E1, 0x102FB), + (0x10300, 0x10323), + (0x1032D, 0x1034A), + (0x10350, 0x10375), + (0x10380, 0x1039D), + (0x103A0, 0x103C3), + (0x103C8, 0x103CF), + (0x103D1, 0x103D5), + (0x10400, 0x1049D), + (0x104A0, 0x104A9), + (0x104B0, 0x104D3), + (0x104D8, 0x104FB), + (0x10500, 0x10527), + (0x10530, 0x10563), + (0x10570, 0x1057A), + (0x1057C, 0x1058A), + (0x1058C, 0x10592), + (0x10594, 0x10595), + (0x10597, 0x105A1), + (0x105A3, 0x105B1), + (0x105B3, 0x105B9), + (0x105BB, 0x105BC), + (0x105C0, 0x105F3), + (0x10600, 0x10736), + (0x10740, 0x10755), + (0x10760, 0x10767), + (0x10780, 0x10785), + (0x10787, 0x107B0), + (0x107B2, 0x107BA), + (0x10800, 0x10805), + (0x10808, 0x10808), + (0x1080A, 0x10835), + (0x10837, 0x10838), + (0x1083C, 0x1083C), + (0x1083F, 0x10855), + (0x10858, 0x10876), + (0x10879, 0x1089E), + (0x108A7, 0x108AF), + (0x108E0, 0x108F2), + (0x108F4, 0x108F5), + (0x108FB, 0x1091B), + (0x10920, 0x10939), + (0x10980, 0x109B7), + (0x109BC, 0x109CF), + (0x109D2, 0x10A00), + (0x10A10, 0x10A13), + (0x10A15, 0x10A17), + (0x10A19, 0x10A35), + (0x10A40, 0x10A48), + (0x10A60, 0x10A7E), + (0x10A80, 0x10A9F), + (0x10AC0, 0x10AC7), + (0x10AC9, 0x10AE4), + (0x10AEB, 0x10AEF), + (0x10B00, 0x10B35), + (0x10B40, 0x10B55), + (0x10B58, 0x10B72), + (0x10B78, 0x10B91), + (0x10BA9, 0x10BAF), + (0x10C00, 0x10C48), + (0x10C80, 0x10CB2), + (0x10CC0, 0x10CF2), + (0x10CFA, 0x10D23), + (0x10D30, 0x10D39), + (0x10D40, 0x10D65), + (0x10D6F, 0x10D85), + (0x10E60, 0x10E7E), + (0x10E80, 0x10EA9), + (0x10EB0, 0x10EB1), + (0x10EC2, 0x10EC4), + (0x10F00, 0x10F27), + (0x10F30, 0x10F45), + (0x10F51, 0x10F54), + (0x10F70, 0x10F81), + (0x10FB0, 0x10FCB), + (0x10FE0, 0x10FF6), + (0x11003, 0x11037), + (0x11052, 0x1106F), + (0x11071, 0x11072), + (0x11075, 0x11075), + (0x11083, 0x110AF), + (0x110D0, 0x110E8), + (0x110F0, 0x110F9), + (0x11103, 0x11126), + (0x11136, 0x1113F), + (0x11144, 0x11144), + (0x11147, 0x11147), + (0x11150, 0x11172), + (0x11176, 0x11176), + (0x11183, 0x111B2), + (0x111C1, 0x111C4), + (0x111D0, 0x111DA), + (0x111DC, 0x111DC), + (0x111E1, 0x111F4), + (0x11200, 0x11211), + (0x11213, 0x1122B), + (0x1123F, 0x11240), + (0x11280, 0x11286), + (0x11288, 0x11288), + (0x1128A, 0x1128D), + (0x1128F, 0x1129D), + (0x1129F, 0x112A8), + (0x112B0, 0x112DE), + (0x112F0, 0x112F9), + (0x11305, 0x1130C), + (0x1130F, 0x11310), + (0x11313, 0x11328), + (0x1132A, 0x11330), + (0x11332, 0x11333), + (0x11335, 0x11339), + (0x1133D, 0x1133D), + (0x11350, 0x11350), + (0x1135D, 0x11361), + (0x11380, 0x11389), + (0x1138B, 0x1138B), + (0x1138E, 0x1138E), + (0x11390, 0x113B5), + (0x113B7, 0x113B7), + (0x113D1, 0x113D1), + (0x113D3, 0x113D3), + (0x11400, 0x11434), + (0x11447, 0x1144A), + (0x11450, 0x11459), + (0x1145F, 0x11461), + (0x11480, 0x114AF), + (0x114C4, 0x114C5), + (0x114C7, 0x114C7), + (0x114D0, 0x114D9), + (0x11580, 0x115AE), + (0x115D8, 0x115DB), + (0x11600, 0x1162F), + (0x11644, 0x11644), + (0x11650, 0x11659), + (0x11680, 0x116AA), + (0x116B8, 0x116B8), + (0x116C0, 0x116C9), + (0x116D0, 0x116E3), + (0x11700, 0x1171A), + (0x11730, 0x1173B), + (0x11740, 0x11746), + (0x11800, 0x1182B), + (0x118A0, 0x118F2), + (0x118FF, 0x11906), + (0x11909, 0x11909), + (0x1190C, 0x11913), + (0x11915, 0x11916), + (0x11918, 0x1192F), + (0x1193F, 0x1193F), + (0x11941, 0x11941), + (0x11950, 0x11959), + (0x119A0, 0x119A7), + (0x119AA, 0x119D0), + (0x119E1, 0x119E1), + (0x119E3, 0x119E3), + (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), + (0x11A3A, 0x11A3A), + (0x11A50, 0x11A50), + (0x11A5C, 0x11A89), + (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), + (0x11BC0, 0x11BE0), + (0x11BF0, 0x11BF9), + (0x11C00, 0x11C08), + (0x11C0A, 0x11C2E), + (0x11C40, 0x11C40), + (0x11C50, 0x11C6C), + (0x11C72, 0x11C8F), + (0x11D00, 0x11D06), + (0x11D08, 0x11D09), + (0x11D0B, 0x11D30), + (0x11D46, 0x11D46), + (0x11D50, 0x11D59), + (0x11D60, 0x11D65), + (0x11D67, 0x11D68), + (0x11D6A, 0x11D89), + (0x11D98, 0x11D98), + (0x11DA0, 0x11DA9), + (0x11EE0, 0x11EF2), + (0x11F02, 0x11F02), + (0x11F04, 0x11F10), + (0x11F12, 0x11F33), + (0x11F50, 0x11F59), + (0x11FB0, 0x11FB0), + (0x11FC0, 0x11FD4), + (0x12000, 0x12399), + (0x12400, 0x1246E), + (0x12480, 0x12543), + (0x12F90, 0x12FF0), + (0x13000, 0x1342F), + (0x13441, 0x13446), + (0x13460, 0x143FA), + (0x14400, 0x14646), + (0x16100, 0x1611D), + (0x16130, 0x16139), + (0x16800, 0x16A38), + (0x16A40, 0x16A5E), + (0x16A60, 0x16A69), + (0x16A70, 0x16ABE), + (0x16AC0, 0x16AC9), + (0x16AD0, 0x16AED), + (0x16B00, 0x16B2F), + (0x16B40, 0x16B43), + (0x16B50, 0x16B59), + (0x16B5B, 0x16B61), + (0x16B63, 0x16B77), + (0x16B7D, 0x16B8F), + (0x16D40, 0x16D6C), + (0x16D70, 0x16D79), + (0x16E40, 0x16E96), + (0x16F00, 0x16F4A), + (0x16F50, 0x16F50), + (0x16F93, 0x16F9F), + (0x16FE0, 0x16FE1), + (0x16FE3, 0x16FE3), + (0x17000, 0x187F7), + (0x18800, 0x18CD5), + (0x18CFF, 0x18D08), + (0x1AFF0, 0x1AFF3), + (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), + (0x1B000, 0x1B122), + (0x1B132, 0x1B132), + (0x1B150, 0x1B152), + (0x1B155, 0x1B155), + (0x1B164, 0x1B167), + (0x1B170, 0x1B2FB), + (0x1BC00, 0x1BC6A), + (0x1BC70, 0x1BC7C), + (0x1BC80, 0x1BC88), + (0x1BC90, 0x1BC99), + (0x1CCF0, 0x1CCF9), + (0x1D2C0, 0x1D2D3), + (0x1D2E0, 0x1D2F3), + (0x1D360, 0x1D378), + (0x1D400, 0x1D454), + (0x1D456, 0x1D49C), + (0x1D49E, 0x1D49F), + (0x1D4A2, 0x1D4A2), + (0x1D4A5, 0x1D4A6), + (0x1D4A9, 0x1D4AC), + (0x1D4AE, 0x1D4B9), + (0x1D4BB, 0x1D4BB), + (0x1D4BD, 0x1D4C3), + (0x1D4C5, 0x1D505), + (0x1D507, 0x1D50A), + (0x1D50D, 0x1D514), + (0x1D516, 0x1D51C), + (0x1D51E, 0x1D539), + (0x1D53B, 0x1D53E), + (0x1D540, 0x1D544), + (0x1D546, 0x1D546), + (0x1D54A, 0x1D550), + (0x1D552, 0x1D6A5), + (0x1D6A8, 0x1D6C0), + (0x1D6C2, 0x1D6DA), + (0x1D6DC, 0x1D6FA), + (0x1D6FC, 0x1D714), + (0x1D716, 0x1D734), + (0x1D736, 0x1D74E), + (0x1D750, 0x1D76E), + (0x1D770, 0x1D788), + (0x1D78A, 0x1D7A8), + (0x1D7AA, 0x1D7C2), + (0x1D7C4, 0x1D7CB), + (0x1D7CE, 0x1D7FF), + (0x1DF00, 0x1DF1E), + (0x1DF25, 0x1DF2A), + (0x1E030, 0x1E06D), + (0x1E100, 0x1E12C), + (0x1E137, 0x1E13D), + (0x1E140, 0x1E149), + (0x1E14E, 0x1E14E), + (0x1E290, 0x1E2AD), + (0x1E2C0, 0x1E2EB), + (0x1E2F0, 0x1E2F9), + (0x1E4D0, 0x1E4EB), + (0x1E4F0, 0x1E4F9), + (0x1E5D0, 0x1E5ED), + (0x1E5F0, 0x1E5FA), + (0x1E7E0, 0x1E7E6), + (0x1E7E8, 0x1E7EB), + (0x1E7ED, 0x1E7EE), + (0x1E7F0, 0x1E7FE), + (0x1E800, 0x1E8C4), + (0x1E8C7, 0x1E8CF), + (0x1E900, 0x1E943), + (0x1E94B, 0x1E94B), + (0x1E950, 0x1E959), + (0x1EC71, 0x1ECAB), + (0x1ECAD, 0x1ECAF), + (0x1ECB1, 0x1ECB4), + (0x1ED01, 0x1ED2D), + (0x1ED2F, 0x1ED3D), + (0x1EE00, 0x1EE03), + (0x1EE05, 0x1EE1F), + (0x1EE21, 0x1EE22), + (0x1EE24, 0x1EE24), + (0x1EE27, 0x1EE27), + (0x1EE29, 0x1EE32), + (0x1EE34, 0x1EE37), + (0x1EE39, 0x1EE39), + (0x1EE3B, 0x1EE3B), + (0x1EE42, 0x1EE42), + (0x1EE47, 0x1EE47), + (0x1EE49, 0x1EE49), + (0x1EE4B, 0x1EE4B), + (0x1EE4D, 0x1EE4F), + (0x1EE51, 0x1EE52), + (0x1EE54, 0x1EE54), + (0x1EE57, 0x1EE57), + (0x1EE59, 0x1EE59), + (0x1EE5B, 0x1EE5B), + (0x1EE5D, 0x1EE5D), + (0x1EE5F, 0x1EE5F), + (0x1EE61, 0x1EE62), + (0x1EE64, 0x1EE64), + (0x1EE67, 0x1EE6A), + (0x1EE6C, 0x1EE72), + (0x1EE74, 0x1EE77), + (0x1EE79, 0x1EE7C), + (0x1EE7E, 0x1EE7E), + (0x1EE80, 0x1EE89), + (0x1EE8B, 0x1EE9B), + (0x1EEA1, 0x1EEA3), + (0x1EEA5, 0x1EEA9), + (0x1EEAB, 0x1EEBB), + (0x1F100, 0x1F10C), + (0x1FBF0, 0x1FBF9), + (0x20000, 0x2A6DF), + (0x2A700, 0x2B739), + (0x2B740, 0x2B81D), + (0x2B820, 0x2CEA1), + (0x2CEB0, 0x2EBE0), + (0x2EBF0, 0x2EE5D), + (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), + (0x31350, 0x323AF), +] diff --git a/crates/sqlbuild-core/src/text/main/is_python_alnum.rs b/crates/sqlbuild-core/src/text/main/is_python_alnum.rs index 198ac5632c..f8575f0ba2 100644 --- a/crates/sqlbuild-core/src/text/main/is_python_alnum.rs +++ b/crates/sqlbuild-core/src/text/main/is_python_alnum.rs @@ -1,15 +1,18 @@ //! Python's `str.isalnum()` for one character, from a table generated by Python. -use crate::text::_helpers::alnum_ranges::PYTHON_ALNUM_RANGES; +use crate::text::models::PythonText; /// Whether Python's `str.isalnum()` is true for one character. -pub fn is_python_alnum(character: char) -> bool { +pub fn is_python_alnum(python: PythonText, character: char) -> bool { if character.is_ascii() { return character.is_ascii_alphanumeric(); } let code_point: u32 = u32::from(character); - let index: usize = PYTHON_ALNUM_RANGES.partition_point(|(_, end)| *end < code_point); - PYTHON_ALNUM_RANGES + let index: usize = python + .alnum_ranges + .partition_point(|(_, end)| *end < code_point); + python + .alnum_ranges .get(index) .is_some_and(|(start, _)| *start <= code_point) } diff --git a/crates/sqlbuild-core/src/text/main/is_python_word.rs b/crates/sqlbuild-core/src/text/main/is_python_word.rs index f63698521c..5fe5bb2989 100644 --- a/crates/sqlbuild-core/src/text/main/is_python_word.rs +++ b/crates/sqlbuild-core/src/text/main/is_python_word.rs @@ -1,8 +1,9 @@ //! A character of Python's `\w` for `str` patterns. use crate::text::main::is_python_alnum::is_python_alnum; +use crate::text::models::PythonText; /// Whether one character matches Python's `\w`: `str.isalnum()` or `_`. -pub fn is_python_word(character: char) -> bool { - character == '_' || is_python_alnum(character) +pub fn is_python_word(python: PythonText, character: char) -> bool { + character == '_' || is_python_alnum(python, character) } diff --git a/crates/sqlbuild-core/src/text/main/mod.rs b/crates/sqlbuild-core/src/text/main/mod.rs index 1c4f886f29..a0a0883793 100644 --- a/crates/sqlbuild-core/src/text/main/mod.rs +++ b/crates/sqlbuild-core/src/text/main/mod.rs @@ -3,6 +3,6 @@ pub mod decode_python_text; pub mod is_python_alnum; pub mod is_python_space; pub mod is_python_word; -pub mod python_alnum_unicode_version; pub mod python_cleandoc; pub mod python_strip; +pub mod python_text; diff --git a/crates/sqlbuild-core/src/text/main/python_alnum_unicode_version.rs b/crates/sqlbuild-core/src/text/main/python_alnum_unicode_version.rs deleted file mode 100644 index f6792f65df..0000000000 --- a/crates/sqlbuild-core/src/text/main/python_alnum_unicode_version.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! The Unicode version of the native `str.isalnum()` table. - -use crate::text::_helpers::alnum_ranges::PYTHON_ALNUM_UNICODE_VERSION; - -/// The Unicode version of the Python whose `str.isalnum()` the native table reproduces. -pub fn python_alnum_unicode_version() -> &'static str { - PYTHON_ALNUM_UNICODE_VERSION -} diff --git a/crates/sqlbuild-core/src/text/main/python_cleandoc.rs b/crates/sqlbuild-core/src/text/main/python_cleandoc.rs index 0619ba36cb..5212fff41f 100644 --- a/crates/sqlbuild-core/src/text/main/python_cleandoc.rs +++ b/crates/sqlbuild-core/src/text/main/python_cleandoc.rs @@ -1,20 +1,25 @@ -//! Python's `inspect.cleandoc` (3.12), which SQL test and scenario bodies go through. +//! Python's `inspect.cleandoc`, which SQL test and scenario bodies go through. use crate::text::_helpers::cleandoc::{expand_tabs, skip_chars}; use crate::text::main::is_python_space::is_python_space; +use crate::text::models::{CleandocMargin, PythonText}; /// `inspect.cleandoc(text)`: expand tabs, dedent later lines and drop outer empty lines. -pub fn python_cleandoc(text: &str) -> String { +pub fn python_cleandoc(python: PythonText, text: &str) -> String { + let is_margin: fn(char) -> bool = match python.cleandoc_margin { + CleandocMargin::Whitespace => is_python_space, + CleandocMargin::Spaces => |character| character == ' ', + }; let expanded: String = expand_tabs(text); let mut lines: Vec<&str> = expanded.split('\n').collect(); let margin: Option = lines[1..] .iter() .filter_map(|line| { - let content = line.trim_start_matches(is_python_space); + let content = line.trim_start_matches(is_margin); (!content.is_empty()).then(|| line.chars().count() - content.chars().count()) }) .min(); - lines[0] = lines[0].trim_start_matches(is_python_space); + lines[0] = lines[0].trim_start_matches(is_margin); let mut cleaned: Vec<&str> = Vec::with_capacity(lines.len()); cleaned.push(lines[0]); cleaned.extend( diff --git a/crates/sqlbuild-core/src/text/main/python_text.rs b/crates/sqlbuild-core/src/text/main/python_text.rs new file mode 100644 index 0000000000..23f6a8efee --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/python_text.rs @@ -0,0 +1,21 @@ +//! Select the Python string semantics of one CPython release. + +use crate::text::_helpers::alnum_tables::ALNUM_TABLES; +use crate::text::models::{CleandocMargin, PythonText}; + +/// The semantics of one Python and its `unicodedata.unidata_version`, or `None` if unknown. +pub fn python_text(python_version: (u8, u8), unicode_version: &str) -> Option { + let cleandoc_margin: CleandocMargin = match python_version { + (3, 12) => CleandocMargin::Whitespace, + (3, 13 | 14) => CleandocMargin::Spaces, + _ => return None, + }; + let alnum_ranges: &'static [(u32, u32)] = ALNUM_TABLES + .iter() + .find(|(version, _)| *version == unicode_version) + .map(|(_, ranges)| *ranges)?; + Some(PythonText { + alnum_ranges, + cleandoc_margin, + }) +} diff --git a/crates/sqlbuild-core/src/text/models.rs b/crates/sqlbuild-core/src/text/models.rs index 7ba97cb723..aad15acdd2 100644 --- a/crates/sqlbuild-core/src/text/models.rs +++ b/crates/sqlbuild-core/src/text/models.rs @@ -99,3 +99,20 @@ impl<'text> LineIndex<'text> { }) } } + +/// The Python string semantics native text helpers reproduce for one CPython release. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PythonText { + /// Inclusive code point ranges where `str.isalnum()` is true, in order. + pub(crate) alnum_ranges: &'static [(u32, u32)], + pub(crate) cleandoc_margin: CleandocMargin, +} + +/// The characters `inspect.cleandoc` strips from line starts. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CleandocMargin { + /// Python 3.12: `str.lstrip()`, any Python whitespace. + Whitespace, + /// Python 3.13 and later: `str.lstrip(' ')`, spaces only. + Spaces, +} diff --git a/crates/sqlbuild-core/src/text/tests/test_python_compat.rs b/crates/sqlbuild-core/src/text/tests/test_python_compat.rs index 02d4907db5..e78053d265 100644 --- a/crates/sqlbuild-core/src/text/tests/test_python_compat.rs +++ b/crates/sqlbuild-core/src/text/tests/test_python_compat.rs @@ -2,7 +2,12 @@ use crate::text::main::close_matches::close_matches; use crate::text::main::is_python_alnum::is_python_alnum; use crate::text::main::is_python_space::is_python_space; use crate::text::main::is_python_word::is_python_word; -use crate::text::tests::test_types::{CharacterClassTestCase, CloseMatchesTestCase}; +use crate::text::main::python_cleandoc::python_cleandoc; +use crate::text::main::python_text::python_text; +use crate::text::models::PythonText; +use crate::text::tests::test_types::{ + CharacterClassTestCase, CleandocTestCase, CloseMatchesTestCase, PythonTextTestCase, +}; const KEYS: &[&str] = &[ "audits", @@ -142,11 +147,12 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { expected_space: true, }, ]; + let python: PythonText = python_text((3, 12), "15.0.0").expect("Python 3.12 is supported"); for test_case in test_cases { assert_eq!( ( - is_python_alnum(test_case.character), - is_python_word(test_case.character), + is_python_alnum(python, test_case.character), + is_python_word(python, test_case.character), is_python_space(test_case.character) ), ( @@ -159,3 +165,92 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { ); } } + +#[test] +fn given_python_runtimes_when_selecting_text_semantics_then_tables_follow_unicode_version() { + let test_cases = [ + PythonTextTestCase { + description: "Python 3.12 with Unicode 15.0 predates CJK extension I", + python_version: (3, 12), + unicode_version: "15.0.0", + expected_alnum: Some([false, false]), + }, + PythonTextTestCase { + description: "Python 3.13 with Unicode 15.1 adds CJK extension I", + python_version: (3, 13), + unicode_version: "15.1.0", + expected_alnum: Some([true, false]), + }, + PythonTextTestCase { + description: "Python 3.14 with Unicode 16.0 adds the Garay script", + python_version: (3, 14), + unicode_version: "16.0.0", + expected_alnum: Some([true, true]), + }, + PythonTextTestCase { + description: "an unknown Unicode version defers", + python_version: (3, 14), + unicode_version: "17.0.0", + expected_alnum: None, + }, + PythonTextTestCase { + description: "an unknown Python version defers", + python_version: (3, 15), + unicode_version: "16.0.0", + expected_alnum: None, + }, + PythonTextTestCase { + description: "an older Python defers", + python_version: (3, 11), + unicode_version: "15.0.0", + expected_alnum: None, + }, + ]; + for test_case in test_cases { + let alnum: Option<[bool; 2]> = + python_text(test_case.python_version, test_case.unicode_version).map(|python| { + [ + is_python_alnum(python, '\u{2ebf0}'), + is_python_alnum(python, '\u{10d40}'), + ] + }); + assert_eq!(alnum, test_case.expected_alnum, "{}", test_case.description); + } +} + +#[test] +fn given_python_versions_when_cleaning_bodies_then_cleandoc_matches_that_version() { + let test_cases = [ + CleandocTestCase { + description: "3.12 strips any leading whitespace", + python_version: (3, 12), + unicode_version: "15.0.0", + text: "\u{c}first\n\u{3000} second\n\u{3000} third", + expected_text: "first\n second\nthird", + }, + CleandocTestCase { + description: "3.13 strips leading spaces only", + python_version: (3, 13), + unicode_version: "15.1.0", + text: "\u{c}first\n\u{3000} second\n third", + expected_text: "\u{c}first\n\u{3000} second\n third", + }, + CleandocTestCase { + description: "3.14 dedents by the common run of spaces", + python_version: (3, 14), + unicode_version: "16.0.0", + text: " first\n second\n\t third\n", + expected_text: "first\nsecond\n third", + }, + ]; + for test_case in test_cases { + let python: PythonText = python_text(test_case.python_version, test_case.unicode_version) + .expect("supported runtime"); + assert_eq!( + python_cleandoc(python, test_case.text), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/text/tests/test_types.rs b/crates/sqlbuild-core/src/text/tests/test_types.rs index 7ed7c2ac12..18e6fbd116 100644 --- a/crates/sqlbuild-core/src/text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/text/tests/test_types.rs @@ -34,3 +34,19 @@ pub(super) struct CharacterClassTestCase { pub(super) expected_word: bool, pub(super) expected_space: bool, } + +pub(super) struct PythonTextTestCase { + pub(super) description: &'static str, + pub(super) python_version: (u8, u8), + pub(super) unicode_version: &'static str, + /// `str.isalnum()` of U+2EBF0 and U+10D40, or `None` when the runtime is unsupported. + pub(super) expected_alnum: Option<[bool; 2]>, +} + +pub(super) struct CleandocTestCase { + pub(super) description: &'static str, + pub(super) python_version: (u8, u8), + pub(super) unicode_version: &'static str, + pub(super) text: &'static str, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-discovery/src/_helpers/header_keys.rs b/crates/sqlbuild-discovery/src/_helpers/header_keys.rs index 93eefd20f1..bb5b075b85 100644 --- a/crates/sqlbuild-discovery/src/_helpers/header_keys.rs +++ b/crates/sqlbuild-discovery/src/_helpers/header_keys.rs @@ -4,6 +4,7 @@ use crate::_helpers::suggestions::unsupported_keys_help; use crate::models::{DiscoveryFailure, FailureKind}; use sqlbuild_core::text::main::is_python_space::is_python_space; use sqlbuild_core::text::main::is_python_word::is_python_word; +use sqlbuild_core::text::models::PythonText; /// The facts of one header that declares keys outside its supported set. pub(crate) struct UnsupportedKeys<'a> { @@ -15,6 +16,7 @@ pub(crate) struct UnsupportedKeys<'a> { pub(crate) header_line: usize, pub(crate) keys: &'a [&'a str], pub(crate) supported_keys: &'a [String], + pub(crate) python: PythonText, } pub(crate) fn unsupported_keys_failure(facts: &UnsupportedKeys<'_>) -> DiscoveryFailure { @@ -26,9 +28,10 @@ pub(crate) fn unsupported_keys_failure(facts: &UnsupportedKeys<'_>) -> Discovery header_line, keys, supported_keys, + python, } = facts; let header_chars: Vec = header.chars().collect(); - let key_offset: usize = header_key_offset(&header_chars, keys[0]); + let key_offset: usize = header_key_offset(*python, &header_chars, keys[0]); let line: usize = header_line + header_chars[..key_offset] .iter() @@ -44,13 +47,25 @@ pub(crate) fn unsupported_keys_failure(facts: &UnsupportedKeys<'_>) -> Discovery } } +/// One key searched for in one header, under one Python's `\w`. +struct KeySearch<'a> { + python: PythonText, + header: &'a [char], + key: &'a [char], +} + /// `re.search(r"(?:^|[(,])(?:\s|--[^\n]*(?:\n|$)|/\*.*?\*/)*(KEY)\b", header, re.S).start(1)`. -fn header_key_offset(header: &[char], key: &str) -> usize { +fn header_key_offset(python: PythonText, header: &[char], key: &str) -> usize { let key: Vec = key.chars().collect(); + let search = KeySearch { + python, + header, + key: &key, + }; let length: usize = header.len(); let mut found: Vec> = vec![None; length + 1]; for position in (0..=length).rev() { - found[position] = skip_then_key(header, &key, position, &found); + found[position] = skip_then_key(&search, position, &found); } for start in 0..=length { if start == 0 @@ -70,11 +85,15 @@ fn header_key_offset(header: &[char], key: &str) -> usize { /// Where the key starts when the skip loop is entered at `position`, in backtracking order. fn skip_then_key( - header: &[char], - key: &[char], + search: &KeySearch<'_>, position: usize, found: &[Option], ) -> Option { + let KeySearch { + python, + header, + key, + } = *search; let length = header.len(); if position < length { if is_python_space(header[position]) @@ -105,20 +124,20 @@ fn skip_then_key( } } let end = position + key.len(); - if header.get(position..end) == Some(key) && word_boundary(header, end) { + if header.get(position..end) == Some(key) && word_boundary(python, header, end) { return Some(position); } None } /// `\b` at `position`: exactly one side is a word character. -fn word_boundary(text: &[char], position: usize) -> bool { +fn word_boundary(python: PythonText, text: &[char], position: usize) -> bool { let before = position .checked_sub(1) .and_then(|index| text.get(index)) - .is_some_and(|character| is_python_word(*character)); + .is_some_and(|character| is_python_word(python, *character)); let after = text .get(position) - .is_some_and(|character| is_python_word(*character)); + .is_some_and(|character| is_python_word(python, *character)); before != after } diff --git a/crates/sqlbuild-discovery/src/_helpers/statement_headers.rs b/crates/sqlbuild-discovery/src/_helpers/statement_headers.rs index e19619d021..4f80974682 100644 --- a/crates/sqlbuild-discovery/src/_helpers/statement_headers.rs +++ b/crates/sqlbuild-discovery/src/_helpers/statement_headers.rs @@ -2,6 +2,7 @@ use crate::_helpers::header_keys::{UnsupportedKeys, unsupported_keys_failure}; use crate::models::{DiscoveryFailure, FailureKind}; +use sqlbuild_core::text::models::PythonText; use sqlbuild_sqltext::compiler::main::model_header_single_parsing::parse_one; use sqlbuild_sqltext::compiler::models::AuthoredValue; @@ -14,6 +15,7 @@ pub(crate) struct StatementHeader<'a> { pub(crate) statement: &'a str, pub(crate) file_path: &'a str, pub(crate) supported_keys: &'a [String], + pub(crate) python: PythonText, } /// `parse_header_values` then `reject_unsupported_header_keys`; `header_line` is one-based. @@ -48,6 +50,7 @@ pub(crate) fn parse_statement_header( header_line, keys: &unsupported, supported_keys: contract.supported_keys, + python: contract.python, })) } diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/output_columns.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/output_columns.rs index fe0ad8c2d9..a042bb5cb6 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/output_columns.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/output_columns.rs @@ -4,6 +4,7 @@ use crate::model_files::_helpers::locations::{absolute_span, line_starts}; use crate::models::LineColumnSpan; use sqlbuild_core::text::main::is_python_space::is_python_space; use sqlbuild_core::text::main::is_python_word::is_python_word; +use sqlbuild_core::text::models::PythonText; const SELECT_KEYWORD: &str = "SELECT"; const UNION_KEYWORD: &str = "UNION"; @@ -12,6 +13,7 @@ const ESCAPE: char = '\\'; /// Python's `matched_model_output_column_locations` from the SQL start (a byte offset). pub(crate) fn output_column_locations( + python: PythonText, contents: &str, sql_start: usize, extract_implicit_alias_columns: bool, @@ -19,7 +21,7 @@ pub(crate) fn output_column_locations( let characters: Vec = contents.chars().collect(); let sql_offset = contents[..sql_start].chars().count(); let sql = &characters[sql_offset..]; - let Some((list_start, list_end)) = top_level_select_list_bounds(sql) else { + let Some((list_start, list_end)) = top_level_select_list_bounds(python, sql) else { return Vec::new(); }; let starts = line_starts(&characters); @@ -56,7 +58,7 @@ fn is_scan_special(character: char) -> bool { ) } -fn top_level_select_list_bounds(sql: &[char]) -> Option<(usize, usize)> { +fn top_level_select_list_bounds(python: PythonText, sql: &[char]) -> Option<(usize, usize)> { let length = sql.len(); let has_union_candidate = contains_lowercase_union(sql); let (mut depth, mut index) = (0_usize, 0_usize); @@ -106,17 +108,20 @@ fn top_level_select_list_bounds(sql: &[char]) -> Option<(usize, usize)> { let upper = character.to_ascii_uppercase(); match list_start { None => { - if upper == 'S' && keyword_at(sql, SELECT_KEYWORD, index) { + if upper == 'S' && keyword_at(python, sql, SELECT_KEYWORD, index) { index += SELECT_KEYWORD.len(); list_start = Some(index); continue; } } Some(start) => { - if upper == 'U' && keyword_at(sql, UNION_KEYWORD, index) { + if upper == 'U' && keyword_at(python, sql, UNION_KEYWORD, index) { return None; } - if list_end.is_none() && upper == 'F' && keyword_at(sql, FROM_KEYWORD, index) { + if list_end.is_none() + && upper == 'F' + && keyword_at(python, sql, FROM_KEYWORD, index) + { list_end = Some(index); if !has_union_candidate { return Some((start, index)); @@ -155,7 +160,7 @@ fn ascii_upper(character: char) -> Option { } } -fn keyword_at(sql: &[char], keyword: &str, index: usize) -> bool { +fn keyword_at(python: PythonText, sql: &[char], keyword: &str, index: usize) -> bool { let end = index + keyword.len(); let slice = &sql[index..end.min(sql.len())]; if slice.len() != keyword.len() @@ -168,7 +173,7 @@ fn keyword_at(sql: &[char], keyword: &str, index: usize) -> bool { } let before = index.checked_sub(1).map_or(' ', |previous| sql[previous]); let after = sql.get(end).copied().unwrap_or(' '); - !is_python_word(before) && !is_python_word(after) + !is_python_word(python, before) && !is_python_word(python, after) } fn split_top_level_items(sql: &[char], start: usize, end: usize) -> Vec<(usize, usize)> { diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs index c402ba4090..24c0a3a454 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs @@ -61,6 +61,7 @@ pub(crate) fn parse_model_file( header_line: contents[..header_start].matches('\n').count() + 1, keys: &unsupported, supported_keys: &options.supported_keys, + python: options.python, })); } let query_sql: String = python_strip(&contents[sql_start..]).to_owned(); @@ -72,7 +73,12 @@ pub(crate) fn parse_model_file( let header_column_locations = header_column_locations(&contents, (header_start, header_end), &column_offsets); let output_column_locations = if options.extract_output_column_locations { - output_column_locations(&contents, sql_start, options.extract_implicit_alias_columns) + output_column_locations( + options.python, + &contents, + sql_start, + options.extract_implicit_alias_columns, + ) } else { Vec::new() }; diff --git a/crates/sqlbuild-discovery/src/model_files/models.rs b/crates/sqlbuild-discovery/src/model_files/models.rs index f23ad90701..619e87d7a6 100644 --- a/crates/sqlbuild-discovery/src/model_files/models.rs +++ b/crates/sqlbuild-discovery/src/model_files/models.rs @@ -1,6 +1,7 @@ //! Native results for SQL model files. use crate::models::LineColumnSpan; +use sqlbuild_core::text::models::PythonText; use sqlbuild_sqltext::compiler::models::AuthoredValue; /// What the Python facade asked for, with the header key sets Python owns during dual running. @@ -10,6 +11,8 @@ pub struct ModelFileOptions { pub removed_keys: Vec, pub extract_implicit_alias_columns: bool, pub extract_output_column_locations: bool, + /// The string semantics of the Python whose discovery output native must reproduce. + pub python: PythonText, } /// One parsed model file; values mirror the fields of Python's `DiscoveredSqlModelFile`. diff --git a/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs index aa5ad43815..9b0680bd90 100644 --- a/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs @@ -2,6 +2,7 @@ use crate::model_files::_helpers::parsing::parse_model_file; use crate::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use crate::model_files::tests::test_types::{Located, ModelSummary, Span}; use crate::models::LineColumnSpan; +use sqlbuild_core::text::main::python_text::python_text; const FILE_PATH: &str = "/project/models/orders.sql"; @@ -23,6 +24,7 @@ fn options() -> ModelFileOptions { removed_keys: vec!["run_despite_unchanged".to_owned()], extract_implicit_alias_columns: true, extract_output_column_locations: true, + python: python_text((3, 12), "15.0.0").expect("Python 3.12 is supported"), } } diff --git a/crates/sqlbuild-discovery/src/sql_tests/_helpers/scenario_parsing.rs b/crates/sqlbuild-discovery/src/sql_tests/_helpers/scenario_parsing.rs index 428efa510c..3cb6794612 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/_helpers/scenario_parsing.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/_helpers/scenario_parsing.rs @@ -2,7 +2,7 @@ use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; use crate::models::{DiscoveryFailure, FailureKind}; -use crate::sql_tests::models::DiscoveredScenarioFile; +use crate::sql_tests::models::{DiscoveredScenarioFile, SqlTestFileOptions}; use sqlbuild_core::text::main::python_cleandoc::python_cleandoc; use sqlbuild_sqltext::compiler::main::statement_header_matching::match_statement_header; @@ -12,7 +12,7 @@ const SCENARIO_STATEMENT: &str = "SCENARIO()"; pub(crate) fn parse_scenario_file( file_path: &str, contents: String, - supported_keys: &[String], + options: &SqlTestFileOptions, ) -> Result { let Some((header_start, header_end, sql_start)) = match_statement_header(&contents, 0, SCENARIO_KEYWORD) @@ -31,12 +31,13 @@ pub(crate) fn parse_scenario_file( statement_name: SCENARIO_KEYWORD, statement: SCENARIO_STATEMENT, file_path, - supported_keys, + supported_keys: &options.scenario_keys, + python: options.python, }, &contents[header_start..header_end], contents[..header_start].matches('\n').count() + 1, )?; - let sql_body: String = python_cleandoc(&contents[sql_start..]); + let sql_body: String = python_cleandoc(options.python, &contents[sql_start..]); Ok(DiscoveredScenarioFile { contents, header_values, diff --git a/crates/sqlbuild-discovery/src/sql_tests/_helpers/test_blocks.rs b/crates/sqlbuild-discovery/src/sql_tests/_helpers/test_blocks.rs index 28b285846c..4c5541f6c0 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/_helpers/test_blocks.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/_helpers/test_blocks.rs @@ -2,7 +2,7 @@ use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; use crate::models::{DiscoveryFailure, FailureKind}; -use crate::sql_tests::models::{DiscoveredSqlTestFile, SqlTestBlock}; +use crate::sql_tests::models::{DiscoveredSqlTestFile, SqlTestBlock, SqlTestFileOptions}; use sqlbuild_core::text::main::is_python_space::is_python_space; use sqlbuild_core::text::main::python_cleandoc::python_cleandoc; use sqlbuild_core::text::main::python_strip::python_strip; @@ -22,7 +22,7 @@ struct BlockMatch { pub(crate) fn parse_sql_test_file( file_path: &str, contents: String, - supported_keys: &[String], + options: &SqlTestFileOptions, ) -> Result { let matches: Vec = block_matches(&contents); let Some(first) = matches.first() else { @@ -36,7 +36,8 @@ pub(crate) fn parse_sql_test_file( statement_name: TEST_KEYWORD, statement: TEST_STATEMENT, file_path, - supported_keys, + supported_keys: &options.test_keys, + python: options.python, }; let mut blocks: Vec = Vec::with_capacity(matches.len()); let mut failure: Option = None; @@ -50,6 +51,7 @@ pub(crate) fn parse_sql_test_file( Ok(header_values) => blocks.push(SqlTestBlock { header_values, sql_body: python_cleandoc( + options.python, contents[block.end.min(next_start)..next_start] .trim_end_matches(is_python_space), ), diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs b/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs index 967eb618f3..3bf3d87fb4 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs @@ -14,6 +14,6 @@ pub fn discover_scenario_files( options: &SqlTestFileOptions, ) -> Result>, StageDeferral> { discover_sql_files(root, tree, SQL_SCENARIOS_ROOT, |file_path, contents| { - parse_scenario_file(file_path, contents, &options.scenario_keys) + parse_scenario_file(file_path, contents, options) }) } diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs b/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs index 41c74c256c..27aded3510 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs @@ -17,6 +17,6 @@ pub fn discover_sql_test_files( root, tree, &SQL_TESTS_ROOT.join("/"), - |file_path, contents| parse_sql_test_file(file_path, contents, &options.test_keys), + |file_path, contents| parse_sql_test_file(file_path, contents, options), ) } diff --git a/crates/sqlbuild-discovery/src/sql_tests/models.rs b/crates/sqlbuild-discovery/src/sql_tests/models.rs index 9e04eae633..7d8e607a66 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/models.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/models.rs @@ -1,13 +1,15 @@ //! Native results for SQL unit-test and scenario files. use crate::models::DiscoveryFailure; +use sqlbuild_core::text::models::PythonText; use sqlbuild_sqltext::compiler::models::AuthoredValue; -/// The keys each statement header supports, passed from Python's constants. +/// The supported header keys from Python's constants and the Python semantics to reproduce. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SqlTestFileOptions { pub test_keys: Vec, pub scenario_keys: Vec, + pub python: PythonText, } /// One `TEST(...)` block whose header parsed with only supported keys. diff --git a/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs b/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs index 1dd21e6c21..9713220e26 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs @@ -1,8 +1,9 @@ use crate::models::DiscoveryFailure; use crate::sql_tests::_helpers::scenario_parsing::parse_scenario_file; use crate::sql_tests::_helpers::test_blocks::parse_sql_test_file; -use crate::sql_tests::models::SqlTestBlock; +use crate::sql_tests::models::{SqlTestBlock, SqlTestFileOptions}; use crate::sql_tests::tests::test_types::StatementFileTestCase; +use sqlbuild_core::text::main::python_text::python_text; pub(super) type BlockRow = (Vec, String); pub(super) const FILE_PATH: &str = "project/tests/unit/orders.sql"; @@ -11,6 +12,14 @@ fn keys(test_keys: &[&str]) -> Vec { test_keys.iter().map(|key| (*key).to_owned()).collect() } +fn options() -> SqlTestFileOptions { + SqlTestFileOptions { + test_keys: keys(&["name", "mode", "parameters", "cases"]), + scenario_keys: keys(&["description", "tags"]), + python: python_text((3, 12), "15.0.0").expect("Python 3.12 is supported"), + } +} + fn block_row(block: &SqlTestBlock) -> BlockRow { ( block @@ -32,35 +41,31 @@ fn failed_rows(failure: DiscoveryFailure) -> (Vec, Option) { /// The test file's parsed block rows and its failure message. pub(super) fn test_file_rows(test_case: &StatementFileTestCase) -> (Vec, Option) { - parse_sql_test_file( - FILE_PATH, - test_case.contents.to_owned(), - &keys(&["name", "mode", "parameters", "cases"]), + parse_sql_test_file(FILE_PATH, test_case.contents.to_owned(), &options()).map_or_else( + failed_rows, + |file| { + ( + file.blocks.iter().map(block_row).collect(), + file.failure.map(message), + ) + }, ) - .map_or_else(failed_rows, |file| { - ( - file.blocks.iter().map(block_row).collect(), - file.failure.map(message), - ) - }) } /// The scenario file's block row and its failure message. pub(super) fn scenario_rows(test_case: &StatementFileTestCase) -> (Vec, Option) { - parse_scenario_file( - FILE_PATH, - test_case.contents.to_owned(), - &keys(&["description", "tags"]), + parse_scenario_file(FILE_PATH, test_case.contents.to_owned(), &options()).map_or_else( + failed_rows, + |file| { + ( + vec![block_row(&SqlTestBlock { + header_values: file.header_values, + sql_body: file.sql_body, + })], + None, + ) + }, ) - .map_or_else(failed_rows, |file| { - ( - vec![block_row(&SqlTestBlock { - header_values: file.header_values, - sql_body: file.sql_body, - })], - None, - ) - }) } /// The expected rows in the shape the parsers' rows take. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs index 52261b6dff..0879b08b11 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, IntoPyObject, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyTuple; use pyo3::{IntoPyObjectExt, pyfunction, pymethods, wrap_pyfunction}; -use sqlbuild_core::text::main::python_alnum_unicode_version::python_alnum_unicode_version; +use sqlbuild_core::text::main::python_text::python_text; +use sqlbuild_core::text::models::PythonText; use sqlbuild_discovery::declarations::main::declaration_layout::declaration_layout; use sqlbuild_discovery::declarations::models::{ DeclarationFileFact, DeclarationGroup, DeclarationLayout, @@ -161,6 +162,9 @@ fn discover_model_files( request: ModelDiscoveryRequest, tree: &NativeProjectTree, ) -> PyResult>> { + let Some(python) = python_text(request.python_version, &request.unicode_version) else { + return Ok(None); + }; let root = ProjectRoot { directory: PathBuf::from(request.project_dir), display_prefix: request.display_prefix, @@ -170,6 +174,7 @@ fn discover_model_files( removed_keys: request.removed_keys, extract_implicit_alias_columns: request.extract_implicit_alias_columns, extract_output_column_locations: request.extract_output_column_locations, + python, }; let discovered: Result>, StageDeferral> = py .compiler_detach(|| Ok(discover_models(&root, &tree.inner, &options))) @@ -253,8 +258,10 @@ fn scenario_object(py: Python<'_>, file: DiscoveredScenarioFile) -> PyResult (ProjectRoot, SqlTestFileOptions) { - ( +/// The root and options of one request, or `None` when native cannot reproduce this Python. +fn sql_test_inputs(request: SqlTestDiscoveryRequest) -> Option<(ProjectRoot, SqlTestFileOptions)> { + let python: PythonText = python_text(request.python_version, &request.unicode_version)?; + Some(( ProjectRoot { directory: PathBuf::from(request.project_dir), display_prefix: request.display_prefix, @@ -262,8 +269,9 @@ fn sql_test_inputs(request: SqlTestDiscoveryRequest) -> (ProjectRoot, SqlTestFil SqlTestFileOptions { test_keys: request.test_keys, scenario_keys: request.scenario_keys, + python, }, - ) + )) } /// Discover and split the SQL test files natively, or return `None` when Python must run. @@ -273,7 +281,9 @@ fn discover_sql_test_files( request: SqlTestDiscoveryRequest, tree: &NativeProjectTree, ) -> PyResult>> { - let (root, options) = sql_test_inputs(request); + let Some((root, options)) = sql_test_inputs(request) else { + return Ok(None); + }; let discovered: Result>, StageDeferral> = py .compiler_detach(|| Ok(discover_tests(&root, &tree.inner, &options))) .map_err(crate::bindings::_helpers::panics::compiler_error)?; @@ -290,7 +300,9 @@ fn discover_scenario_files( request: SqlTestDiscoveryRequest, tree: &NativeProjectTree, ) -> PyResult>> { - let (root, options) = sql_test_inputs(request); + let Some((root, options)) = sql_test_inputs(request) else { + return Ok(None); + }; let discovered: Result>, StageDeferral> = py .compiler_detach(|| Ok(discover_scenarios(&root, &tree.inner, &options))) .map_err(crate::bindings::_helpers::panics::compiler_error)?; @@ -337,16 +349,19 @@ fn load_yaml_files( } } +/// Whether native discovery reproduces the string semantics of this Python and Unicode version. +#[pyfunction] +fn native_text_supported(python_version: (u8, u8), unicode_version: &str) -> bool { + python_text(python_version, unicode_version).is_some() +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(native_text_supported, module)?)?; module.add_function(wrap_pyfunction!(load_yaml_files, module)?)?; module.add_function(wrap_pyfunction!(discover_sql_test_files, module)?)?; module.add_function(wrap_pyfunction!(discover_scenario_files, module)?)?; module.add_class::()?; module.add_function(wrap_pyfunction!(discover_model_files, module)?)?; module.add_function(wrap_pyfunction!(discover_declaration_layout, module)?)?; - module.add( - "PYTHON_ALNUM_UNICODE_VERSION", - python_alnum_unicode_version(), - )?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs index 8697cff25d..00e219eafa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs @@ -17,8 +17,8 @@ use sqlbuild_core::json::models::{ use sqlbuild_core::text::main::close_matches::close_matches; use sqlbuild_core::text::main::decode_python_text::decode_python_text; use sqlbuild_core::text::main::is_python_alnum::is_python_alnum; -use sqlbuild_core::text::main::python_alnum_unicode_version::python_alnum_unicode_version; use sqlbuild_core::text::main::python_cleandoc::python_cleandoc; +use sqlbuild_core::text::main::python_text::python_text; use sqlbuild_core::text::models::LineIndex; use std::path::Path; @@ -231,12 +231,19 @@ fn _oracle_project_config(project_dir: &str) -> String { } #[pyfunction] -fn _oracle_python_alnum(code_points: Vec) -> (String, Vec) { - ( - python_alnum_unicode_version().to_owned(), +fn _oracle_python_alnum( + python_version: (u8, u8), + unicode_version: &str, + code_points: Vec, +) -> Option> { + let python = python_text(python_version, unicode_version)?; + Some( code_points .into_iter() - .map(|code_point| char::from_u32(code_point).is_some_and(is_python_alnum)) + .map(|code_point| { + char::from_u32(code_point) + .is_some_and(|character| is_python_alnum(python, character)) + }) .collect(), ) } @@ -253,8 +260,18 @@ fn _oracle_close_matches( } #[pyfunction] -fn _oracle_cleandoc(texts: Vec) -> Vec { - texts.iter().map(|text| python_cleandoc(text)).collect() +fn _oracle_cleandoc( + python_version: (u8, u8), + unicode_version: &str, + texts: Vec, +) -> Option> { + let python = python_text(python_version, unicode_version)?; + Some( + texts + .iter() + .map(|text| python_cleandoc(python, text)) + .collect(), + ) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs index 800836b3df..bf31ede3ef 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -116,6 +116,8 @@ pub(crate) struct ModelDiscoveryRequest { pub(crate) removed_keys: Vec, pub(crate) extract_implicit_alias_columns: bool, pub(crate) extract_output_column_locations: bool, + pub(crate) python_version: (u8, u8), + pub(crate) unicode_version: String, } /// One native SQL test or scenario discovery request from the Python discovery facade. @@ -126,4 +128,6 @@ pub(crate) struct SqlTestDiscoveryRequest { pub(crate) display_prefix: String, pub(crate) test_keys: Vec, pub(crate) scenario_keys: Vec, + pub(crate) python_version: (u8, u8), + pub(crate) unicode_version: String, } diff --git a/scripts/generate_python_char_table.py b/scripts/generate_python_char_table.py index 3d5d6ee034..1aa907d86c 100644 --- a/scripts/generate_python_char_table.py +++ b/scripts/generate_python_char_table.py @@ -1,4 +1,4 @@ -"""Direct wrapper for regenerating the native `str.isalnum()` table; run it with the CI Python.""" +"""Direct wrapper for generating the native `str.isalnum()` table of the running Python.""" from __future__ import annotations diff --git a/scripts/python_char_table/_helpers/table.py b/scripts/python_char_table/_helpers/table.py index 29c505aa17..89d9822654 100644 --- a/scripts/python_char_table/_helpers/table.py +++ b/scripts/python_char_table/_helpers/table.py @@ -3,8 +3,21 @@ from __future__ import annotations import unicodedata +from pathlib import Path -from scripts.python_char_table.constants import MAX_CODE_POINT +from scripts.python_char_table.constants import ( + MAX_CODE_POINT, + TABLE_DIRECTORY, + TABLE_FILE_PREFIX, + TABLE_FILE_SUFFIX, +) + + +def table_path() -> Path: + """Return the Rust file holding the table of the running Python's Unicode version.""" + + version: str = unicodedata.unidata_version.replace(".", "_") + return TABLE_DIRECTORY / f"{TABLE_FILE_PREFIX}{version}{TABLE_FILE_SUFFIX}" def alnum_ranges() -> list[tuple[int, int]]: @@ -25,17 +38,14 @@ def alnum_ranges() -> list[tuple[int, int]]: def render_table(ranges: list[tuple[int, int]]) -> str: - """Render the Rust source of the range table.""" + """Render the Rust expression of the range table, which `alnum_tables.rs` includes.""" lines: list[str] = [ - "//! Generated by `scripts/generate_python_char_table.py`; do not edit.", - "", - "/// The Unicode version of the Python that generated the table.", - f'pub(crate) const PYTHON_ALNUM_UNICODE_VERSION: &str = "{unicodedata.unidata_version}";', - "", - "/// Inclusive code point ranges where Python's `str.isalnum()` is true, in order.", - "pub(crate) const PYTHON_ALNUM_RANGES: &[(u32, u32)] = &[", + "// Generated by `scripts/generate_python_char_table.py`; do not edit.", + f"// Inclusive code point ranges where `str.isalnum()` is true in Unicode " + f"{unicodedata.unidata_version}, in order.", + "&[", ] lines.extend(f" (0x{start:04X}, 0x{end:04X})," for start, end in ranges) - lines.append("];") + lines.append("]") return "\n".join(lines) + "\n" diff --git a/scripts/python_char_table/constants.py b/scripts/python_char_table/constants.py index 46faf65387..2b55fcb5f9 100644 --- a/scripts/python_char_table/constants.py +++ b/scripts/python_char_table/constants.py @@ -2,13 +2,14 @@ from pathlib import Path -TABLE_PATH: Path = ( +TABLE_DIRECTORY: Path = ( Path(__file__).resolve().parents[2] / "crates" / "sqlbuild-core" / "src" / "text" - / "_helpers" - / "alnum_ranges.rs" + / "alnum_ranges" ) +TABLE_FILE_PREFIX: str = "unicode_" +TABLE_FILE_SUFFIX: str = ".in" MAX_CODE_POINT: int = 0x10FFFF diff --git a/scripts/python_char_table/main/generate.py b/scripts/python_char_table/main/generate.py index b0e9113c64..47877b4291 100644 --- a/scripts/python_char_table/main/generate.py +++ b/scripts/python_char_table/main/generate.py @@ -1,17 +1,18 @@ -"""Focused entry for regenerating the native `str.isalnum()` table with the CI Python.""" +"""Focused entry for generating the native `str.isalnum()` table of the running Python.""" from __future__ import annotations import sys import unicodedata +from pathlib import Path -from scripts.python_char_table._helpers.table import alnum_ranges, render_table -from scripts.python_char_table.constants import TABLE_PATH +from scripts.python_char_table._helpers.table import alnum_ranges, render_table, table_path def generate_python_char_table() -> int: - """Write the table next to the native text helpers and report its Unicode version.""" + """Write the table of this Python's Unicode version next to the native text helpers.""" - _ = TABLE_PATH.write_text(render_table(alnum_ranges()), encoding="utf-8") - print(f"wrote {TABLE_PATH} (Unicode {unicodedata.unidata_version})", file=sys.stderr) + path: Path = table_path() + _ = path.write_text(render_table(alnum_ranges()), encoding="utf-8") + print(f"wrote {path} (Unicode {unicodedata.unidata_version})", file=sys.stderr) return 0 diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 2026ce2ab1..5b66d1ea59 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -43,8 +43,7 @@ def parse_model_headers( headers: list[str], ) -> list[tuple[dict[str, object] | None, list[tuple[str, int, int]] | None, str | None]]: ... def tokenize_model_header(header: str) -> list[tuple[int, str, int]]: ... - -PYTHON_ALNUM_UNICODE_VERSION: str +def native_text_supported(python_version: tuple[int, int], unicode_version: str) -> bool: ... class NativeProjectTree: def __init__(self, project_dir: str) -> None: ... @@ -84,8 +83,12 @@ def _oracle_text_positions( def _oracle_yaml_load(text: str) -> str: ... def _oracle_toml_load(text: str) -> str: ... def _oracle_project_config(project_dir: str) -> str: ... -def _oracle_python_alnum(code_points: list[int]) -> tuple[str, list[bool]]: ... -def _oracle_cleandoc(texts: list[str]) -> list[str]: ... +def _oracle_python_alnum( + python_version: tuple[int, int], unicode_version: str, code_points: list[int] +) -> list[bool] | None: ... +def _oracle_cleandoc( + python_version: tuple[int, int], unicode_version: str, texts: list[str] +) -> list[str] | None: ... def _oracle_close_matches( word: str, possibilities: list[str], count: int, cutoff: float ) -> list[str]: ... diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py index a50aa42d3f..8f0ea0ee33 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py @@ -16,6 +16,7 @@ native_failure, native_locations, native_project_tree, + native_text_runtime, seed_snapshot_listings, ) from sqlbuild.compiler.discovery._helpers.sql.declarations import ( @@ -89,6 +90,7 @@ def _discover_native_files( "removed_keys": sorted(REMOVED_SQL_MODEL_HEADER_KEYS), "extract_implicit_alias_columns": extract_implicit_alias_columns, "extract_output_column_locations": extract_output_column_locations, + **native_text_runtime(), }, tree, ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py index aeff70119d..e72357241a 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +import sys import unicodedata from pathlib import Path @@ -65,12 +66,18 @@ def native_locations( } +def native_text_runtime() -> dict[str, object]: + """Return the request fields naming the Python whose string semantics native reproduces.""" + + return {"python_version": _python_version(), "unicode_version": unicodedata.unidata_version} + + def native_discovery_supported(*, project_dir: Path, display_prefix: str) -> bool: - """Whether native discovery reproduces Python here: same Unicode data, UTF-8 paths, POSIX.""" + """Whether native discovery reproduces Python here: known Python, UTF-8 paths, POSIX.""" return ( os.name != _WINDOWS_OS_NAME - and unicodedata.unidata_version == _native.PYTHON_ALNUM_UNICODE_VERSION + and _native.native_text_supported(_python_version(), unicodedata.unidata_version) and _is_utf8_text(str(project_dir)) and _is_utf8_text(display_prefix) ) @@ -118,3 +125,7 @@ def _is_utf8_text(text: str) -> bool: except UnicodeEncodeError: return False return True + + +def _python_version() -> tuple[int, int]: + return (sys.version_info[0], sys.version_info[1]) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py index eb10063887..89391f59a2 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py @@ -16,6 +16,7 @@ native_display_prefix, native_failure, native_project_tree, + native_text_runtime, seed_snapshot_listings, ) from sqlbuild.compiler.discovery._helpers.sql.model_files import project_native_header_values @@ -89,6 +90,7 @@ def _native_files(*, project_dir: Path, discover: _NativeDiscovery) -> _NativeFi "display_prefix": display_prefix, "test_keys": sorted(SQL_TEST_HEADER_KEYS), "scenario_keys": sorted(SQL_SCENARIO_HEADER_KEYS), + **native_text_runtime(), }, tree, ) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index 37de05fda3..9157d3e3ea 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -1,9 +1,9 @@ from __future__ import annotations +import sys +import unicodedata from dataclasses import dataclass, field -import sqlbuild._native as _native - @dataclass(frozen=True) class GeneratedDocumentOracleTestCase: @@ -83,7 +83,10 @@ class NativeDeferralTestCase: description: str project_name: str expected_native_calls: int - unidata_version: str = field(default_factory=lambda: _native.PYTHON_ALNUM_UNICODE_VERSION) + unidata_version: str = field(default_factory=lambda: unicodedata.unidata_version) + python_version: tuple[int, int] = field( + default_factory=lambda: (sys.version_info[0], sys.version_info[1]) + ) @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py index 13d227cd94..da3da2ab9b 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py @@ -4,6 +4,7 @@ import os import random +import sys import unicodedata from pathlib import Path @@ -170,6 +171,20 @@ def test_given_project_when_discovering_through_the_engine_switch_then_inputs_ma unidata_version="0.0.0", expected_native_calls=0, ), + NativeDeferralTestCase( + description="another supported Python and its Unicode data run natively", + project_name="orders", + unidata_version="16.0.0", + python_version=(3, 14), + expected_native_calls=1, + ), + NativeDeferralTestCase( + description="an unreleased Python defers to Python", + project_name="orders", + unidata_version="16.0.0", + python_version=(3, 15), + expected_native_calls=0, + ), NativeDeferralTestCase( description="a non-UTF-8 project root defers to Python", project_name=os.fsdecode(b"orders\xff"), @@ -187,6 +202,7 @@ def test_given_runtime_and_root_when_discovering_natively_then_python_runs_where counter: CallCounter = CallCounter(_native.discover_model_files) monkeypatch.setattr(_native, "discover_model_files", counter) monkeypatch.setattr(unicodedata, "unidata_version", test_case.unidata_version) + monkeypatch.setattr(sys, "version_info", (*test_case.python_version, 0, "final", 0)) native: object = model_discovery_outcome(project_dir=project_dir, native=True) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_file_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_file_discovery.py index 6f6ce814e9..7a773da3f6 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_file_discovery.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_file_discovery.py @@ -8,7 +8,6 @@ import pytest -import sqlbuild._native as _native from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( EngineSwitchParityTestCase, FactCacheFallbackTestCase, @@ -191,7 +190,7 @@ def test_given_big_integers_when_loading_natively_then_values_and_error_order_ma ), FactCacheFallbackTestCase( description="native discovery itself reads no cached facts", - unidata_version=_native.PYTHON_ALNUM_UNICODE_VERSION, + unidata_version=unicodedata.unidata_version, expected_same_keys_as_python=False, expected_native_keys=0, ), diff --git a/tests/integration/src/sqlbuild/compiler/helpers.py b/tests/integration/src/sqlbuild/compiler/helpers.py index 0a0e398415..1043358d25 100644 --- a/tests/integration/src/sqlbuild/compiler/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/helpers.py @@ -301,7 +301,6 @@ def python_read_text(*, path: Path, data: bytes) -> str: return path.read_text(encoding="utf-8") -TABLE_PYTHON_VERSION: tuple[int, int] = (3, 12) CLEANDOC_FRAGMENTS: tuple[str, ...] = ( "", " ", " ", "\t", " \t", "\u3000", "\x0b", "\x1c", "\xa0", "\r", "\n", "\n", "\n\n", "SELECT 1", "a", "é", "x\ty", "--c", diff --git a/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py b/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py index 0a3f83f3e8..a715730cbc 100644 --- a/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py +++ b/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py @@ -18,17 +18,20 @@ CloseMatchesOracleTestCase, ) from tests.integration.src.sqlbuild.compiler.helpers import ( - TABLE_PYTHON_VERSION, cleandoc_text, mismatches, mutated_word, ) - -@pytest.mark.skipif( - sys.version_info[:2] != TABLE_PYTHON_VERSION, - reason="the native table reproduces the CI Python; other versions defer to Python discovery", +RUNTIME: tuple[tuple[int, int], str] = ( + (sys.version_info[0], sys.version_info[1]), + unicodedata.unidata_version, ) +NATIVE_TEXT_SUPPORTED: bool = _native.native_text_supported(*RUNTIME) +UNSUPPORTED_REASON: str = "native discovery defers to Python on this Python or Unicode version" + + +@pytest.mark.skipif(not NATIVE_TEXT_SUPPORTED, reason=UNSUPPORTED_REASON) @pytest.mark.parametrize( "test_case", [CharacterClassOracleTestCase(description="every code point", first=0, last=sys.maxunicode)], @@ -38,10 +41,9 @@ def test_given_code_points_when_classifying_natively_then_isalnum_matches_python test_case: CharacterClassOracleTestCase, ) -> None: code_points: list[int] = list(range(test_case.first, test_case.last + 1)) - assert _native.PYTHON_ALNUM_UNICODE_VERSION == unicodedata.unidata_version expected: list[object] = [chr(code_point).isalnum() for code_point in code_points] - actual: list[object] = list(_native._oracle_python_alnum(code_points)[1]) + actual: list[object] = list(_native._oracle_python_alnum(*RUNTIME, code_points) or ()) assert mismatches(inputs=list(code_points), expected=expected, actual=actual) == list( test_case.expected_mismatches @@ -91,10 +93,7 @@ def test_given_words_when_finding_close_matches_natively_then_difflib_agrees( ) -@pytest.mark.skipif( - sys.version_info[:2] != TABLE_PYTHON_VERSION, - reason="native discovery runs only on the CI Python, whose cleandoc the port reproduces", -) +@pytest.mark.skipif(not NATIVE_TEXT_SUPPORTED, reason=UNSUPPORTED_REASON) @pytest.mark.parametrize( "test_case", [ @@ -111,7 +110,7 @@ def test_given_bodies_when_cleaning_natively_then_inspect_cleandoc_agrees( texts: list[str] = [cleandoc_text(rng=rng) for _ in range(test_case.case_count)] expected: list[object] = [inspect.cleandoc(text) for text in texts] - actual: list[object] = list(_native._oracle_cleandoc(texts)) + actual: list[object] = list(_native._oracle_cleandoc(*RUNTIME, texts) or ()) assert mismatches(inputs=list(texts), expected=expected, actual=actual) == list( test_case.expected_mismatches From f0c9847f1cb115fcea6c60a9d4d2bf3d0e797cea Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:14:12 +0100 Subject: [PATCH 2/5] feat(native): make native discovery the default compiler engine With SQLBUILD_COMPILER_ENGINE unset, compiles now discover project inputs natively. The differential harness is byte-identical on the per-PR corpus (on Python 3.12, 3.13 and 3.14), the dense corpus and a private real project, and cold, edit and no-change compiles are no slower. SQLBUILD_COMPILER_ENGINE=python keeps the Python discovery as the oracle, and CI keeps comparing both engines. Compiler and Rules stores stay namespaced by engine, so the first compile after upgrading is cold once; semantic fingerprints do not change. Tests that hardcoded the Python engine's cache paths now resolve the active engine's directories, and an E2E test proves the unset default runs native discovery with output identical to the Python engine. --- .../compiler/frontier/_helpers/engine.py | 2 +- src/sqlbuild/compiler/frontier/constants.py | 2 +- src/sqlbuild/compiler/scopes/constants.py | 3 -- .../test_runtime_artifact_preservation.py | 3 +- .../cli/commands/main/compile/helpers.py | 12 +++++-- .../compile/test_compiler_engine_switch.py | 35 +++++++++++++++++++ .../cli/commands/main/rules/helpers.py | 9 +++++ .../rules/test_rules_incremental_sequence.py | 9 ++--- .../cli/commands/main/scope/helpers.py | 5 +-- .../sqlbuild/cli/commands/main/test_rules.py | 11 +++--- .../test_compiler_engine_reuse_key.py | 8 ++--- .../compile/_helpers/test_analysis_cache.py | 17 ++++----- .../frontier/test_resolve_compiler_engine.py | 6 ++-- .../compiler/scopes/test_offline_cache.py | 30 +++++++++++----- 14 files changed, 110 insertions(+), 42 deletions(-) diff --git a/src/sqlbuild/compiler/frontier/_helpers/engine.py b/src/sqlbuild/compiler/frontier/_helpers/engine.py index 875cbbd141..93b9cb5ff0 100644 --- a/src/sqlbuild/compiler/frontier/_helpers/engine.py +++ b/src/sqlbuild/compiler/frontier/_helpers/engine.py @@ -12,7 +12,7 @@ def active_compiler_engine() -> CompilerEngine: - """Return the engine selected by SQLBUILD_COMPILER_ENGINE, defaulting to Python.""" + """Return the engine selected by SQLBUILD_COMPILER_ENGINE, defaulting to native.""" raw_value: str | None = os.environ.get(COMPILER_ENGINE_ENV_VAR) if not raw_value: diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index b74f1abdf5..68818cdc2a 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -4,7 +4,7 @@ COMPILER_ENGINE_ENV_VAR: str = "SQLBUILD_COMPILER_ENGINE" COMPILER_ENGINE_OPTION: str = "--compiler-engine" -DEFAULT_COMPILER_ENGINE: CompilerEngine = CompilerEngine.PYTHON +DEFAULT_COMPILER_ENGINE: CompilerEngine = CompilerEngine.NATIVE COMPILER_ENGINE_VALUES: tuple[str, ...] = tuple(engine.value for engine in CompilerEngine) NATIVE_CACHE_NAMESPACE_SUFFIX: str = "-native-v1" TARGET_DIRECTORY_NAME: str = "target" diff --git a/src/sqlbuild/compiler/scopes/constants.py b/src/sqlbuild/compiler/scopes/constants.py index a78506c9bf..02d72e4e92 100644 --- a/src/sqlbuild/compiler/scopes/constants.py +++ b/src/sqlbuild/compiler/scopes/constants.py @@ -2,15 +2,12 @@ from __future__ import annotations -from pathlib import Path - from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind SCOPE_METADATA_SCHEMA_VERSION: int = 3 SCOPE_CACHE_SCHEMA_VERSION: int = 3 SCOPE_FINGERPRINT_ALGORITHM_VERSION: int = 2 SCOPE_CACHE_DIRECTORY_NAME: str = "declaration-scopes-v2" -SCOPE_CACHE_DIRECTORY: Path = Path("target/cache/compiler") / SCOPE_CACHE_DIRECTORY_NAME SCOPE_CACHE_FILENAME: str = "scope-index.json" SCOPE_CACHE_MAX_BYTES: int = 16 * 1024 * 1024 SCOPE_SOURCE_SUFFIXES: frozenset[str] = frozenset({".sql", ".yml", ".yaml", ".py"}) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py index 79487b7949..789e841de6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( RuntimeArtifactPreservationBuildE2ETestCase, ) @@ -73,7 +74,7 @@ def test_given_full_build_when_running_selected_rerun_then_existing_runtime_arti path = project_dir / relative_path assert path.exists(), f"expected compiled artifact to exist: {path}" compiler_caches: tuple[Path, ...] = tuple( - (project_dir / "target" / "cache" / "compiler").rglob("*.sqlite3") + compiler_cache_directory(project_dir).rglob("*.sqlite3") ) assert len(compiler_caches) == test_case.expected_compiler_cache_count assert (project_dir / ".sqlbuild").exists() is test_case.expected_local_history diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 40101e5a8f..08293a650f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -54,6 +54,8 @@ REUSE_RENDER_STATE_SUFFIX, ) from sqlbuild.cli.compile_reuse.models import StoredCompileHeader, StoredCompileInputs +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( FreshProcessCompileCachePerformanceGuardTestCase, IncrementalEditStep, @@ -617,7 +619,7 @@ def run_fresh_process_compile_cache_benchmark( compile_args=(), ) ) - cache_dir: Path = target_dir / "cache" / "compiler" + cache_dir: Path = compiler_cache_directory(project_dir) cache_bytes: int = sum(path.stat().st_size for path in cache_dir.rglob("*.json")) + sum( path.stat().st_size for path in cache_dir.rglob("*.sqlite3") ) @@ -3133,7 +3135,7 @@ def compile_reuse_entry_paths(*, project_dir: Path) -> tuple[Path, ...]: """Return every stored compile reuse entry of a project.""" return tuple( - sorted((project_dir / "target" / "cache" / "compiler" / "project-reuse-v1").glob("*.entry")) + sorted((compiler_cache_directory(project_dir) / "project-reuse-v1").glob("*.entry")) ) @@ -4163,6 +4165,12 @@ def engine_reuse_compile(*, project_dir: Path, engine: str) -> CompileReuseRun: return run_reuse_compile(project_dir=project_dir, global_args=("--compiler-engine", engine)) +def environment_engine_reuse_compile(*, project_dir: Path, engine: str) -> CompileReuseRun: + """Compile with reuse enabled under the engine the environment selects; empty means default.""" + + return run_reuse_compile(project_dir=project_dir, env={COMPILER_ENGINE_ENV_VAR: engine}) + + def engine_compile_and_rules(*, project_dir: Path, engine: str, rules_selector: str) -> int: """Compile and run Rules under one engine; return the Rules exit code.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py index b8fcbd0d16..12439f824d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py @@ -16,6 +16,7 @@ copy_compile_project, engine_compile_and_rules, engine_reuse_compile, + environment_engine_reuse_compile, prepare_compile_reuse_project, report_engine, report_without_engine, @@ -57,6 +58,40 @@ def test_given_same_project_when_compiling_with_each_engine_then_outputs_are_ide assert left.compiled +@pytest.mark.parametrize( + "test_case", + [ + CompilerEngineParityTestCase( + description="python_oracle_and_unset_default", + left_engine="python", + right_engine="", + expected_engines=("python", "native"), + expected_exit_codes=(0, 0), + ) + ], + ids=lambda case: case.description, +) +def test_given_no_engine_selection_when_compiling_then_native_runs_and_matches_python( + test_case: CompilerEngineParityTestCase, tmp_path: Path +) -> None: + prepared_project: Path = tmp_path / "orders" + prepare_compile_reuse_project(project_dir=prepared_project) + oracle: CompileReuseRun = environment_engine_reuse_compile( + project_dir=copy_compile_project(source=prepared_project, destination=tmp_path / "oracle"), + engine=test_case.left_engine, + ) + default: CompileReuseRun = environment_engine_reuse_compile( + project_dir=copy_compile_project(source=prepared_project, destination=tmp_path / "default"), + engine=test_case.right_engine, + ) + + assert (oracle.returncode, default.returncode) == test_case.expected_exit_codes + assert (report_engine(oracle), report_engine(default)) == test_case.expected_engines + assert report_without_engine(default) == report_without_engine(oracle) + assert default.compiled == oracle.compiled + assert oracle.compiled + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index 82f8007dd0..c4013f5e32 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -11,6 +11,7 @@ from typing import Any from scripts.rules_benchmark._helpers.custom_rules import write_custom_rules +from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import replace_project_text __all__ = ("write_custom_rules",) @@ -592,3 +593,11 @@ def move_order_totals_description_to_path_defaults(root: Path) -> None: + '\n[path_defaults.marts]\ndescription = "Order totals per order"\n', encoding="utf-8", ) + + +def write_bulk_rules_cache(*, root: Path, file_name: str, text: str) -> None: + """Overwrite one bulk Rules cache file of the active compiler engine.""" + + _ = rules_bulk_cache_path(project_dir=root, file_name=file_name).write_text( + text, encoding="utf-8" + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_incremental_sequence.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_incremental_sequence.py index b9ec54381c..c4c373c8bc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_incremental_sequence.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_incremental_sequence.py @@ -18,6 +18,7 @@ BUILT_IN_RULES_PROJECT, rule_cache_counts, rules_compile_outcome, + write_bulk_rules_cache, write_project_files, ) @@ -144,14 +145,14 @@ def test_given_warm_rules_cache_when_applying_edit_chain_then_every_compile_equa ( RulesCacheEditCase( "corrupt_model_findings_cache", - lambda root: (root / "target/rules-cache/bulk/native.json").write_text( - '{"fingerprint": "trunc', encoding="utf-8" + lambda root: write_bulk_rules_cache( + root=root, file_name="native.json", text='{"fingerprint": "trunc' ), ), RulesCacheEditCase( "corrupt_response_memo", - lambda root: (root / "target/rules-cache/bulk/native-response.json").write_text( - '{"identity": ', encoding="utf-8" + lambda root: write_bulk_rules_cache( + root=root, file_name="native-response.json", text='{"identity": ' ), ), ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py index e2af4ae838..0ace9735ce 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py @@ -8,7 +8,8 @@ from pathlib import Path from time import perf_counter -from sqlbuild.compiler.scopes.constants import SCOPE_CACHE_DIRECTORY, SCOPE_CACHE_FILENAME +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory +from sqlbuild.compiler.scopes.constants import SCOPE_CACHE_DIRECTORY_NAME, SCOPE_CACHE_FILENAME from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import REPO_ROOT @@ -150,4 +151,4 @@ def median_seconds(values: tuple[float, ...]) -> float: def scope_cache_path(*, project_dir: Path) -> Path: """Return the persistent Scope Explorer cache path.""" - return project_dir / SCOPE_CACHE_DIRECTORY / SCOPE_CACHE_FILENAME + return compiler_cache_directory(project_dir) / SCOPE_CACHE_DIRECTORY_NAME / SCOPE_CACHE_FILENAME diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index ee2a0d9085..d449de76ef 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -23,6 +23,7 @@ from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.lint.classes.stop_context import LintStopContext from sqlbuild.lint.models import LintRunResult +from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( CustomHostSplitIntegrationTestCase, DynamicPivotRulesIntegrationTestCase, @@ -747,7 +748,7 @@ def test_given_cached_sql_test_rules_when_test_or_macro_changes_then_findings_ar assert plain_finding not in after_test_edit assert macro_finding in after_test_edit assert macro_finding not in after_macro_edit - assert (tmp_path / "target" / "rules-cache" / "bulk" / "sql.json").is_file() + assert rules_bulk_cache_path(project_dir=tmp_path, file_name="sql.json").is_file() @pytest.mark.parametrize( @@ -1104,7 +1105,9 @@ def compile_payload(*, hosts: int) -> dict[str, Any]: monkeypatch.setattr(native_module, "run_custom_host_json", recording_host) single: dict[str, Any] = compile_payload(hosts=1) single_runs: int = len(host_runs) - (tmp_path / "target" / "rules-cache").rename(tmp_path / "single-rules-cache") + rules_bulk_cache_path(project_dir=tmp_path, file_name="custom-rules.json").parent.parent.rename( + tmp_path / "single-rules-cache" + ) split: dict[str, Any] = compile_payload(hosts=test_case.hosts) split_runs: int = len(host_runs) - single_runs warm: dict[str, Any] = compile_payload(hosts=test_case.hosts) @@ -1115,8 +1118,8 @@ def compile_payload(*, hosts: int) -> dict[str, Any]: assert warm["diagnostics"] == single["diagnostics"] assert warm["compile_timings"]["rule_cache_hits"] >= test_case.model_count assert (tmp_path / "single-rules-cache" / "bulk" / "custom-rules.json").read_bytes() == ( - tmp_path / "target" / "rules-cache" / "bulk" / "custom-rules.json" - ).read_bytes() + rules_bulk_cache_path(project_dir=tmp_path, file_name="custom-rules.json").read_bytes() + ) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_compiler_engine_reuse_key.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_compiler_engine_reuse_key.py index a8cd940fe7..e122193509 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_compiler_engine_reuse_key.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_compiler_engine_reuse_key.py @@ -31,15 +31,15 @@ expected_same_digest=False, ), EngineReuseDigestTestCase( - description="unset_then_native", + description="unset_then_python", first_environment={}, - second_environment={COMPILER_ENGINE_ENV_VAR: "native"}, + second_environment={COMPILER_ENGINE_ENV_VAR: "python"}, expected_same_digest=False, ), EngineReuseDigestTestCase( - description="unset_then_explicit_python", + description="unset_then_explicit_native", first_environment={}, - second_environment={COMPILER_ENGINE_ENV_VAR: "python"}, + second_environment={COMPILER_ENGINE_ENV_VAR: "native"}, expected_same_digest=True, ), EngineReuseDigestTestCase( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py index 30ab6049fb..f96a30d800 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py @@ -31,6 +31,7 @@ CompileSqlReference, PolyglotAnalysisResult, ) +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.compiler.lineage.types import ( ColumnLineageConfidence, ColumnTransformKind, @@ -145,7 +146,7 @@ def test_given_successful_analysis_when_compiling_again_then_reuses_identical_ca analyzer.assert_not_called() signature_builder.assert_not_called() reference_scanner.assert_not_called() - assert len(tuple((tmp_path / "target" / "cache" / "compiler").rglob("*.sqlite3"))) == ( + assert len(tuple((compiler_cache_directory(tmp_path)).rglob("*.sqlite3"))) == ( test_case.expected_count + 1 ) @@ -193,7 +194,7 @@ def test_given_exact_compact_batch_when_entry_rows_are_absent_then_warm_compile_ write_repo_files(tmp_path, _CACHE_REPO_FILES) monkeypatch.setattr(assembly_project, "_COMPACT_BATCH_CACHE_MIN_MODEL_COUNT", 1) cold_project: CompiledProject = compile_project_with_cache(project_dir=tmp_path) - cache_path: Path = tmp_path / "target" / "cache" / "compiler" / "v12" / "model-analysis.sqlite3" + cache_path: Path = compiler_cache_directory(tmp_path) / "v12" / "model-analysis.sqlite3" with sqlite3.connect(cache_path) as connection: batch_count: int = connection.execute( "SELECT COUNT(*) FROM model_analysis_compact_batch" @@ -226,7 +227,7 @@ def test_given_one_changed_model_when_entry_rows_are_absent_then_compact_batch_r write_repo_files(tmp_path, _SELECTION_REPO_FILES) monkeypatch.setattr(assembly_project, "_COMPACT_BATCH_CACHE_MIN_MODEL_COUNT", 1) cold_project: CompiledProject = compile_project_with_cache(project_dir=tmp_path) - cache_path: Path = tmp_path / "target" / "cache" / "compiler" / "v12" / "model-analysis.sqlite3" + cache_path: Path = compiler_cache_directory(tmp_path) / "v12" / "model-analysis.sqlite3" with sqlite3.connect(cache_path) as connection: _ = connection.execute("DELETE FROM model_analysis") (tmp_path / "models" / "unrelated.sql").write_text( @@ -264,7 +265,7 @@ def test_given_corrupt_compact_batch_when_entry_rows_are_valid_then_warm_compile write_repo_files(tmp_path, _CACHE_REPO_FILES) monkeypatch.setattr(assembly_project, "_COMPACT_BATCH_CACHE_MIN_MODEL_COUNT", 1) cold_project: CompiledProject = compile_project_with_cache(project_dir=tmp_path) - cache_path: Path = tmp_path / "target" / "cache" / "compiler" / "v12" / "model-analysis.sqlite3" + cache_path: Path = compiler_cache_directory(tmp_path) / "v12" / "model-analysis.sqlite3" with sqlite3.connect(cache_path) as connection: _ = connection.execute( "UPDATE model_analysis_compact_batch SET payload = ?", @@ -315,7 +316,7 @@ def test_given_corrupt_analysis_when_compiling_then_reanalyzes_and_repairs_the_e ) -> None: write_repo_files(tmp_path, _CACHE_REPO_FILES) cold_project: CompiledProject = compile_project_with_cache(project_dir=tmp_path) - cache_path: Path = tmp_path / "target" / "cache" / "compiler" / "v12" / "model-analysis.sqlite3" + cache_path: Path = compiler_cache_directory(tmp_path) / "v12" / "model-analysis.sqlite3" with sqlite3.connect(cache_path) as connection: persisted_contents: str = connection.execute( "SELECT payload FROM model_analysis" @@ -356,7 +357,7 @@ def test_given_non_text_analysis_cache_when_compiling_then_reanalyzes_safely( ) -> None: write_repo_files(tmp_path, _CACHE_REPO_FILES) _ = compile_project_with_cache(project_dir=tmp_path) - cache_path: Path = tmp_path / "target" / "cache" / "compiler" / "v12" / "model-analysis.sqlite3" + cache_path: Path = compiler_cache_directory(tmp_path) / "v12" / "model-analysis.sqlite3" with sqlite3.connect(cache_path) as connection: _ = connection.execute( "UPDATE model_analysis SET payload = ?", @@ -384,7 +385,7 @@ def test_given_corrupt_reference_cache_when_compiling_then_rescans_and_repairs_t write_repo_files(tmp_path, _CACHE_REPO_FILES) cold_project: CompiledProject = compile_project_with_cache(project_dir=tmp_path) cache_path: Path = next( - (tmp_path / "target" / "cache" / "compiler" / "references-v2").glob("*.sqlite3") + (compiler_cache_directory(tmp_path) / "references-v2").glob("*.sqlite3") ) with sqlite3.connect(cache_path) as connection: persisted_contents: str = connection.execute( @@ -424,7 +425,7 @@ def test_given_non_text_reference_cache_when_compiling_then_rescans_safely( write_repo_files(tmp_path, _CACHE_REPO_FILES) _ = compile_project_with_cache(project_dir=tmp_path) cache_path: Path = next( - (tmp_path / "target" / "cache" / "compiler" / "references-v2").glob("*.sqlite3") + (compiler_cache_directory(tmp_path) / "references-v2").glob("*.sqlite3") ) with sqlite3.connect(cache_path) as connection: _ = connection.execute( diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_resolve_compiler_engine.py b/tests/unit/src/sqlbuild/compiler/frontier/test_resolve_compiler_engine.py index a4c9d19334..ea3f6a405d 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_resolve_compiler_engine.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_resolve_compiler_engine.py @@ -22,7 +22,7 @@ "test_case", [ EngineResolutionTestCase( - description="empty", raw_value="", expected_engine=CompilerEngine.PYTHON + description="empty", raw_value="", expected_engine=CompilerEngine.NATIVE ), EngineResolutionTestCase( description="python", raw_value="python", expected_engine=CompilerEngine.PYTHON @@ -45,12 +45,12 @@ def test_given_engine_variable_when_resolving_then_returns_selected_engine( "test_case", [ EngineResolutionTestCase( - description="unset", raw_value="", expected_engine=CompilerEngine.PYTHON + description="unset", raw_value="", expected_engine=CompilerEngine.NATIVE ) ], ids=lambda case: case.description, ) -def test_given_no_engine_variable_when_resolving_then_python_is_the_default( +def test_given_no_engine_variable_when_resolving_then_native_is_the_default( test_case: EngineResolutionTestCase, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.delenv(COMPILER_ENGINE_ENV_VAR, raising=False) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py index d7167b9f2b..f0bf863cd9 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py @@ -9,8 +9,9 @@ import pytest +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.compiler.scopes._helpers.cache import scope_index_fingerprint -from sqlbuild.compiler.scopes.constants import SCOPE_CACHE_DIRECTORY, SCOPE_CACHE_FILENAME +from sqlbuild.compiler.scopes.constants import SCOPE_CACHE_DIRECTORY_NAME, SCOPE_CACHE_FILENAME from sqlbuild.compiler.scopes.main.load_or_build_scope_index import load_or_build_scope_index from sqlbuild.compiler.scopes.models import DeclarationIdentity, ScopeIndex from sqlbuild.compiler.scopes.types import ( @@ -56,7 +57,9 @@ def test_given_valid_project_when_loading_twice_then_warm_cache_reconstructs_exa write_scope_cache_project(root=tmp_path, write_repo_files=write_repo_files) cold: ScopeIndex = load_or_build_scope_index(project_dir=tmp_path) - cache_path: Path = tmp_path / SCOPE_CACHE_DIRECTORY / SCOPE_CACHE_FILENAME + cache_path: Path = ( + compiler_cache_directory(tmp_path) / SCOPE_CACHE_DIRECTORY_NAME / SCOPE_CACHE_FILENAME + ) first_bytes: bytes = cache_path.read_bytes() warm: ScopeIndex = load_or_build_scope_index(project_dir=tmp_path) @@ -150,9 +153,10 @@ def test_given_no_cache_when_loading_twice_then_cache_is_bypassed( first: ScopeIndex = load_or_build_scope_index(project_dir=tmp_path, no_cache=True) second: ScopeIndex = load_or_build_scope_index(project_dir=tmp_path, no_cache=True) - assert (first == second and not (tmp_path / SCOPE_CACHE_DIRECTORY).exists()) is ( - test_case.expected_result - ) + assert ( + first == second + and not (compiler_cache_directory(tmp_path) / SCOPE_CACHE_DIRECTORY_NAME).exists() + ) is (test_case.expected_result) @pytest.mark.parametrize( @@ -296,7 +300,9 @@ def test_given_authored_values_and_secrets_when_caching_then_payload_is_value_fr ) load_or_build_scope_index(project_dir=tmp_path) - raw: str = (tmp_path / SCOPE_CACHE_DIRECTORY / SCOPE_CACHE_FILENAME).read_text(encoding="ascii") + raw: str = ( + compiler_cache_directory(tmp_path) / SCOPE_CACHE_DIRECTORY_NAME / SCOPE_CACHE_FILENAME + ).read_text(encoding="ascii") assert (secret not in raw and str(tmp_path) not in raw) is test_case.expected_result @@ -482,7 +488,9 @@ def test_given_corrupt_cache_when_loading_then_it_is_a_miss_and_is_atomically_re ) -> None: write_scope_cache_project(root=tmp_path, write_repo_files=write_repo_files) expected: ScopeIndex = load_or_build_scope_index(project_dir=tmp_path) - cache_path: Path = tmp_path / SCOPE_CACHE_DIRECTORY / SCOPE_CACHE_FILENAME + cache_path: Path = ( + compiler_cache_directory(tmp_path) / SCOPE_CACHE_DIRECTORY_NAME / SCOPE_CACHE_FILENAME + ) cache_path.write_bytes(b"\xffnot-json") actual: ScopeIndex = load_or_build_scope_index(project_dir=tmp_path) @@ -512,7 +520,9 @@ def test_given_invalid_cache_envelope_when_loading_then_fault_is_a_repairable_mi ) -> None: write_scope_cache_project(root=tmp_path, write_repo_files=write_repo_files) expected: ScopeIndex = load_or_build_scope_index(project_dir=tmp_path) - cache_path: Path = tmp_path / SCOPE_CACHE_DIRECTORY / SCOPE_CACHE_FILENAME + cache_path: Path = ( + compiler_cache_directory(tmp_path) / SCOPE_CACHE_DIRECTORY_NAME / SCOPE_CACHE_FILENAME + ) test_case.mutate(path=cache_path) actual: ScopeIndex = load_or_build_scope_index(project_dir=tmp_path) @@ -710,7 +720,9 @@ def test_given_partial_project_when_loading_then_incomplete_index_is_not_cached( ) index: ScopeIndex = load_or_build_scope_index(project_dir=tmp_path) - cache_path: Path = tmp_path / SCOPE_CACHE_DIRECTORY / SCOPE_CACHE_FILENAME + cache_path: Path = ( + compiler_cache_directory(tmp_path) / SCOPE_CACHE_DIRECTORY_NAME / SCOPE_CACHE_FILENAME + ) assert (not index.completeness.complete and not cache_path.exists()) is ( test_case.expected_result From c082dbe62f57e422adc7c9e28eafda484632becb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:09:04 +0100 Subject: [PATCH 3/5] fix(native): name engine-namespaced cache paths and test native discovery on Python 3.13 and 3.14 Docs now name the native engine's compile reuse and Rules cache directories, the unused Python-engine reuse path constant is removed, and a CI job runs the native text oracles and a bounded engine differential on Python 3.13 and 3.14. --- .github/workflows/verify.yml | 44 +++++++++++++++++++ .../sqlbuild/references/docs/cli/compile.md | 2 +- .../concepts/rules/execution-and-caching.md | 2 +- src/sqlbuild/cli/compile_reuse/constants.py | 6 --- .../cli/commands/main/compile/helpers.py | 8 ++-- website/src/content/docs/docs/cli/compile.mdx | 2 +- .../concepts/rules/execution-and-caching.mdx | 2 +- 7 files changed, 53 insertions(+), 13 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 1645e78582..d3a210bd92 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -296,6 +296,50 @@ jobs: COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage --require-render-coverage" + compiler_differential_python_versions: + name: Native discovery on Python ${{ matrix.python }} + needs: [changes, lockfile] + if: needs.changes.outputs.compiler_differential == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + python: ["3.13", "3.14"] + env: + UV_PYTHON: ${{ matrix.python }} + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + enable-cache: true + cache-dependency-glob: | + uv.lock + Cargo.lock + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - name: Install Python + run: uv python install ${{ matrix.python }} + - name: Install dependencies + run: uv sync --locked --all-groups --all-extras --python ${{ matrix.python }} + - name: Require native discovery to support this Python + run: >- + uv run --no-sync python -c "import sys, unicodedata, sqlbuild._native as n; + assert n.native_text_supported(sys.version_info[:2], unicodedata.unidata_version)" + - name: Compare native text helpers with this Python + run: >- + uv run --no-sync pytest + tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py + -n auto --dist loadfile + - name: Compare engines on fixtures, examples, the failure corpus and a few seeds + env: + PYTHONUNBUFFERED: "1" + run: >- + uv run --no-sync python -m scripts.run_compiler_differential + --corpus fixtures examples failures seeds --seeds 4 --jobs 4 + format_performance: name: Formatter performance guards needs: [changes, lockfile] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 7c89acd0ad..e28a528a3f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -118,7 +118,7 @@ and when a provider's settings use sources SQLBuild cannot list exactly, such as `settings_customise_sources` or command-line parsing. The reason is logged at debug level. SQLBuild keeps one stored result per project and target under -`target/cache/compiler/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is +`target/cache/compiler-native-v1/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is ignored and compile runs in full. Storing a result normally takes a fraction of a second after the report is ready. When it has to hash a lot of data, the project has very many files, or it records thousands of new model renders, compile prints `Recording compile for reuse...` and a completion diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md index fa461cda58..18b0c99bc1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md @@ -26,7 +26,7 @@ normal compiler before treating the complete project as valid. ## Dependency-aware caching -Rules cache under `target/rules-cache`. A cache identity includes the Rule code, implementation and +Rules cache under `target/rules-cache-native-v1`. A cache identity includes the Rule code, implementation and imported helpers, options, invocation subject, accessed compiler facts, dialect, tracked project observations, and compatibility versions. The SQLBuild version in that identity also carries a hash of the native engine's source, so a local development build never reuses results from a different build diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index e8896df728..0f9e81a3b0 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -14,12 +14,6 @@ REUSE_FORMAT_VERSION: int = 1 REUSE_ENTRY_DIRECTORY_NAME: str = "project-reuse-v1" -REUSE_ENTRY_DIRECTORY_PARTS: tuple[str, ...] = ( - "target", - "cache", - "compiler", - REUSE_ENTRY_DIRECTORY_NAME, -) REUSE_ENTRY_MAGIC: bytes = b"SQBREUSE1\n" REUSE_ENTRY_SUFFIX: str = ".entry" REUSE_STDOUT_SUFFIX: str = ".stdout" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 08293a650f..4e793077a0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -50,7 +50,7 @@ from sqlbuild.cli.compile_reuse.classes.stored_artifacts import StoredArtifacts from sqlbuild.cli.compile_reuse.constants import ( REUSE_DISABLE_ENV_VAR, - REUSE_ENTRY_DIRECTORY_PARTS, + REUSE_ENTRY_DIRECTORY_NAME, REUSE_RENDER_STATE_SUFFIX, ) from sqlbuild.cli.compile_reuse.models import StoredCompileHeader, StoredCompileInputs @@ -3135,7 +3135,7 @@ def compile_reuse_entry_paths(*, project_dir: Path) -> tuple[Path, ...]: """Return every stored compile reuse entry of a project.""" return tuple( - sorted((compiler_cache_directory(project_dir) / "project-reuse-v1").glob("*.entry")) + sorted((compiler_cache_directory(project_dir) / REUSE_ENTRY_DIRECTORY_NAME).glob("*.entry")) ) @@ -4089,7 +4089,9 @@ def render_store_files(project_dir: Path) -> int: return len( list( - project_dir.joinpath(*REUSE_ENTRY_DIRECTORY_PARTS).glob(f"*{REUSE_RENDER_STATE_SUFFIX}") + (compiler_cache_directory(project_dir) / REUSE_ENTRY_DIRECTORY_NAME).glob( + f"*{REUSE_RENDER_STATE_SUFFIX}" + ) ) ) diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index fd4852890a..6c24b05777 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -109,7 +109,7 @@ and when a provider's settings use sources SQLBuild cannot list exactly, such as `settings_customise_sources` or command-line parsing. The reason is logged at debug level. SQLBuild keeps one stored result per project and target under -`target/cache/compiler/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is +`target/cache/compiler-native-v1/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is ignored and compile runs in full. Storing a result normally takes a fraction of a second after the report is ready. When it has to hash a lot of data, the project has very many files, or it records thousands of new model renders, compile prints `Recording compile for reuse...` and a completion diff --git a/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx b/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx index dcb0247ba5..dbc194d6ca 100644 --- a/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx +++ b/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx @@ -23,7 +23,7 @@ normal compiler before treating the complete project as valid. ## Dependency-aware caching -Rules cache under `target/rules-cache`. A cache identity includes the Rule code, implementation and +Rules cache under `target/rules-cache-native-v1`. A cache identity includes the Rule code, implementation and imported helpers, options, invocation subject, accessed compiler facts, dialect, tracked project observations, and compatibility versions. The SQLBuild version in that identity also carries a hash of the native engine's source, so a local development build never reuses results from a different build From 54c5320059ef853b35ad7d62a0e1316744d19afb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:03:39 +0100 Subject: [PATCH 4/5] ci: generate the base benchmark project with the base checkout's own environment --- .github/workflows/verify.yml | 2 +- scripts/compile_performance_ratio/main/compare.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index d3a210bd92..f950f7e307 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -532,7 +532,7 @@ jobs: run: uv sync --locked --all-groups --all-extras - name: Install base dependencies working-directory: base - run: uv sync --locked --all-extras + run: uv sync --locked --all-groups --all-extras - name: Compare base and head cold, warm and one-model edit compiles env: KIND: ${{ matrix.kind }} diff --git a/scripts/compile_performance_ratio/main/compare.py b/scripts/compile_performance_ratio/main/compare.py index 237879c674..fec5e8d32c 100644 --- a/scripts/compile_performance_ratio/main/compare.py +++ b/scripts/compile_performance_ratio/main/compare.py @@ -43,7 +43,7 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: print(f"Generating the base project with {args.base_root}'s generator", file=sys.stderr) write_base_benchmark_project( base_root=args.base_root.absolute(), - python=args.head_python.absolute(), + python=args.base_python.absolute(), kind=args.kind, project_dir=base_project_dir, models=args.models, From b6aec6a58277b26da5fa47b322bcaeed0633f557 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:42:35 +0100 Subject: [PATCH 5/5] test(lineage): read the structural cache from the active engine's directory --- .../src/sqlbuild/cli/commands/main/lineage/test_lineage.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py b/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py index c783acc950..6f87aa44dc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild.compiler.frontier.main.engine_cache_name import engine_cache_name from tests.e2e.src.sqlbuild.cli.commands.main.lineage._test_types import ( ColumnLineageCacheCliTestCase, DiamondLineageTreeCliTestCase, @@ -34,7 +35,9 @@ r"^[│ ]*[├└]── (?![^\n]*\(already shown\))[^\n]*?(orders_(?:left_|right_)?\d+(?:\.amount)?)", re.MULTILINE, ) -_LINEAGE_CACHE_RELATIVE_PATH: Path = Path("target/cache/lineage/v1/structural-graph.sqlite3") +_LINEAGE_CACHE_RELATIVE_PATH: Path = ( + Path("target/cache") / engine_cache_name("lineage") / "v1" / "structural-graph.sqlite3" +) @pytest.mark.parametrize(